diff --git a/0005-haoc-kernel.patch b/0005-haoc-kernel.patch index afefa7e513879234b5ac1fc0e9042527f2b96bdd..2a07de101306b9f446181bfaafec0f4d5e9b50e7 100644 --- a/0005-haoc-kernel.patch +++ b/0005-haoc-kernel.patch @@ -1,19899 +1,4380 @@ -From 89f8508281edfaa2ab98e49fee6759cf82452dea Mon Sep 17 00:00:00 2001 -From: liuzh -Date: Thu, 12 Dec 2024 02:23:00 +0000 -Subject: [PATCH] Publish k1 x86 and Arm64 code on 6.6.0-69.0.0 +From 1c983610e5d0e64bfa498529e4f8c09c8f9fd70e Mon Sep 17 00:00:00 2001 +From: sh +Date: Wed, 12 Feb 2025 18:01:42 +0800 +Subject: [PATCH] Haoc: Add support for x86 Isolated Execution Environment - internal branch: ptdesc-test4 - internal commit: 1aaccdb +Support Isolated Execution Environment for x86. +IEE depends on CR0.wp - disable KOI. +Signed-off-by: sh +Signed-off-by: Liu Zhehui + +HAOC: Add support for AArch64 Isolated Execution Environment. + +Co-authored-by: ljl +Co-authored-by: zsy +Signed-off-by: Liu Zhehui + +Haoc: Add support for x86 Sensitive Instruction Protection + +Support Sensitive Instruction Protection for x86 +SIP depends on IEE + +Signed-off-by: hubing +Signed-off-by: Liu Zhehui + +HAOC: Add support for ARM64 Sensitive Instruction Protection(IEE_SIP). + +Signed-off-by: ljl +Signed-off-by: Liu Zhehui + +Haoc: Support pointer protection for x86 IEE. + +Support IEE token to protect pgd pointer + +Signed-off-by: sh +Signed-off-by: Liu Zhehui + +HAOC: Support ARM64 pointer protection for IEE.(IEE_PTRP) + +Co-authored-by: ljl +Co-authored-by: zsy +Signed-off-by: Liu Zhehui + +HAOC: Support ARM64 Page Table Protection.(PTP) + +Signed-off-by: zsy +Signed-off-by: Liu Zhehui + +Haoc: Support page table protection for x86 IEE. + +Support page table protection + +Signed-off-by: sh +Signed-off-by: Liu Zhehui --- - Makefile | 7 + - arch/arm64/Kconfig | 35 + - arch/arm64/configs/openeuler_defconfig | 9 +- - arch/arm64/include/asm/assembler.h | 67 + - arch/arm64/include/asm/daifflags.h | 16 + - arch/arm64/include/asm/efi.h | 4 + - arch/arm64/include/asm/fixmap.h | 3 + - arch/arm64/include/asm/hw_breakpoint.h | 12 + - arch/arm64/include/asm/iee-access.h | 46 + - arch/arm64/include/asm/iee-cred.h | 150 ++ - arch/arm64/include/asm/iee-def.h | 130 ++ - arch/arm64/include/asm/iee-key.h | 149 ++ - arch/arm64/include/asm/iee-koi.h | 13 + - arch/arm64/include/asm/iee-selinuxp.h | 27 + - arch/arm64/include/asm/iee-si.h | 61 + - arch/arm64/include/asm/iee-slab.h | 23 + - arch/arm64/include/asm/iee-token.h | 33 + - arch/arm64/include/asm/iee.h | 10 + - arch/arm64/include/asm/kernel-pgtable.h | 21 + - arch/arm64/include/asm/koi.h | 532 +++++ - arch/arm64/include/asm/memory.h | 28 + - arch/arm64/include/asm/mmu_context.h | 20 + - arch/arm64/include/asm/pgalloc.h | 4 + - arch/arm64/include/asm/pgtable-hwdef.h | 14 + - arch/arm64/include/asm/pgtable.h | 191 +- - arch/arm64/include/asm/pgtable_slab.h | 8 + - arch/arm64/include/asm/pointer_auth.h | 5 + - arch/arm64/include/asm/stack_slab.h | 8 + - arch/arm64/include/asm/sysreg.h | 58 + - arch/arm64/include/asm/tlb.h | 20 + - arch/arm64/include/asm/tlbflush.h | 56 +- - arch/arm64/kernel/Makefile | 3 + - arch/arm64/kernel/armv8_deprecated.c | 16 + - arch/arm64/kernel/asm-offsets.c | 13 + - arch/arm64/kernel/cpu_errata.c | 12 + - arch/arm64/kernel/cpufeature.c | 80 + - arch/arm64/kernel/debug-monitors.c | 4 + - arch/arm64/kernel/entry-common.c | 4 + - arch/arm64/kernel/entry.S | 623 +++++- - arch/arm64/kernel/fpsimd.c | 4 + - arch/arm64/kernel/head.S | 5 + - arch/arm64/kernel/hibernate.c | 9 + - arch/arm64/kernel/hw_breakpoint.c | 99 + - arch/arm64/kernel/iee/Makefile | 5 + - arch/arm64/kernel/iee/iee-func.c | 722 +++++++ - arch/arm64/kernel/iee/iee-gate.S | 314 +++ - arch/arm64/kernel/iee/iee-pgtable.c | 402 ++++ - arch/arm64/kernel/iee/iee-selinuxp.c | 36 + - arch/arm64/kernel/iee/iee.c | 1720 +++++++++++++++ - arch/arm64/kernel/iee/pgtable_slab.c | 107 + - arch/arm64/kernel/iee/stack_slab.c | 19 + - arch/arm64/kernel/irq.c | 4 +- - arch/arm64/kernel/koi/Makefile | 1 + - arch/arm64/kernel/koi/koi.c | 1688 +++++++++++++++ - arch/arm64/kernel/mte.c | 5 + - arch/arm64/kernel/process.c | 13 +- - arch/arm64/kernel/proton-pack.c | 8 + - arch/arm64/kernel/setup.c | 34 + - arch/arm64/kernel/sfi_bpf_arch.c | 85 + - arch/arm64/kernel/traps.c | 26 + - arch/arm64/kernel/vmlinux.lds.S | 55 + - arch/arm64/mm/context.c | 125 +- - arch/arm64/mm/fault.c | 28 + - arch/arm64/mm/fixmap.c | 75 +- - arch/arm64/mm/init.c | 40 + - arch/arm64/mm/mmu.c | 1916 +++++++++++++++-- - arch/arm64/mm/pgd.c | 32 + - arch/arm64/mm/proc.S | 8 + - arch/arm64/mm/trans_pgd.c | 25 +- - arch/arm64/net/bpf_jit_comp.c | 105 + - arch/x86/Kconfig | 25 + - arch/x86/boot/compressed/ident_map_64.c | 34 + - arch/x86/boot/compressed/pgtable_64.c | 4 + - arch/x86/entry/entry_64.S | 32 + - arch/x86/include/asm/desc.h | 18 + - arch/x86/include/asm/fixmap.h | 5 + - arch/x86/include/asm/iee-access.h | 39 + - arch/x86/include/asm/iee-cred.h | 148 ++ - arch/x86/include/asm/iee-def.h | 124 ++ - arch/x86/include/asm/iee-key.h | 147 ++ - arch/x86/include/asm/iee-koi.h | 5 + - arch/x86/include/asm/iee-selinuxp.h | 26 + - arch/x86/include/asm/iee-si.h | 25 + - arch/x86/include/asm/iee-slab.h | 21 + - arch/x86/include/asm/iee-token.h | 32 + - arch/x86/include/asm/iee.h | 5 + - arch/x86/include/asm/koi.h | 432 ++++ - arch/x86/include/asm/page.h | 20 + - arch/x86/include/asm/page_64.h | 4 + - arch/x86/include/asm/page_types.h | 7 + - arch/x86/include/asm/pgalloc.h | 73 + - arch/x86/include/asm/pgtable.h | 77 +- - arch/x86/include/asm/pgtable_64.h | 119 + - arch/x86/include/asm/pgtable_64_types.h | 5 + - arch/x86/include/asm/pgtable_slab.h | 13 + - arch/x86/include/asm/special_insns.h | 15 + - arch/x86/include/asm/stack_slab.h | 8 + - arch/x86/include/asm/tlb.h | 6 + - arch/x86/kernel/Makefile | 5 + - arch/x86/kernel/asm-offsets.c | 6 + - arch/x86/kernel/cpu/common.c | 12 + - arch/x86/kernel/espfix_64.c | 14 + - arch/x86/kernel/head64.c | 17 + - arch/x86/kernel/idt.c | 8 +- - arch/x86/kernel/iee/Makefile | 4 + - arch/x86/kernel/iee/iee-func.c | 365 ++++ - arch/x86/kernel/iee/iee-gate.S | 209 ++ - arch/x86/kernel/iee/iee-selinuxp.c | 36 + - arch/x86/kernel/iee/iee.c | 1048 +++++++++ - arch/x86/kernel/iee/pgtable-slab.c | 169 ++ - arch/x86/kernel/iee/stack-slab.c | 25 + - arch/x86/kernel/koi/Makefile | 1 + - arch/x86/kernel/koi/koi.c | 1414 ++++++++++++ - arch/x86/kernel/ldt.c | 4 + - arch/x86/kernel/machine_kexec_64.c | 40 +- - arch/x86/kernel/paravirt.c | 4 + - arch/x86/kernel/setup.c | 174 ++ - arch/x86/kernel/sfi_bpf_arch.c | 85 + - arch/x86/kernel/vmlinux.lds.S | 78 + - arch/x86/mm/fault.c | 3 + - arch/x86/mm/ident_map_for_iee.c | 197 ++ - arch/x86/mm/init.c | 659 ++++++ - arch/x86/mm/init_64.c | 341 +++ - arch/x86/mm/ioremap.c | 37 + - arch/x86/mm/kaslr.c | 17 + - arch/x86/mm/mm_internal.h | 9 + - arch/x86/mm/pat/set_memory.c | 72 +- - arch/x86/mm/pgtable.c | 75 + - arch/x86/mm/pti.c | 52 + - arch/x86/net/bpf_jit_comp.c | 62 + - arch/x86/platform/efi/efi_64.c | 25 + - arch/x86/power/hibernate_64.c | 18 + - arch/x86/xen/mmu_pv.c | 6 + - block/sed-opal.c | 8 + - certs/blacklist.c | 8 + - certs/system_keyring.c | 18 + - crypto/af_alg.c | 16 + - crypto/asymmetric_keys/asymmetric_type.c | 20 + - crypto/asymmetric_keys/public_key.c | 16 + - crypto/asymmetric_keys/signature.c | 10 + - drivers/firmware/efi/arm-runtime.c | 4 + - drivers/firmware/efi/memmap.c | 20 + - drivers/iommu/amd/io_pgtable.c | 4 + - drivers/md/Makefile | 1 + - drivers/md/dm-crypt.c | 20 + - drivers/md/dm-verity-verify-sig.c | 8 + - drivers/md/dm-zero.c | 2 +- - drivers/nvdimm/security.c | 24 + - drivers/rtc/rtc-test.c | 2 +- - drivers/rtc/rtc-test_glue.h | 13 + - drivers/tty/serial/earlycon.c | 4 + - drivers/usb/early/ehci-dbgp.c | 4 + - fs/coredump.c | 8 + - fs/crypto/keyring.c | 13 + - fs/crypto/keysetup_v1.c | 12 + - fs/ecryptfs/ecryptfs_kernel.h | 4 + - fs/ecryptfs/keystore.c | 52 + - fs/ecryptfs/main.c | 4 + - fs/exec.c | 23 + - fs/nfs/flexfilelayout/flexfilelayout.c | 9 + - fs/nfs/nfs4idmap.c | 24 + - fs/nfsd/auth.c | 38 + - fs/nfsd/nfs4callback.c | 12 +- - fs/nfsd/nfs4recover.c | 9 + - fs/nfsd/nfsfh.c | 9 + - fs/open.c | 26 + - fs/overlayfs/dir.c | 9 + - fs/overlayfs/super.c | 12 + - fs/smb/client/cifs_spnego.c | 29 + - fs/smb/client/cifsacl.c | 50 + - fs/smb/client/connect.c | 8 + - fs/smb/client/sess.c | 4 + - fs/smb/client/smb2pdu.c | 4 + - fs/ubifs/auth.c | 8 + - fs/verity/signature.c | 4 + - include/asm-generic/early_ioremap.h | 3 + - include/asm-generic/fixmap.h | 18 + - include/asm-generic/memory_model.h | 20 + - include/asm-generic/pgalloc.h | 7 + - include/asm-generic/pgtable-nop4d.h | 5 + - include/asm-generic/tlb.h | 26 +- - include/asm-generic/vmlinux.lds.h | 34 +- - include/keys/asymmetric-subtype.h | 4 + - include/keys/asymmetric-type.h | 8 + - include/keys/request_key_auth-type.h | 4 + - include/linux/bpf.h | 18 + - include/linux/bpf_verifier.h | 19 + - include/linux/cred.h | 45 +- - include/linux/efi.h | 9 + - include/linux/filter.h | 3 + - include/linux/iee-func.h | 31 + - include/linux/key.h | 86 + - include/linux/mm.h | 59 + - include/linux/mm_types.h | 39 + - include/linux/module.h | 1 + - include/linux/pgtable.h | 32 + - include/linux/sched.h | 19 + - include/linux/sfi_bpf.h | 86 + - include/linux/skbuff.h | 3 + - include/uapi/linux/bpf.h | 1 + - include/uapi/linux/bpf_common.h | 1 + - init/main.c | 32 +- - kernel/bpf/Kconfig | 11 + - kernel/bpf/Makefile | 2 + - kernel/bpf/arraymap.c | 194 +- - kernel/bpf/hashtab.c | 323 ++- - kernel/bpf/sfi_bpf.c | 1387 ++++++++++++ - kernel/bpf/verifier.c | 360 +++- - kernel/cred.c | 184 ++ - kernel/exit.c | 8 + - kernel/fork.c | 284 ++- - kernel/groups.c | 7 + - kernel/kthread.c | 12 + - kernel/module/main.c | 105 +- - kernel/sys.c | 107 + - kernel/umh.c | 10 + - kernel/user_namespace.c | 18 + - lib/digsig.c | 8 + - mm/Kconfig | 5 + - mm/damon/ops-common.c | 1 + - mm/debug_vm_pgtable.c | 32 + - mm/early_ioremap.c | 57 + - mm/huge_memory.c | 26 +- - mm/hugetlb_vmemmap.c | 2 +- - mm/init-mm.c | 16 + - mm/memory.c | 14 + - mm/mmap.c | 14 + - mm/mmu_gather.c | 176 ++ - mm/pgtable-generic.c | 29 + - mm/slab.h | 39 +- - mm/slab_common.c | 39 + - mm/slub.c | 608 +++++- - mm/sparse-vmemmap.c | 12 + - mm/vmalloc.c | 2 +- - net/ceph/ceph_common.c | 4 + - net/ceph/crypto.c | 4 + - net/core/filter.c | 18 +- - net/dns_resolver/dns_key.c | 24 + - net/dns_resolver/dns_query.c | 18 + - net/rxrpc/af_rxrpc.c | 4 + - net/rxrpc/conn_event.c | 5 + - net/rxrpc/key.c | 16 + - net/rxrpc/rxkad.c | 40 + - net/rxrpc/security.c | 8 + - net/rxrpc/sendmsg.c | 4 + - net/rxrpc/server_key.c | 8 + - net/sched/cls_bpf.c | 78 + - net/sunrpc/auth.c | 14 + - security/commoncap.c | 169 ++ - security/integrity/evm/evm_crypto.c | 12 + - security/keys/big_key.c | 16 + - security/keys/dh.c | 8 + - security/keys/encrypted-keys/encrypted.c | 28 + - .../keys/encrypted-keys/masterkey_trusted.c | 5 + - security/keys/gc.c | 52 + - security/keys/internal.h | 8 + - security/keys/key.c | 237 +- - security/keys/keyctl.c | 91 + - security/keys/keyring.c | 269 +++ - security/keys/proc.c | 28 + - security/keys/process_keys.c | 88 + - security/keys/request_key.c | 16 + - security/keys/request_key_auth.c | 20 + - security/keys/trusted-keys/trusted_core.c | 8 + - security/keys/user_defined.c | 12 + - security/security.c | 15 + - security/selinux/hooks.c | 60 + - security/selinux/ima.c | 20 + - security/selinux/include/security.h | 8 + - security/selinux/selinuxfs.c | 66 + - security/selinux/ss/services.c | 56 + - security/selinux/status.c | 32 + - security/smack/smack_lsm.c | 12 + - 273 files changed, 23476 insertions(+), 373 deletions(-) - create mode 100644 arch/arm64/include/asm/iee-access.h - create mode 100644 arch/arm64/include/asm/iee-cred.h - create mode 100644 arch/arm64/include/asm/iee-def.h - create mode 100644 arch/arm64/include/asm/iee-key.h - create mode 100644 arch/arm64/include/asm/iee-koi.h - create mode 100644 arch/arm64/include/asm/iee-selinuxp.h - create mode 100644 arch/arm64/include/asm/iee-si.h - create mode 100644 arch/arm64/include/asm/iee-slab.h - create mode 100644 arch/arm64/include/asm/iee-token.h - create mode 100644 arch/arm64/include/asm/iee.h - create mode 100644 arch/arm64/include/asm/koi.h - create mode 100644 arch/arm64/include/asm/pgtable_slab.h - create mode 100644 arch/arm64/include/asm/stack_slab.h - create mode 100644 arch/arm64/kernel/iee/Makefile - create mode 100644 arch/arm64/kernel/iee/iee-func.c - create mode 100644 arch/arm64/kernel/iee/iee-gate.S - create mode 100644 arch/arm64/kernel/iee/iee-pgtable.c - create mode 100644 arch/arm64/kernel/iee/iee-selinuxp.c - create mode 100644 arch/arm64/kernel/iee/iee.c - create mode 100644 arch/arm64/kernel/iee/pgtable_slab.c - create mode 100644 arch/arm64/kernel/iee/stack_slab.c - create mode 100644 arch/arm64/kernel/koi/Makefile - create mode 100644 arch/arm64/kernel/koi/koi.c - create mode 100644 arch/arm64/kernel/sfi_bpf_arch.c - create mode 100644 arch/x86/include/asm/iee-access.h - create mode 100644 arch/x86/include/asm/iee-cred.h - create mode 100644 arch/x86/include/asm/iee-def.h - create mode 100644 arch/x86/include/asm/iee-key.h - create mode 100644 arch/x86/include/asm/iee-koi.h - create mode 100644 arch/x86/include/asm/iee-selinuxp.h - create mode 100644 arch/x86/include/asm/iee-si.h - create mode 100644 arch/x86/include/asm/iee-slab.h - create mode 100644 arch/x86/include/asm/iee-token.h - create mode 100644 arch/x86/include/asm/iee.h - create mode 100644 arch/x86/include/asm/koi.h - create mode 100644 arch/x86/include/asm/pgtable_slab.h - create mode 100644 arch/x86/include/asm/stack_slab.h - create mode 100644 arch/x86/kernel/iee/Makefile - create mode 100644 arch/x86/kernel/iee/iee-func.c - create mode 100644 arch/x86/kernel/iee/iee-gate.S - create mode 100644 arch/x86/kernel/iee/iee-selinuxp.c - create mode 100644 arch/x86/kernel/iee/iee.c - create mode 100644 arch/x86/kernel/iee/pgtable-slab.c - create mode 100644 arch/x86/kernel/iee/stack-slab.c - create mode 100644 arch/x86/kernel/koi/Makefile - create mode 100644 arch/x86/kernel/koi/koi.c - create mode 100644 arch/x86/kernel/sfi_bpf_arch.c - create mode 100644 arch/x86/mm/ident_map_for_iee.c - create mode 100644 drivers/rtc/rtc-test_glue.h - create mode 100644 include/linux/iee-func.h - create mode 100644 include/linux/sfi_bpf.h - create mode 100644 kernel/bpf/sfi_bpf.c + arch/arm64/Kconfig | 4 + + arch/arm64/include/asm/assembler.h | 6 + + arch/arm64/include/asm/fixmap.h | 4 + + arch/arm64/include/asm/haoc/haoc-bitmap.h | 78 + + arch/arm64/include/asm/haoc/haoc-def.h | 128 ++ + arch/arm64/include/asm/haoc/haoc.h | 144 ++ + arch/arm64/include/asm/haoc/iee-access.h | 25 + + arch/arm64/include/asm/haoc/iee-asm.h | 53 + + arch/arm64/include/asm/haoc/iee-cred.h | 235 +++ + arch/arm64/include/asm/haoc/iee-fixmap.h | 23 + + arch/arm64/include/asm/haoc/iee-func.h | 18 + + arch/arm64/include/asm/haoc/iee-init.h | 21 + + arch/arm64/include/asm/haoc/iee-key.h | 151 ++ + arch/arm64/include/asm/haoc/iee-mmu.h | 24 + + arch/arm64/include/asm/haoc/iee-pgtable.h | 1732 ++++++++++++++++++++ + arch/arm64/include/asm/haoc/iee-ptp-init.h | 56 + + arch/arm64/include/asm/haoc/iee-selinux.h | 53 + + arch/arm64/include/asm/haoc/iee-si.h | 45 + + arch/arm64/include/asm/haoc/iee-token.h | 70 + + arch/arm64/include/asm/haoc/iee-vmlinux.h | 54 + + arch/arm64/include/asm/haoc/iee.h | 70 + + arch/arm64/include/asm/memory.h | 4 + + arch/arm64/include/asm/mmu_context.h | 15 + + arch/arm64/include/asm/pgtable.h | 6 + + arch/arm64/include/asm/tlb.h | 4 + + arch/arm64/kernel/Makefile | 1 + + arch/arm64/kernel/armv8_deprecated.c | 19 + + arch/arm64/kernel/cpu_errata.c | 15 + + arch/arm64/kernel/cpufeature.c | 43 + + arch/arm64/kernel/entry-common.c | 4 + + arch/arm64/kernel/entry.S | 6 + + arch/arm64/kernel/fpsimd.c | 7 + + arch/arm64/kernel/haoc/Kconfig | 49 + + arch/arm64/kernel/haoc/Makefile | 13 + + arch/arm64/kernel/haoc/credp/Makefile | 3 + + arch/arm64/kernel/haoc/credp/credp.c | 405 +++++ + arch/arm64/kernel/haoc/haoc-bitmap.c | 213 +++ + arch/arm64/kernel/haoc/haoc.c | 106 ++ + arch/arm64/kernel/haoc/iee/Makefile | 5 + + arch/arm64/kernel/haoc/iee/iee-func.c | 479 ++++++ + arch/arm64/kernel/haoc/iee/iee-gate.S | 92 ++ + arch/arm64/kernel/haoc/iee/iee-init.c | 56 + + arch/arm64/kernel/haoc/iee/iee-mmu.c | 867 ++++++++++ + arch/arm64/kernel/haoc/iee/iee-si-gate.S | 81 + + arch/arm64/kernel/haoc/iee/iee-si.c | 105 ++ + arch/arm64/kernel/haoc/iee/iee-token.c | 203 +++ + arch/arm64/kernel/haoc/iee/iee.c | 37 + + arch/arm64/kernel/haoc/keyp/Makefile | 1 + + arch/arm64/kernel/haoc/keyp/keyp.c | 264 +++ + arch/arm64/kernel/haoc/ptp/Makefile | 1 + + arch/arm64/kernel/haoc/ptp/iee-ptp-init.c | 406 +++++ + arch/arm64/kernel/haoc/ptp/ptp.c | 280 ++++ + arch/arm64/kernel/haoc/selinuxp/Makefile | 7 + + arch/arm64/kernel/haoc/selinuxp/selinuxp.c | 52 + + arch/arm64/kernel/mte.c | 8 + + arch/arm64/kernel/process.c | 4 + + arch/arm64/kernel/proton-pack.c | 11 + + arch/arm64/kernel/setup.c | 10 + + arch/arm64/kernel/vmlinux.lds.S | 8 + + arch/arm64/mm/context.c | 40 + + arch/arm64/mm/fault.c | 12 + + arch/arm64/mm/fixmap.c | 31 + + arch/arm64/mm/init.c | 4 + + arch/arm64/mm/mmu.c | 70 + + arch/arm64/mm/pgd.c | 11 + + arch/arm64/mm/proc.S | 15 + + arch/arm64/mm/trans_pgd.c | 4 + + arch/x86/Kconfig | 21 + + arch/x86/boot/compressed/ident_map_64.c | 34 + + arch/x86/boot/compressed/pgtable_64.c | 4 + + arch/x86/include/asm/desc.h | 15 + + arch/x86/include/asm/haoc/haoc-def.h | 38 + + arch/x86/include/asm/haoc/haoc.h | 34 + + arch/x86/include/asm/haoc/iee-access.h | 29 + + arch/x86/include/asm/haoc/iee-func.h | 11 + + arch/x86/include/asm/haoc/iee-si.h | 43 + + arch/x86/include/asm/haoc/iee-token.h | 53 + + arch/x86/include/asm/haoc/iee.h | 26 + + arch/x86/include/asm/haoc/ptp.h | 64 + + arch/x86/include/asm/pgalloc.h | 11 + + arch/x86/include/asm/pgtable.h | 33 + + arch/x86/include/asm/pgtable_64.h | 39 + + arch/x86/include/asm/special_insns.h | 14 + + arch/x86/include/asm/tlb.h | 8 + + arch/x86/kernel/Makefile | 1 + + arch/x86/kernel/alternative.c | 27 + + arch/x86/kernel/asm-offsets.c | 9 + + arch/x86/kernel/cpu/common.c | 24 + + arch/x86/kernel/haoc/Makefile | 3 + + arch/x86/kernel/haoc/haoc.c | 28 + + arch/x86/kernel/haoc/iee/Makefile | 4 + + arch/x86/kernel/haoc/iee/iee-func.c | 71 + + arch/x86/kernel/haoc/iee/iee-gate.S | 145 ++ + arch/x86/kernel/haoc/iee/iee-init.c | 157 ++ + arch/x86/kernel/haoc/iee/iee-si.c | 127 ++ + arch/x86/kernel/haoc/iee/iee-token.c | 181 ++ + arch/x86/kernel/haoc/iee/iee.c | 24 + + arch/x86/kernel/haoc/ptp/Makefile | 1 + + arch/x86/kernel/haoc/ptp/ident_map.c | 182 ++ + arch/x86/kernel/haoc/ptp/ptp-gate.S | 133 ++ + arch/x86/kernel/haoc/ptp/ptp.c | 358 ++++ + arch/x86/kernel/head64.c | 4 + + arch/x86/kernel/paravirt.c | 4 + + arch/x86/kernel/vmlinux.lds.S | 22 + + arch/x86/mm/init_64.c | 20 + + arch/x86/mm/pgtable.c | 43 + + arch/x86/mm/pti.c | 32 + + arch/x86/platform/efi/efi_64.c | 24 + + drivers/firmware/efi/arm-runtime.c | 4 + + drivers/tty/serial/earlycon.c | 4 + + drivers/usb/early/ehci-dbgp.c | 4 + + fs/exec.c | 23 + + include/asm-generic/tlb.h | 25 + + include/linux/efi.h | 3 + + include/linux/mm.h | 11 + + include/linux/mm_types.h | 3 + + include/linux/ptp-cache.h | 37 + + kernel/exit.c | 6 + + kernel/fork.c | 31 + + kernel/kthread.c | 9 + + kernel/sched/core.c | 18 + + mm/Makefile | 1 + + mm/debug_vm_pgtable.c | 8 + + mm/early_ioremap.c | 22 + + mm/haoc/Makefile | 1 + + mm/haoc/ptp-pg_cache.c | 251 +++ + mm/huge_memory.c | 10 + + mm/memory.c | 14 + + mm/mmap.c | 35 + + mm/mmu_gather.c | 196 +++ + mm/mprotect.c | 12 + + mm/pgtable-generic.c | 25 + + mm/slub.c | 34 + + mm/sparse-vmemmap.c | 20 +- + 134 files changed, 9978 insertions(+), 1 deletion(-) + create mode 100644 arch/arm64/include/asm/haoc/haoc-bitmap.h + create mode 100644 arch/arm64/include/asm/haoc/haoc-def.h + create mode 100644 arch/arm64/include/asm/haoc/haoc.h + create mode 100644 arch/arm64/include/asm/haoc/iee-access.h + create mode 100644 arch/arm64/include/asm/haoc/iee-asm.h + create mode 100644 arch/arm64/include/asm/haoc/iee-cred.h + create mode 100644 arch/arm64/include/asm/haoc/iee-fixmap.h + create mode 100644 arch/arm64/include/asm/haoc/iee-func.h + create mode 100644 arch/arm64/include/asm/haoc/iee-init.h + create mode 100644 arch/arm64/include/asm/haoc/iee-key.h + create mode 100644 arch/arm64/include/asm/haoc/iee-mmu.h + create mode 100644 arch/arm64/include/asm/haoc/iee-pgtable.h + create mode 100644 arch/arm64/include/asm/haoc/iee-ptp-init.h + create mode 100644 arch/arm64/include/asm/haoc/iee-selinux.h + create mode 100644 arch/arm64/include/asm/haoc/iee-si.h + create mode 100644 arch/arm64/include/asm/haoc/iee-token.h + create mode 100644 arch/arm64/include/asm/haoc/iee-vmlinux.h + create mode 100644 arch/arm64/include/asm/haoc/iee.h + create mode 100644 arch/arm64/kernel/haoc/Kconfig + create mode 100644 arch/arm64/kernel/haoc/Makefile + create mode 100644 arch/arm64/kernel/haoc/credp/Makefile + create mode 100644 arch/arm64/kernel/haoc/credp/credp.c + create mode 100644 arch/arm64/kernel/haoc/haoc-bitmap.c + create mode 100644 arch/arm64/kernel/haoc/haoc.c + create mode 100644 arch/arm64/kernel/haoc/iee/Makefile + create mode 100644 arch/arm64/kernel/haoc/iee/iee-func.c + create mode 100644 arch/arm64/kernel/haoc/iee/iee-gate.S + create mode 100644 arch/arm64/kernel/haoc/iee/iee-init.c + create mode 100644 arch/arm64/kernel/haoc/iee/iee-mmu.c + create mode 100644 arch/arm64/kernel/haoc/iee/iee-si-gate.S + create mode 100644 arch/arm64/kernel/haoc/iee/iee-si.c + create mode 100644 arch/arm64/kernel/haoc/iee/iee-token.c + create mode 100644 arch/arm64/kernel/haoc/iee/iee.c + create mode 100644 arch/arm64/kernel/haoc/keyp/Makefile + create mode 100644 arch/arm64/kernel/haoc/keyp/keyp.c + create mode 100644 arch/arm64/kernel/haoc/ptp/Makefile + create mode 100644 arch/arm64/kernel/haoc/ptp/iee-ptp-init.c + create mode 100644 arch/arm64/kernel/haoc/ptp/ptp.c + create mode 100644 arch/arm64/kernel/haoc/selinuxp/Makefile + create mode 100644 arch/arm64/kernel/haoc/selinuxp/selinuxp.c + create mode 100644 arch/x86/include/asm/haoc/haoc-def.h + create mode 100644 arch/x86/include/asm/haoc/haoc.h + create mode 100644 arch/x86/include/asm/haoc/iee-access.h + create mode 100644 arch/x86/include/asm/haoc/iee-func.h + create mode 100644 arch/x86/include/asm/haoc/iee-si.h + create mode 100644 arch/x86/include/asm/haoc/iee-token.h + create mode 100644 arch/x86/include/asm/haoc/iee.h + create mode 100644 arch/x86/include/asm/haoc/ptp.h + create mode 100644 arch/x86/kernel/haoc/Makefile + create mode 100644 arch/x86/kernel/haoc/haoc.c + create mode 100644 arch/x86/kernel/haoc/iee/Makefile + create mode 100644 arch/x86/kernel/haoc/iee/iee-func.c + create mode 100644 arch/x86/kernel/haoc/iee/iee-gate.S + create mode 100644 arch/x86/kernel/haoc/iee/iee-init.c + create mode 100644 arch/x86/kernel/haoc/iee/iee-si.c + create mode 100644 arch/x86/kernel/haoc/iee/iee-token.c + create mode 100644 arch/x86/kernel/haoc/iee/iee.c + create mode 100644 arch/x86/kernel/haoc/ptp/Makefile + create mode 100644 arch/x86/kernel/haoc/ptp/ident_map.c + create mode 100644 arch/x86/kernel/haoc/ptp/ptp-gate.S + create mode 100644 arch/x86/kernel/haoc/ptp/ptp.c + create mode 100644 include/linux/ptp-cache.h + create mode 100644 mm/haoc/Makefile + create mode 100644 mm/haoc/ptp-pg_cache.c -diff --git a/Makefile b/Makefile -index ee377cec01f3..7b354f051344 100644 ---- a/Makefile -+++ b/Makefile -@@ -555,6 +555,9 @@ LINUXINCLUDE := \ - $(USERINCLUDE) - - KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE -+ifeq ($(ARCH), arm64) -+ KBUILD_AFLAGS += -march=armv8.1-a -+endif - - KBUILD_CFLAGS := - KBUILD_CFLAGS += -std=gnu11 -@@ -563,6 +566,10 @@ KBUILD_CFLAGS += -funsigned-char - KBUILD_CFLAGS += -fno-common - KBUILD_CFLAGS += -fno-PIE - KBUILD_CFLAGS += -fno-strict-aliasing -+ifeq ($(ARCH), arm64) -+ KBUILD_CFLAGS += -march=armv8.1-a -+endif -+#KBUILD_CFLAGS += -fPIC - - KBUILD_CPPFLAGS := -D__KERNEL__ - KBUILD_RUSTFLAGS := $(rust_common_flags) \ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig -index 9da9d58f1c02..730fb3c9a889 100644 +index c3b38c890b45..d434e8b6f1bd 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig -@@ -1776,6 +1776,41 @@ config UNMAP_KERNEL_AT_EL0 +@@ -2754,4 +2754,8 @@ source "drivers/acpi/Kconfig" - If unsure, say Y. + source "arch/arm64/kvm/Kconfig" -+# Config for iee -+config IEE -+ depends on ARM64 -+ depends on ARM64_PAN -+ depends on ARM64_VA_BITS_48 -+ depends on ARM64_4K_PAGES -+ def_bool y -+ -+# Config for support of interruption of iee -+config IEE_INTERRUPTABLE -+ depends on IEE -+ def_bool n -+ -+# Config for credentials isolation -+config CREDP -+ depends on IEE -+ def_bool y -+ -+# Config for kernel module isolation -+config KOI -+ depends on ARM64 -+ depends on ARM64_VA_BITS_48 -+ depends on ARM64_4K_PAGES -+ def_bool n ++source "arch/arm64/kernel/haoc/Kconfig" + -+config IEE_SELINUX_P -+ depends on IEE -+ depends on SECURITY_SELINUX -+ def_bool n -+ -+# Config for key isolation -+config KEYP -+ depends on IEE -+ def_bool y + source "arch/arm64/mm/Kconfig" + - config MITIGATE_SPECTRE_BRANCH_HISTORY - bool "Mitigate Spectre style attacks against branch history" if EXPERT - default y -diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig -index fc6053a500db..633b5c411662 100644 ---- a/arch/arm64/configs/openeuler_defconfig -+++ b/arch/arm64/configs/openeuler_defconfig -@@ -89,6 +89,7 @@ CONFIG_BPF_JIT_DEFAULT_ON=y - # CONFIG_BPF_PRELOAD is not set - CONFIG_BPF_LSM=y - CONFIG_BPF_SCHED=y -+CONFIG_HIVE=y - # end of BPF subsystem - - CONFIG_PREEMPT_NONE_BUILD=y -@@ -1389,7 +1390,7 @@ CONFIG_NETFILTER_NETLINK_ACCT=m - CONFIG_NETFILTER_NETLINK_QUEUE=m - CONFIG_NETFILTER_NETLINK_LOG=m - CONFIG_NETFILTER_NETLINK_OSF=m --CONFIG_NF_CONNTRACK=m -+CONFIG_NF_CONNTRACK=y - CONFIG_NF_LOG_SYSLOG=m - CONFIG_NETFILTER_CONNCOUNT=m - CONFIG_NF_CONNTRACK_MARK=y -@@ -1420,7 +1421,7 @@ CONFIG_NF_CT_NETLINK=m - CONFIG_NF_CT_NETLINK_TIMEOUT=m - CONFIG_NF_CT_NETLINK_HELPER=m - CONFIG_NETFILTER_NETLINK_GLUE_CT=y --CONFIG_NF_NAT=m -+CONFIG_NF_NAT=y - CONFIG_NF_NAT_AMANDA=m - CONFIG_NF_NAT_FTP=m - CONFIG_NF_NAT_IRC=m -@@ -1624,7 +1625,7 @@ CONFIG_IP_VS_PE_SIP=m - # - # IP: Netfilter Configuration - # --CONFIG_NF_DEFRAG_IPV4=m -+CONFIG_NF_DEFRAG_IPV4=y - CONFIG_NF_SOCKET_IPV4=m - CONFIG_NF_TPROXY_IPV4=m - CONFIG_NF_TABLES_IPV4=y -@@ -1696,7 +1697,7 @@ CONFIG_IP6_NF_TARGET_MASQUERADE=m - CONFIG_IP6_NF_TARGET_NPT=m - # end of IPv6: Netfilter Configuration - --CONFIG_NF_DEFRAG_IPV6=m -+CONFIG_NF_DEFRAG_IPV6=y - CONFIG_NF_TABLES_BRIDGE=m - # CONFIG_NFT_BRIDGE_META is not set - CONFIG_NFT_BRIDGE_REJECT=m ++source "arch/arm64/kernel/haoc/Kconfig" +\ No newline at end of file diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h -index 38b23786aeb4..6af10d509c2e 100644 +index f6f92c582e0f..20a6eb2c289e 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h -@@ -26,6 +26,41 @@ +@@ -25,6 +25,9 @@ + #include #include #include - +#ifdef CONFIG_IEE -+ .macro iee_si_restore_daif, flags:req -+ msr daifclr, #0xf -+ tbnz \flags, #6, 114221f -+ tbnz \flags, #7, 114210f -+ tbnz \flags, #8, 114100f -+ msr daifset, #0b000 -+ b 114514f -+114221: -+ tbnz \flags, #7, 114211f -+ tbnz \flags, #8, 114101f -+ msr daifset, #0b001 -+ b 114514f -+114211: -+ tbnz \flags, #8, 114111f -+ msr daifset, #0b011 -+ b 114514f -+114210: -+ tbnz \flags, #8, 114110f -+ msr daifset, #0b010 -+ b 114514f -+114100: -+ msr daifset, #0b100 -+ b 114514f -+114101: -+ msr daifset, #0b101 -+ b 114514f -+114110: -+ msr daifset, #0b110 -+ b 114514f -+114111: -+ msr daifset, #0b111 -+114514: -+ .endm ++#include +#endif + /* * Provide a wxN alias for each wN register so what we can paste a xN - * reference after a 'w' to obtain the 32-bit version. -@@ -52,7 +87,11 @@ alternative_else_nop_endif - - .macro disable_daif - disable_allint -+// #ifdef CONFIG_IEE -+// msr daifset, #0x7 -+// #else - msr daifset, #0xf -+// #endif - .endm - - .macro enable_daif -@@ -69,7 +108,11 @@ alternative_else_nop_endif - .endm - - .macro restore_irq, flags -+// #ifdef CONFIG_IEE -+// iee_si_restore_daif \flags -+// #else - msr daif, \flags -+// #endif - .endm - - .macro enable_dbg -@@ -77,20 +120,44 @@ alternative_else_nop_endif - .endm - - .macro disable_step_tsk, flgs, tmp -+// #ifdef CONFIG_IEE -+// 1145: -+// tbz \flgs, #TIF_SINGLESTEP, 9990f -+// mrs \tmp, mdscr_el1 -+// bic \tmp, \tmp, #DBG_MDSCR_SS -+// orr \tmp, \tmp, #DBG_MDSCR_MDE -+// msr mdscr_el1, \tmp -+// isb // Synchronise with enable_dbg -+// mrs \tmp, mdscr_el1 -+// tbz \tmp, #15, 1145b -+// #else - tbz \flgs, #TIF_SINGLESTEP, 9990f - mrs \tmp, mdscr_el1 - bic \tmp, \tmp, #DBG_MDSCR_SS - msr mdscr_el1, \tmp - isb // Synchronise with enable_dbg -+// #endif - 9990: - .endm - - /* call with daif masked */ - .macro enable_step_tsk, flgs, tmp -+// #ifdef CONFIG_IEE -+// 1146: -+// tbz \flgs, #TIF_SINGLESTEP, 9990f -+// mrs \tmp, mdscr_el1 -+// orr \tmp, \tmp, #DBG_MDSCR_SS -+// orr \tmp, \tmp, #DBG_MDSCR_MDE -+// msr mdscr_el1, \tmp -+// isb // Synchronise with enable_dbg -+// mrs \tmp, mdscr_el1 -+// tbz \tmp, #15, 1146b -+// #else - tbz \flgs, #TIF_SINGLESTEP, 9990f - mrs \tmp, mdscr_el1 - orr \tmp, \tmp, #DBG_MDSCR_SS - msr mdscr_el1, \tmp -+// #endif - 9990: +@@ -496,6 +499,9 @@ alternative_endif + .macro load_ttbr1, pgtbl, tmp1, tmp2 + phys_to_ttbr \tmp1, \pgtbl + offset_ttbr1 \tmp1, \tmp2 ++ #ifdef CONFIG_IEE ++ orr \tmp1, \tmp1, #IEE_ASM_ASID ++ #endif + msr ttbr1_el1, \tmp1 + isb .endm - -diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h -index 2417cc6b1631..cb5b4c2e03b8 100644 ---- a/arch/arm64/include/asm/daifflags.h -+++ b/arch/arm64/include/asm/daifflags.h -@@ -26,11 +26,19 @@ static inline void local_daif_mask(void) - (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | - GIC_PRIO_PSR_I_SET))); - -+// #ifdef CONFIG_IEE -+// asm volatile( -+// "msr daifset, #0x7 // local_daif_mask\n" -+// : -+// : -+// : "memory"); -+// #else - asm volatile( - "msr daifset, #0xf // local_daif_mask\n" - : - : - : "memory"); -+// #endif - - /* Don't really care for a dsb here, we don't intend to enable IRQs */ - if (system_uses_irq_prio_masking()) -@@ -118,7 +126,11 @@ static inline void local_daif_restore(unsigned long flags) - gic_write_pmr(pmr); - } - -+// #ifdef CONFIG_IEE -+// iee_si_write_daif(flags); -+// #else - write_sysreg(flags, daif); -+// #endif - - /* If we can take asynchronous errors we can take NMIs */ - if (system_uses_nmi()) { -@@ -151,7 +163,11 @@ static inline void local_daif_inherit(struct pt_regs *regs) - * system_has_prio_mask_debugging() won't restore the I bit if it can - * use the pmr instead. - */ -+// #ifdef CONFIG_IEE -+// iee_si_write_daif(flags); -+// #else - write_sysreg(flags, daif); -+// #endif - - /* The ALLINT field is at the same position in pstate and ALLINT */ - if (system_uses_nmi()) { -diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h -index bcd5622aa096..76c4bd6c2b20 100644 ---- a/arch/arm64/include/asm/efi.h -+++ b/arch/arm64/include/asm/efi.h -@@ -58,7 +58,11 @@ void arch_efi_call_virt_teardown(void); - #define arch_efi_save_flags(state_flags) \ - ((void)((state_flags) = read_sysreg(daif))) - -+// #ifdef CONFIG_IEE -+// #define arch_efi_restore_flags(state_flags) iee_si_write_daif(state_flags) -+// #else - #define arch_efi_restore_flags(state_flags) write_sysreg(state_flags, daif) -+// #endif - - - /* arch specific definitions used by the stub code */ diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h -index 58c294a96676..095a0731dce3 100644 +index 58c294a96676..85d92fb42024 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h -@@ -108,6 +108,9 @@ void __init fixmap_copy(pgd_t *pgdir); - #define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) +@@ -109,6 +109,10 @@ void __init fixmap_copy(pgd_t *pgdir); extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); + +#ifdef CONFIG_PTP -+extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); ++#include +#endif - ++ #include -diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h -index 84055329cd8b..f72d89bb9a32 100644 ---- a/arch/arm64/include/asm/hw_breakpoint.h -+++ b/arch/arm64/include/asm/hw_breakpoint.h -@@ -104,6 +104,18 @@ static inline void decode_ctrl_reg(u32 reg, - write_sysreg(VAL, dbg##REG##N##_el1);\ - } while (0) - -+#ifdef CONFIG_IEE -+#define IEE_SI_AARCH64_DBG_READ(N, REG, VAL) do{\ -+ VAL = this_cpu_read(iee_si_user_##REG##N);\ -+} while (0) -+ -+#define IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL) do{\ -+ u64 __val = (u64)(VAL); \ -+ this_cpu_write(iee_si_user_##REG##N, __val);\ -+ iee_rwx_gate_entry(IEE_WRITE_AFSR0);\ -+} while (0) -+#endif -+ - struct task_struct; - struct notifier_block; - struct perf_event_attr; -diff --git a/arch/arm64/include/asm/iee-access.h b/arch/arm64/include/asm/iee-access.h + #endif /* !__ASSEMBLY__ */ +diff --git a/arch/arm64/include/asm/haoc/haoc-bitmap.h b/arch/arm64/include/asm/haoc/haoc-bitmap.h new file mode 100644 -index 000000000000..9be519b707c5 +index 000000000000..1e6926afaa2a --- /dev/null -+++ b/arch/arm64/include/asm/iee-access.h -@@ -0,0 +1,46 @@ -+#ifndef _LINUX_IEE_ACCESS_H -+#define _LINUX_IEE_ACCESS_H ++++ b/arch/arm64/include/asm/haoc/haoc-bitmap.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_HAOC_BITMAP_H ++#define _LINUX_HAOC_BITMAP_H ++ ++#include ++#include ++#include + -+#include -+#include ++#define HAOC_BITMAP_START VMEMMAP_END ++#define haoc_bitmap_base ((uint8_t *)HAOC_BITMAP_START - (memstart_addr >> PAGE_SHIFT)) + -+extern unsigned long long iee_rw_gate(int flag, ...); ++#define __pfn_to_haoc_bitmap(pfn) (haoc_bitmap_base + (pfn)) + -+#ifdef CONFIG_IEE -+static inline void iee_write_in_byte(void *ptr, u64 data, int length) ++#define __va_to_haoc_bitmap(va) ({ \ ++ uint8_t *__ret; \ ++ if (__is_lm_address((u64)va)) \ ++ __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa(va))); \ ++ else \ ++ __ret = __pfn_to_haoc_bitmap(PHYS_PFN(__pa_symbol(va))); \ ++ __ret; \ ++}) ++ ++/* HAOC_NORMAL means */ ++enum HAOC_BITMAP_TYPE { ++ IEE_NORMAL = 0, /* Non-IEE pages */ ++ IEE_DATA, ++ IEE_PGTABLE, ++ IEE_CRED, ++ IEE_KEY, ++ IEE_SELINUX, ++}; ++ ++extern bool haoc_bitmap_ready; ++ ++extern int haoc_bitmap_sparse_init(void); ++extern void haoc_bitmap_setup(void); ++extern void setup_iee_early_data_bitmap(void); ++ ++static inline enum HAOC_BITMAP_TYPE iee_get_bitmap_type(unsigned long va) +{ -+ iee_rw_gate(IEE_WRITE_IN_BYTE, ptr, data, length); ++ return *__va_to_haoc_bitmap(va); +} + -+static inline void iee_memset(void *ptr, int data, size_t n) ++void _iee_set_bitmap_type(unsigned long __unused, ++ u64 va, enum HAOC_BITMAP_TYPE type, int num_pages); ++ ++static inline void iee_set_bitmap_type(unsigned long va, ++ int num_pages, enum HAOC_BITMAP_TYPE type) +{ -+ iee_rw_gate(IEE_MEMSET, ptr, data, n); ++ // iee_rw_gate(IEE_OP_SET_BITMAP_TYPE, va, type, num_pages); +} + -+static inline void iee_memcpy(void *dst, const void *src, size_t n) ++static inline void iee_verify_type(unsigned long va, enum HAOC_BITMAP_TYPE type, ++ const char *name) +{ -+ iee_rw_gate(IEE_MEMCPY, dst, src, n); ++ // uint8_t bit_type = iee_get_bitmap_type(va); ++ ++ // if (unlikely(bit_type != type)) ++ // pr_err("IEE detected type: %d, fake %s: va(0x%lx)", bit_type, name, va); +} + -+static inline void iee_set_track(struct track *ptr, struct track *data) ++static inline void iee_verify_not_normal(void *start, void *end) +{ -+ iee_rw_gate(IEE_OP_SET_TRACK, ptr, data); ++ // unsigned long addr; ++ ++ // if (!haoc_bitmap_ready) ++ // return; ++ // addr = start; ++ // while (addr < end) { ++ // if (unlikely(iee_get_bitmap_type(addr) == IEE_NORMAL)) { ++ // pr_err("IEE Detected data type normal, va(0x%lx)", addr); ++ // return; ++ // } ++ // addr += PAGE_SIZE; ++ // } +} + -+static inline void iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr) ++#endif +diff --git a/arch/arm64/include/asm/haoc/haoc-def.h b/arch/arm64/include/asm/haoc/haoc-def.h +new file mode 100644 +index 000000000000..7f32efd97716 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/haoc-def.h +@@ -0,0 +1,128 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_HAOC_DEF_H ++#define _LINUX_HAOC_DEF_H ++ ++enum { ++ IEE_OP_MEMCPY, ++ IEE_OP_MEMSET, ++ IEE_OP_SET_FREEPTR, ++#ifdef CONFIG_IEE_PTRP ++ IEE_OP_SET_TOKEN_PGD, ++ IEE_OP_INIT_TOKEN, ++ IEE_OP_INVALIDATE_TOKEN, ++ IEE_OP_VALIDATE_TOKEN, ++#endif ++ IEE_OP_SET_BITMAP_TYPE, ++#ifdef CONFIG_CREDP ++ IEE_OP_COPY_CRED, ++ IEE_OP_COPY_KERNEL_CRED, ++ IEE_OP_INIT_COPIED_CRED, ++ IEE_OP_COMMIT_CRED, ++ IEE_OP_ABORT_CRED, ++ IEE_OP_OVERRIDE_CRED, ++ IEE_OP_REVERT_CRED, ++ IEE_OP_SET_CRED_UID, ++ IEE_OP_SET_CRED_GID, ++ IEE_OP_SET_CRED_SUID, ++ IEE_OP_SET_CRED_SGID, ++ IEE_OP_SET_CRED_EUID, ++ IEE_OP_SET_CRED_EGID, ++ IEE_OP_SET_CRED_FSUID, ++ IEE_OP_SET_CRED_FSGID, ++ IEE_OP_SET_CRED_USER, ++ IEE_OP_SET_CRED_USER_NS, ++ IEE_OP_SET_CRED_GROUP_INFO, ++ IEE_OP_SET_CRED_SECUREBITS, ++ IEE_OP_SET_CRED_CAP_INHER, ++ IEE_OP_SET_CRED_CAP_PERM, ++ IEE_OP_SET_CRED_CAP_EFFECT, ++ IEE_OP_SET_CRED_CAP_BSET, ++ IEE_OP_SET_CRED_CAP_AMBIENT, ++ IEE_OP_SET_CRED_JIT_KEYRING, ++ IEE_OP_SET_CRED_SESS_KEYRING, ++ IEE_OP_SET_CRED_PROC_KEYRING, ++ IEE_OP_SET_CRED_THREAD_KEYRING, ++ IEE_OP_SET_CRED_REQ_KEYRING, ++ IEE_OP_SET_CRED_NON_RCU, ++ IEE_OP_SET_CRED_ATSET_USAGE, ++ IEE_OP_SET_CRED_ATOP_USAGE, ++ IEE_OP_SET_CRED_SECURITY, ++ IEE_OP_SET_CRED_RCU, ++ IEE_OP_SET_CRED_UCOUNTS, ++#endif ++#ifdef CONFIG_PTP ++ IEE_OP_SET_TRAMP_PGD, ++ IEE_OP_SET_BM_PTE, ++ IEE_OP_SET_PTE, ++ IEE_OP_SET_PMD, ++ IEE_OP_SET_PUD, ++ IEE_OP_SET_P4D, ++ IEE_OP_SET_SWAPPER_PGD, ++ IEE_OP_SET_XCHG, ++ IEE_OP_SET_PMD_XCHG, ++ IEE_OP_SET_CMPXCHG, ++ IEE_OP_SET_PMD_CMPXCHG, ++ IEE_OP_SET_SENSITIVE_PTE, ++ IEE_OP_UNSET_SENSITIVE_PTE, ++#endif ++#ifdef CONFIG_KEYP ++ IEE_OP_SET_KEY_UNION, ++ IEE_OP_SET_KEY_STRUCT, ++ IEE_OP_SET_KEY_PAYLOAD, ++ IEE_OP_SET_KEY_USAGE, ++ IEE_OP_SET_KEY_SERIAL, ++ IEE_OP_SET_KEY_WATCHERS, ++ IEE_OP_SET_KEY_USERS, ++ IEE_OP_SET_KEY_SECURITY, ++ IEE_OP_SET_KEY_EXPIRY, ++ IEE_OP_SET_KEY_REVOKED_AT, ++ IEE_OP_SET_KEY_LAST_USED_AT, ++ IEE_OP_SET_KEY_UID, ++ IEE_OP_SET_KEY_GID, ++ IEE_OP_SET_KEY_PERM, ++ IEE_OP_SET_KEY_QUOTALEN, ++ IEE_OP_SET_KEY_DATALEN, ++ IEE_OP_SET_KEY_STATE, ++ IEE_OP_SET_KEY_MAGIC, ++ IEE_OP_SET_KEY_FLAGS, ++ IEE_OP_SET_KEY_INDEX_KEY, ++ IEE_OP_SET_KEY_HASH, ++ IEE_OP_SET_KEY_LEN_DESC, ++ IEE_OP_SET_KEY_TYPE, ++ IEE_OP_SET_KEY_TAG, ++ IEE_OP_SET_KEY_DESCRIPTION, ++ IEE_OP_SET_KEY_RESTRICT_LINK, ++ IEE_OP_SET_KEY_FLAG_BIT, ++#endif ++#ifdef CONFIG_IEE_SELINUX_P ++ IEE_SEL_SET_STATUS_PG, ++ IEE_SEL_SET_ENFORCING, ++ IEE_SEL_SET_INITIALIZED, ++ IEE_SEL_SET_POLICY_CAP, ++ IEE_SEL_RCU_ASSIGN_POLICY, ++#endif ++ IEE_FLAG_END ++}; ++ ++#ifdef CONFIG_CREDP ++#define AT_ADD 1 ++#define AT_INC_NOT_ZERO 2 ++#define AT_SUB_AND_TEST 3 ++#endif ++ ++#ifdef CONFIG_KEYP ++#define REFCOUNT_INC 1 ++#define REFCOUNT_SET 2 ++#define REFCOUNT_DEC_AND_TEST 3 ++#define REFCOUNT_INC_NOT_ZERO 4 ++ ++#define SET_BIT_OP 1 ++#define TEST_AND_CLEAR_BIT 2 ++#define TEST_AND_SET_BIT 3 ++#endif ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++#define __iee_code __section(".iee.text") ++#define __iee_data __section(".iee.data") ++ ++#endif +diff --git a/arch/arm64/include/asm/haoc/haoc.h b/arch/arm64/include/asm/haoc/haoc.h +new file mode 100644 +index 000000000000..24c04c3b70a9 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/haoc.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_HAOC_H ++#define _LINUX_HAOC_H ++ ++#include ++#include ++ ++void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); ++void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); ++void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); ++#ifdef CONFIG_IEE_PTRP ++void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, ++ pgd_t *pgd); ++void _iee_init_token(unsigned long __unused, struct task_struct *tsk); ++void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); ++void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); ++#endif ++ ++#ifdef CONFIG_CREDP ++#include ++ ++void _iee_copy_cred(unsigned long __unused, struct cred *new); ++void _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, ++ struct cred *new); ++void _iee_abort_cred(unsigned long __unused, const struct cred *cred); ++void _iee_init_copied_cred(unsigned long __unused, ++ struct task_struct *new_task, struct cred *new); ++void _iee_commit_creds(unsigned long __unused, const struct cred *new); ++void _iee_override_creds(unsigned long __unused, struct cred *new); ++void _iee_revert_creds(unsigned long __unused, const struct cred *old); ++void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); ++void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); ++void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); ++void _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid); ++void _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid); ++void _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid); ++void _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid); ++void _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid); ++void _iee_set_cred_user(unsigned long __unused, struct cred *cred, ++ struct user_struct *user); ++void _iee_set_cred_user_ns(unsigned long __unused, ++ struct cred *cred, struct user_namespace *user_ns); ++void _iee_set_cred_group_info(unsigned long __unused, ++ struct cred *cred, struct group_info *group_info); ++void _iee_set_cred_securebits(unsigned long __unused, ++ struct cred *cred, unsigned int securebits); ++void _iee_set_cred_cap_inheritable(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_inheritable); ++void _iee_set_cred_cap_permitted(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_permitted); ++void _iee_set_cred_cap_effective(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_effective); ++void _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, ++ kernel_cap_t cap_bset); ++void _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, ++ kernel_cap_t cap_ambient); ++void _iee_set_cred_jit_keyring(unsigned long __unused, ++ struct cred *cred, unsigned char jit_keyring); ++void _iee_set_cred_session_keyring(unsigned long __unused, ++ struct cred *cred, struct key *session_keyring); ++void _iee_set_cred_process_keyring(unsigned long __unused, ++ struct cred *cred, struct key *process_keyring); ++void _iee_set_cred_thread_keyring(unsigned long __unused, ++ struct cred *cred, struct key *thread_keyring); ++void _iee_set_cred_request_key_auth(unsigned long __unused, ++ struct cred *cred, struct key *request_key_auth); ++void _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu); ++void _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i); ++bool _iee_set_cred_atomic_op_usage(unsigned long __unused, ++ struct cred *cred, int flag, int nr); ++void _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security); ++void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu); ++void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, ++ struct ucounts *ucounts); ++#endif ++#include ++#include ++ ++#ifdef CONFIG_PTP ++void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd); ++void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte); ++void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte); ++void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd); ++void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud); ++void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d); ++void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd); ++pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, pteval_t pteval); ++pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, pmdval_t pmdval); ++pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, pteval_t old_pteval, ++ pteval_t new_pteval); ++pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, ++ pmdval_t old_pmdval, pmdval_t new_pmdval); ++void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, ++ int order, int use_block_pmd); ++void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, ++ int order, int use_block_pmd); ++#endif ++ ++#ifdef CONFIG_KEYP ++#include ++struct watch_list; ++ ++void _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); ++void _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); ++void _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); ++unsigned long _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); ++void _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); ++void _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); ++void _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); ++void _iee_set_key_security(unsigned long __unused, struct key *key, void *security); ++void _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); ++void _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); ++void _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); ++void _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); ++void _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); ++void _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); ++void _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); ++void _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); ++void _iee_set_key_state(unsigned long __unused, struct key *key, short state); ++void _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned int magic); ++void _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); ++void _iee_set_key_index_key(unsigned long __unused, struct key *key, ++ struct keyring_index_key *index_key); ++void _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); ++void _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); ++void _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); ++void _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); ++void _iee_set_key_description(unsigned long __unused, struct key *key, char *description); ++void _iee_set_key_restrict_link(unsigned long __unused, struct key *key, ++ struct key_restriction *restrict_link); ++unsigned long _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); ++#endif ++ ++#ifdef CONFIG_IEE_SELINUX_P ++#include ++void _iee_set_selinux_status_pg(unsigned long iee_offset, struct page *new_page); ++void _iee_set_selinux_enforcing(unsigned long iee_offset, bool value); ++void _iee_mark_selinux_initialized(unsigned long iee_offset); ++void _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap); ++void _iee_sel_rcu_assign_policy(unsigned long iee_offset, ++ struct selinux_policy *new_policy, struct selinux_policy *iee_new_policy); ++#endif ++ ++#endif +diff --git a/arch/arm64/include/asm/haoc/iee-access.h b/arch/arm64/include/asm/haoc/iee-access.h +new file mode 100644 +index 000000000000..c111eb6336c3 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-access.h +@@ -0,0 +1,25 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_ACCESS_H ++#define _LINUX_IEE_ACCESS_H ++ ++#include ++#include ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++static inline void iee_memcpy(void *dst, const void *src, size_t n) +{ -+ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); ++ iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); +} + -+static inline void iee_copy_pte_range(pte_t *new_dst, pte_t *old_dst, pte_t *src_pte, struct vm_area_struct *src_vma, unsigned long dst_vm_flags, pte_t *end_pte) ++static inline void iee_memset(void *ptr, int data, size_t n) +{ -+ iee_rw_gate(IEE_COPY_PTE_RANGE, new_dst, old_dst, src_pte, src_vma, dst_vm_flags, end_pte); ++ iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); +} + -+static inline void iee_split_huge_pmd(pmd_t *pmdp, pte_t *pgtable) ++static inline void iee_set_freeptr(void **pptr, void *ptr) +{ -+ iee_rw_gate(IEE_SPLIT_HUGE_PMD, pmdp, pgtable); ++ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); +} ++ ++#endif +diff --git a/arch/arm64/include/asm/haoc/iee-asm.h b/arch/arm64/include/asm/haoc/iee-asm.h +new file mode 100644 +index 000000000000..2e4ef179a416 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-asm.h +@@ -0,0 +1,53 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_ASM_H ++#define _LINUX_IEE_ASM_H ++ ++#include ++ ++#define BAD_ELR_EL1 0 ++#define BAD_TCR_EL1 1 ++ ++#define ASID_BIT (UL(1) << 48) ++ ++#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++#define IEE_ASID 0xfffe ++#else ++#define IEE_ASID 0xffff +#endif ++#define IEE_ASM_ASID (UL(IEE_ASID) << 48) ++ ++#define TCR_HPD1 (UL(1) << 42) ++#define TCR_A1 (UL(1) << 22) ++#define IEE_TCR_MASK (~(TCR_HD | TCR_E0PD1 | TCR_T0SZ_MASK)) + ++#ifdef CONFIG_IEE_SIP ++/* IEE exit code Remember that ARM instructions are aligned with 8 byte. */ ++#define IEE_SI_EXIT_OFFSET (4*4) ++#define IEE_SI_TCR_MASK (~IEE_TCR_MASK) +#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-cred.h b/arch/arm64/include/asm/iee-cred.h ++ ++#ifdef __ASSEMBLY__ ++ /* IEE code shall not be interrupted. */ ++ .macro iee_elr_check ++ ldr x1, =__iee_code_start ++ cmp x1, x22 /* ELR_EL1 was stored in x22 on kernel entry. */ ++ b.hi 114f ++ #ifdef CONFIG_IEE_SIP ++ ldr x2, =__iee_si_text_end ++ #else ++ ldr x2, =__iee_code_end ++ #endif ++ cmp x2, x22 ++ b.lo 114f ++ /* ELR check fail */ ++ mov x0, sp ++ mov x1, #BAD_ELR_EL1 ++ mrs x2, esr_el1 ++ mov x3, x22 ++ bl iee_bad_mode ++ ASM_BUG() ++114: ++ .endm ++#endif ++ ++#endif +diff --git a/arch/arm64/include/asm/haoc/iee-cred.h b/arch/arm64/include/asm/haoc/iee-cred.h new file mode 100644 -index 000000000000..b8c3bb53f98a +index 000000000000..9b2447531b66 --- /dev/null -+++ b/arch/arm64/include/asm/iee-cred.h -@@ -0,0 +1,150 @@ ++++ b/arch/arm64/include/asm/haoc/iee-cred.h +@@ -0,0 +1,235 @@ ++/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _LINUX_IEE_CRED_H +#define _LINUX_IEE_CRED_H + ++#include ++#include +#include -+#include + +extern unsigned long long iee_rw_gate(int flag, ...); + -+#ifdef CONFIG_CREDP -+static void __maybe_unused iee_copy_cred(const struct cred *old, struct cred *new) ++#ifdef CONFIG_IEE_PTRP ++#include ++/* Verify cred on critical kernel opeartions that need permission checks. */ ++#define iee_verify_cred(__unused) { \ ++ struct task_struct *__curr_task = current; \ ++ struct task_token *__token = (struct task_token *)__addr_to_iee(__curr_task);\ ++ const struct cred *__curr_cred = current_cred(); \ ++ \ ++ if (!__token->valid) \ ++ panic("IEE: (%s) Invalid Token.", __func__); \ ++ if (__token->curr_cred != __curr_cred) \ ++ panic("IEE: (%s) Task cred corruptted! token cred 0x%llx " \ ++ "task cred 0x%llx real 0x%llx " \ ++ "pid %d uid %d suid %d euid %d fsuid %d", \ ++ __func__, (u64)__token->curr_cred, (u64)__curr_cred, \ ++ (u64)__curr_task->real_cred, \ ++ __curr_task->pid, __curr_cred->uid.val, __curr_cred->suid.val, \ ++ __curr_cred->euid.val, __curr_cred->fsuid.val); \ ++} ++ ++/* Verify cred before update it. */ ++#define iee_verify_update_cred(new) { \ ++ struct task_struct *__curr_task = current; \ ++ struct task_token *__token = (struct task_token *)__addr_to_iee(__curr_task);\ ++ const struct cred *__curr_cred = current_cred(); \ ++ \ ++ if (!__token->valid) \ ++ panic("IEE: (%s) Invalid Token.", __func__); \ ++ if (__curr_task != &init_task) { \ ++ if (__token->new_cred != new) \ ++ pr_err("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); \ ++ /* cred of task_struct shall be the same with the one recorded in token. */ \ ++ if (__token->curr_cred != __curr_cred) { \ ++ panic("IEE: (%s) Task cred corruptted! token cred 0x%llx, curr 0x%llx" \ ++ "real 0x%llx pid %d uid %d suid %d euid %d fsuid %d", \ ++ __func__, (u64)__token->curr_cred, (u64)__curr_cred, \ ++ (u64)__curr_task->real_cred, \ ++ __curr_task->pid, __curr_cred->uid.val, __curr_cred->suid.val,\ ++ __curr_cred->euid.val, __curr_cred->fsuid.val); \ ++ } \ ++ } \ ++} ++#else ++#define iee_verify_cred(__unused) ++#define iee_verify_update_cred(new) ++#endif ++ ++static void __maybe_unused iee_copy_cred(struct cred *new) ++{ ++ iee_rw_gate(IEE_OP_COPY_CRED, new); ++} ++ ++static void __maybe_unused iee_copy_kernel_cred(const struct cred *old, ++ struct cred *new) ++{ ++ iee_rw_gate(IEE_OP_COPY_KERNEL_CRED, old, new); ++} ++ ++static void __maybe_unused iee_init_copied_cred(struct task_struct *new_task, ++ const struct cred *new) ++{ ++ iee_rw_gate(IEE_OP_INIT_COPIED_CRED, new_task, new); ++} ++ ++static void __maybe_unused iee_abort_creds(struct cred *cred) ++{ ++ iee_rw_gate(IEE_OP_ABORT_CRED, cred); ++} ++ ++static void __maybe_unused iee_commit_creds(const struct cred *new) ++{ ++ iee_rw_gate(IEE_OP_COMMIT_CRED, new); ++} ++ ++static void __maybe_unused iee_override_creds(const struct cred *new) +{ -+ iee_rw_gate(IEE_OP_COPY_CRED,old,new); ++ iee_rw_gate(IEE_OP_OVERRIDE_CRED, new); ++} ++ ++static void __maybe_unused iee_revert_creds(const struct cred *old) ++{ ++ iee_rw_gate(IEE_OP_REVERT_CRED, old); +} + +static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_UID,cred,uid); ++ iee_rw_gate(IEE_OP_SET_CRED_UID, cred, uid); +} + +static void __maybe_unused iee_set_cred_gid(struct cred *cred, kgid_t gid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_GID,cred,gid); ++ iee_rw_gate(IEE_OP_SET_CRED_GID, cred, gid); +} + +static void __maybe_unused iee_set_cred_suid(struct cred *cred, kuid_t suid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_SUID,cred,suid); ++ iee_rw_gate(IEE_OP_SET_CRED_SUID, cred, suid); +} + +static void __maybe_unused iee_set_cred_sgid(struct cred *cred, kgid_t sgid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_SGID,cred,sgid); ++ iee_rw_gate(IEE_OP_SET_CRED_SGID, cred, sgid); +} + +static void __maybe_unused iee_set_cred_euid(struct cred *cred, kuid_t euid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_EUID,cred,euid); ++ iee_rw_gate(IEE_OP_SET_CRED_EUID, cred, euid); +} + +static void __maybe_unused iee_set_cred_egid(struct cred *cred, kgid_t egid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_EGID,cred,egid); ++ iee_rw_gate(IEE_OP_SET_CRED_EGID, cred, egid); +} + +static void __maybe_unused iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_FSUID,cred,fsuid); ++ iee_rw_gate(IEE_OP_SET_CRED_FSUID, cred, fsuid); +} + +static void __maybe_unused iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_FSGID,cred,fsgid); ++ iee_rw_gate(IEE_OP_SET_CRED_FSGID, cred, fsgid); +} + +static void __maybe_unused iee_set_cred_user(struct cred *cred, struct user_struct *user) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_USER,cred,user); ++ iee_rw_gate(IEE_OP_SET_CRED_USER, cred, user); +} + +static void __maybe_unused iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_USER_NS,cred,user_ns); ++ iee_rw_gate(IEE_OP_SET_CRED_USER_NS, cred, user_ns); +} + +static void __maybe_unused iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS,cred,ucounts); ++ iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS, cred, ucounts); +} + +static void __maybe_unused iee_set_cred_group_info(struct cred *cred, struct group_info *group_info) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO,cred,group_info); ++ iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO, cred, group_info); +} + -+static void __maybe_unused iee_set_cred_securebits(struct cred *cred, unsigned securebits) ++static void __maybe_unused iee_set_cred_securebits(struct cred *cred, ++ unsigned int securebits) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS,cred,securebits); ++ iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS, cred, securebits); +} + -+static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable) ++static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, ++ kernel_cap_t cap_inheritable) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER,cred,cap_inheritable); ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER, cred, cap_inheritable); +} + +static void __maybe_unused iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM,cred,cap_permitted); ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM, cred, cap_permitted); +} + +static void __maybe_unused iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT,cred,cap_effective); ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT, cred, cap_effective); +} + +static void __maybe_unused iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET,cred,cap_bset); ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET, cred, cap_bset); +} + +static void __maybe_unused iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT,cred,cap_ambient); ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT, cred, cap_ambient); +} + -+#ifdef CONFIG_KEYS -+static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) ++static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING,cred,jit_keyring); ++ iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE, cred, i); +} + -+static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring) ++static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING,cred,session_keyring); ++ iee_rw_gate(IEE_OP_SET_CRED_RCU, cred, rcu); +} + -+static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring) ++#ifdef CONFIG_KEYS ++static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING,cred,process_keyring); ++ iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING, cred, jit_keyring); +} + -+static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring) ++static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, ++ struct key *session_keyring) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING,cred,thread_keyring); ++ iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING, cred, session_keyring); +} + -+static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth) ++static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, ++ struct key *process_keyring) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING,cred,request_key_auth); ++ iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING, cred, process_keyring); +} -+#endif + -+static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i) ++static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, ++ struct key *thread_keyring) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE,cred,i); ++ iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING, cred, thread_keyring); +} + -+#ifdef CONFIG_SECURITY -+static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security) ++static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, ++ struct key *request_key_auth) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_SECURITY,cred,security); ++ iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING, cred, request_key_auth); +} +#endif + -+static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) ++#ifdef CONFIG_SECURITY ++static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_RCU,cred,rcu); ++ iee_rw_gate(IEE_OP_SET_CRED_SECURITY, cred, security); +} +#endif + +#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-def.h b/arch/arm64/include/asm/iee-def.h +diff --git a/arch/arm64/include/asm/haoc/iee-fixmap.h b/arch/arm64/include/asm/haoc/iee-fixmap.h new file mode 100644 -index 000000000000..d33d8570a546 +index 000000000000..628ff4a5c687 --- /dev/null -+++ b/arch/arm64/include/asm/iee-def.h -@@ -0,0 +1,130 @@ -+#ifndef _LINUX_IEE_DEF_H -+#define _LINUX_IEE_DEF_H ++++ b/arch/arm64/include/asm/haoc/iee-fixmap.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_FIXMAP_H ++#define _LINUX_IEE_FIXMAP_H + -+/* Add new IEE ops here */ ++#include + -+#define AT_ADD 1 -+#define AT_INC_NOT_ZERO 2 -+#define AT_SUB_AND_TEST 3 -+/* Atomic ops for atomic_t */ ++#ifndef clear_fixmap_pre_init ++#define clear_fixmap_pre_init(idx) \ ++ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR) ++#endif + -+#define REFCOUNT_INC 1 -+#define REFCOUNT_SET 2 -+#define REFCOUNT_DEC_AND_TEST 3 -+#define REFCOUNT_INC_NOT_ZERO 4 ++#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \ ++({ \ ++ unsigned long ________addr; \ ++ __iee_set_fixmap_pre_init(idx, phys, flags); \ ++ ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \ ++ ________addr; \ ++}) + -+#define SET_BIT_OP 1 -+#define TEST_AND_CLEAR_BIT 2 -+#define TEST_AND_SET_BIT 3 ++#define iee_set_fixmap_offset_pre_init(idx, phys) \ ++ __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL) + -+// Function Identifiers with Parameters Description -+enum { -+ IEE_WRITE_IN_BYTE=0, // Parameters: void *ptr, __u64 data, int length -+ IEE_OP_SET_PTE, // Parameters: pte_t *ptep, pte_t pte -+ IEE_OP_SET_PMD, // Parameters: pmd_t *pmdp, pmd_t pmd -+ IEE_OP_SET_PUD, // Parameters: pud_t *pudp, pud_t pud -+ IEE_OP_SET_P4D, // Parameters: p4d_t *p4dp, p4d_t p4d -+ IEE_OP_SET_BM_PTE, // Parameters: pte_t *ptep, pte_t pte -+ IEE_OP_SET_SWAPPER_PGD, // Parameters: pgd_t *pgdp, pgd_t pgd -+ IEE_OP_SET_TRAMP_PGD, // Parameters: pgd_t *pgdp, pgd_t pgd -+ IEE_MEMSET, // Parameters: void *ptr, int data, size_t n -+ IEE_OP_SET_TRACK, // Parameters: struct track *ptr, struct track *data -+ IEE_OP_SET_FREEPTR, // Parameters: void **pptr, void *ptr -+ IEE_OP_SET_PTE_U, // Parameters: pte_t *ptep -+ IEE_OP_SET_PTE_P, // Parameters: pte_t *ptep -+ IEE_SET_TOKEN_PGD, // Parameters: struct task_token *token, pgd_t *pgd -+ IEE_INIT_TOKEN, // Parameters: struct task_struct *tsk, void *kernel_stack, void *iee_stack -+ IEE_INVALIDATE_TOKEN, // Parameters: struct task_struct *tsk -+ IEE_SET_SENSITIVE_PTE, // Parameters: pte_t *lm_ptep, pte_t *iee_ptep -+ IEE_UNSET_SENSITIVE_PTE, // Parameters: pte_t *lm_ptep, pte_t *iee_ptep -+ IEE_SET_TOKEN, // Parameters: pte_t *ptep, void *new, unsigned long order -+ IEE_UNSET_TOKEN, // Parameters: pte_t *ptep, void *token_addr, void *token_page, unsigned long order -+ IEE_COPY_PTE_RANGE, // Parameters: pte_t *new_dst, pte_t *old_dst, pte_t *src_pte, struct vm_area_struct *src_vma, -+ // unsigned long dst_vm_flags, pte_t *end_pte -+ IEE_SPLIT_HUGE_PMD, -+ IEE_VALIDATE_TOKEN, -+ IEE_MEMCPY, -+#ifdef CONFIG_KOI -+ _IEE_READ_KOI_STACK, // Parameters: struct task_struct *tsk -+ _IEE_WRITE_KOI_STACK, // Parameters: struct task_struct *tsk, unsigned long koi_stack -+ _IEE_READ_TOKEN_TTBR1, // Parameters: struct task_struct *tsk -+ _IEE_WRITE_TOKEN_TTBR1, // Parameters: struct task_struct *tsk, unsigned long current_ttbr1 -+ _IEE_READ_KOI_KERNEL_STACK, // Parameters: struct task_struct *tsk -+ _IEE_WRITE_KOI_KERNEL_STACK, // Parameters: struct task_struct *tsk, unsigned long kernel_stack -+ _IEE_READ_KOI_STACK_BASE, // Parameters: struct task_struct *tsk -+ _IEE_WRITE_KOI_STACK_BASE, // Parameters: struct task_struct *tsk, unsigned long koi_stack_base -+ _IEE_SET_KOI_PGD, // Parameters: unsigned long koi_pgd_addr -+#endif -+#ifdef CONFIG_CREDP -+ IEE_OP_COPY_CRED, // Parameters: struct cred *old, struct cred *new -+ IEE_OP_SET_CRED_UID, // Parameters: struct cred *cred, kuid_t uid -+ IEE_OP_SET_CRED_GID, // Parameters: struct cred *cred, kgid_t gid -+ IEE_OP_SET_CRED_SUID, // Parameters: struct cred *cred, kuid_t suid -+ IEE_OP_SET_CRED_SGID, // Parameters: struct cred *cred, kgid_t sgid -+ IEE_OP_SET_CRED_EUID, // Parameters: struct cred *cred, kuid_t euid -+ IEE_OP_SET_CRED_EGID, // Parameters: struct cred *cred, kgid_t egid -+ IEE_OP_SET_CRED_FSUID, // Parameters: struct cred *cred, kuid_t fsuid -+ IEE_OP_SET_CRED_FSGID, // Parameters: struct cred *cred, kgid_t fsgid -+ IEE_OP_SET_CRED_USER, // Parameters: struct cred *cred, struct user_struct *user -+ IEE_OP_SET_CRED_USER_NS, // Parameters: struct cred *cred, struct user_namespace *user_ns -+ IEE_OP_SET_CRED_GROUP_INFO, // Parameters: struct cred *cred, struct group_info *group_info -+ IEE_OP_SET_CRED_SECUREBITS, // Parameters: struct cred *cred, unsigned securebits -+ IEE_OP_SET_CRED_CAP_INHER, // Parameters: struct cred *cred, kernel_cap_t cap_inheritable -+ IEE_OP_SET_CRED_CAP_PERM, // Parameters: struct cred *cred, kernel_cap_t cap_permitted -+ IEE_OP_SET_CRED_CAP_EFFECT, // Parameters: struct cred *cred, kernel_cap_t cap_effective -+ IEE_OP_SET_CRED_CAP_BSET, // Parameters: struct cred *cred, kernel_cap_t cap_bset -+ IEE_OP_SET_CRED_CAP_AMBIENT, // Parameters: struct cred *cred, kernel_cap_t cap_ambient -+ IEE_OP_SET_CRED_JIT_KEYRING, // Parameters: struct cred *cred, unsigned char jit_keyring -+ IEE_OP_SET_CRED_SESS_KEYRING, // Parameters: struct cred *cred, struct key *session_keyring -+ IEE_OP_SET_CRED_PROC_KEYRING, // Parameters: struct cred *cred, struct key *process_keyring -+ IEE_OP_SET_CRED_THREAD_KEYRING, // Parameters: struct cred *cred, struct key *thread_keyring -+ IEE_OP_SET_CRED_REQ_KEYRING, // Parameters: struct cred *cred, struct key *request_key_auth -+ IEE_OP_SET_CRED_NON_RCU, // Parameters: struct cred *cred, int non_rcu -+ IEE_OP_SET_CRED_ATSET_USAGE, // Parameters: struct cred *cred, int i -+ IEE_OP_SET_CRED_ATOP_USAGE, // Parameters: struct cred *cred, int flag -+ IEE_OP_SET_CRED_SECURITY, // Parameters: struct cred *cred, void *security -+ IEE_OP_SET_CRED_RCU, // Parameters: struct cred *cred, struct rcu_head *rcu -+ IEE_OP_SET_CRED_UCOUNTS, // Parameters: struct cred *cred, struct ucounts *ucounts -+// IEE_OP_COMMIT_CRED, // Parameters: struct cred *cred -+// IEE_OP_COPY_CRED_KERNEL, // Parameters: struct cred *old, struct cred *new -+#endif -+#ifdef CONFIG_KEYP -+ IEE_OP_SET_KEY_UNION, -+ IEE_OP_SET_KEY_STRUCT, -+ IEE_OP_SET_KEY_PAYLOAD, -+ IEE_OP_SET_KEY_USAGE, -+ IEE_OP_SET_KEY_SERIAL, -+ IEE_OP_SET_KEY_WATCHERS, -+ IEE_OP_SET_KEY_USERS, -+ IEE_OP_SET_KEY_SECURITY, -+ IEE_OP_SET_KEY_EXPIRY, -+ IEE_OP_SET_KEY_REVOKED_AT, -+ IEE_OP_SET_KEY_LAST_USED_AT, -+ IEE_OP_SET_KEY_UID, -+ IEE_OP_SET_KEY_GID, -+ IEE_OP_SET_KEY_PERM, -+ IEE_OP_SET_KEY_QUOTALEN, -+ IEE_OP_SET_KEY_DATALEN, -+ IEE_OP_SET_KEY_STATE, -+ IEE_OP_SET_KEY_MAGIC, -+ IEE_OP_SET_KEY_FLAGS, -+ IEE_OP_SET_KEY_INDEX_KEY, -+ IEE_OP_SET_KEY_HASH, -+ IEE_OP_SET_KEY_LEN_DESC, -+ IEE_OP_SET_KEY_TYPE, -+ IEE_OP_SET_KEY_TAG, -+ IEE_OP_SET_KEY_DESCRIPTION, -+ IEE_OP_SET_KEY_RESTRICT_LINK, -+ IEE_OP_SET_KEY_FLAG_BIT, +#endif -+#ifdef CONFIG_IEE_SELINUX_P -+ IEE_SEL_SET_STATUS_PG, // Parameters: struct page* new_page -+ IEE_SEL_SET_ENFORCING, // Parameters: bool value -+ IEE_SEL_SET_INITIALIZED, -+ IEE_SEL_SET_POLICY_CAP, // Parameters: unsigned int idx, int cap -+ IEE_SEL_RCU_ASSIGN_POLICY, // Parameters: struct selinux_policy* new_policy, struct selinux_policy* iee_new_policy +diff --git a/arch/arm64/include/asm/haoc/iee-func.h b/arch/arm64/include/asm/haoc/iee-func.h +new file mode 100644 +index 000000000000..da2c3e2760df +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-func.h +@@ -0,0 +1,18 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_ASM_FUNC_H ++#define _LINUX_IEE_ASM_FUNC_H ++ ++#include ++ ++extern void set_iee_address(unsigned long addr, unsigned int order, bool valid); ++extern void set_iee_address_valid(unsigned long lm_addr, unsigned int order); ++extern void set_iee_address_invalid(unsigned long lm_addr, unsigned int order); ++extern void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot); ++extern void iee_set_logical_mem_ro(unsigned long addr); ++extern void iee_set_logical_mem_rw(unsigned long addr); ++extern void put_pages_into_iee(unsigned long addr, int order); ++extern void remove_pages_from_iee(unsigned long addr, int order); ++extern void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type); ++extern void unset_iee_page(unsigned long addr, int order); ++ +#endif -+ IEE_FLAG_END -+}; +diff --git a/arch/arm64/include/asm/haoc/iee-init.h b/arch/arm64/include/asm/haoc/iee-init.h +new file mode 100644 +index 000000000000..c3a450a9d527 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-init.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_INIT_H ++#define _LINUX_IEE_INIT_H ++ ++#define NO_BLOCK_MAPPINGS BIT(0) ++#define NO_CONT_MAPPINGS BIT(1) ++#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ ++ ++extern char iee_init_data_begin[]; ++extern char iee_init_data_end[]; ++extern char __iee_cred_data_start[]; ++extern char __iee_cred_data_end[]; ++extern char __iee_ptp_data_start[]; ++extern char __iee_ptp_data_end[]; ++extern char __iee_selinux_data_start[]; ++extern char __iee_selinux_data_end[]; ++ ++extern spinlock_t swapper_pgdir_lock; ++extern struct mutex fixmap_lock; + +#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-key.h b/arch/arm64/include/asm/iee-key.h +diff --git a/arch/arm64/include/asm/haoc/iee-key.h b/arch/arm64/include/asm/haoc/iee-key.h new file mode 100644 -index 000000000000..5653720badb6 +index 000000000000..86d499d9aff6 --- /dev/null -+++ b/arch/arm64/include/asm/iee-key.h -@@ -0,0 +1,149 @@ ++++ b/arch/arm64/include/asm/haoc/iee-key.h +@@ -0,0 +1,151 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_KEY_H +#define _LINUX_IEE_KEY_H + +#include -+#include ++#include + +extern unsigned long long iee_rw_gate(int flag, ...); + -+#ifdef CONFIG_KEYP +static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); ++ iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); +} + +static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); ++ iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); +} + +static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); ++ iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); +} + +extern bool iee_set_key_usage(struct key *key, int n, int flag); + +static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); ++ iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); +} + +#ifdef CONFIG_KEY_NOTIFICATIONS +static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); ++ iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); +} +#endif + +static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); ++ iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); +} + +static void __maybe_unused iee_set_key_security(struct key *key, void *security) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); ++ iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); +} + +static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); ++ iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); +} + +static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); ++ iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); +} + +static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); ++ iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); +} + +static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); ++ iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); +} + +static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); ++ iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); +} + +static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); ++ iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); +} + +static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); ++ iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); +} + +static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); ++ iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); +} + +static void __maybe_unused iee_set_key_state(struct key *key, short state) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); ++ iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); +} + +#ifdef KEY_DEBUGGING -+static void __maybe_unused iee_set_key_magic(struct key *key, unsigned magic) ++static void __maybe_unused iee_set_key_magic(struct key *key, unsigned int magic) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); ++ iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); +} +#endif + +static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); ++ iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); +} + -+static void __maybe_unused iee_set_key_index_key(struct key *key, struct keyring_index_key *index_key) ++static void __maybe_unused iee_set_key_index_key(struct key *key, ++ struct keyring_index_key *index_key) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); ++ iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); +} + +static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); ++ iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); +} + +static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); ++ iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); +} + +static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); ++ iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); +} + +static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); ++ iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); +} + +static void __maybe_unused iee_set_key_description(struct key *key, char *description) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); ++ iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); +} + -+static void __maybe_unused iee_set_key_restrict_link(struct key *key, struct key_restriction *restrict_link) ++static void __maybe_unused iee_set_key_restrict_link(struct key *key, ++ struct key_restriction *restrict_link) +{ -+ iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); ++ iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); +} + +static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) +{ -+ bool ret; -+ ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); -+ return ret; ++ bool ret; ++ ++ ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); ++ return ret; +} -+#endif + +#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-koi.h b/arch/arm64/include/asm/iee-koi.h -new file mode 100644 -index 000000000000..8f7bfcc7e581 ---- /dev/null -+++ b/arch/arm64/include/asm/iee-koi.h -@@ -0,0 +1,13 @@ -+#if defined(CONFIG_KOI) && defined(CONFIG_IEE) -+#define IEE_SWITCH_TO_KERNEL 7 -+#define IEE_SWITCH_TO_KOI 8 -+#define IEE_READ_KOI_STACK 24 -+#define IEE_WRITE_KOI_STACK 25 -+#define IEE_READ_TOKEN_TTBR1 26 -+#define IEE_WRITE_TOKEN_TTBR1 27 -+#define IEE_READ_KOI_KERNEL_STACK 28 -+#define IEE_WRITE_KOI_KERNEL_STACK 29 -+#define IEE_READ_KOI_STACK_BASE 30 -+#define IEE_WRITE_KOI_STACK_BASE 31 -+#define IEE_SET_KOI_PGD 32 -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-selinuxp.h b/arch/arm64/include/asm/iee-selinuxp.h +diff --git a/arch/arm64/include/asm/haoc/iee-mmu.h b/arch/arm64/include/asm/haoc/iee-mmu.h new file mode 100644 -index 000000000000..b1cf52d8c2d8 +index 000000000000..bb4d9dfe4dad --- /dev/null -+++ b/arch/arm64/include/asm/iee-selinuxp.h -@@ -0,0 +1,27 @@ -+#ifndef _LINUX_IEE_SELINUX_P_H -+#define _LINUX_IEE_SELINUX_P_H -+ -+#include -+#include -+#include "security.h" -+#include "ss/services.h" -+ -+static inline struct mutex* iee_get_selinux_policy_lock(void) -+{ -+ return (struct mutex*)(selinux_state.policy_mutex.owner.counter); -+} -+ -+static inline struct mutex* iee_get_selinux_status_lock(void) -+{ -+ return (struct mutex*)(selinux_state.status_lock.owner.counter); -+} ++++ b/arch/arm64/include/asm/haoc/iee-mmu.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_MMU_H ++#define _LINUX_IEE_MMU_H + -+/* APIs for modifying selinux_state */ -+extern void iee_set_selinux_status_pg(struct page* new_page); -+extern void iee_set_sel_policy_cap(unsigned int idx, int cap); -+extern void iee_sel_rcu_assign_policy(struct selinux_policy* new_policy, -+ struct selinux_policy* iee_new_policy); ++#ifdef CONFIG_PTP ++#include ++#endif + -+extern struct kmem_cache *policy_jar; ++extern phys_addr_t __init early_iee_stack_alloc(int order); ++extern phys_addr_t __init early_iee_data_alloc(int shift); ++extern phys_addr_t __init early_iee_pgtable_alloc(int shift); ++extern phys_addr_t __init early_pgtable_alloc(int shift); ++extern phys_addr_t __pgd_pgtable_alloc(int shift); ++extern phys_addr_t pgd_pgtable_alloc(int shift); ++extern void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags); ++extern void __init iee_init_mappings(pgd_t *pgdp); ++extern void __init init_early_iee_data(void); ++extern void __init early_iee_data_cache_init(void); + +#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-si.h b/arch/arm64/include/asm/iee-si.h +diff --git a/arch/arm64/include/asm/haoc/iee-pgtable.h b/arch/arm64/include/asm/haoc/iee-pgtable.h new file mode 100644 -index 000000000000..02b77547b29c +index 000000000000..0a3c12a2aa6c --- /dev/null -+++ b/arch/arm64/include/asm/iee-si.h -@@ -0,0 +1,61 @@ -+#ifndef _LINUX_IEE_SI_H -+#define _LINUX_IEE_SI_H -+ -+#include -+#define __iee_si_code __section(".iee.si_text") -+#define __iee_si_data __section(".iee.si_data") -+ -+/* Used for copying globals that iee rwx gate needs. */ -+extern unsigned long iee_base_swapper_pg_dir; -+extern unsigned long iee_base_idmap_pg_dir; -+extern unsigned long iee_base_reserved_pg_dir; -+extern unsigned long iee_base__bp_harden_el1_vectors; -+extern bool iee_init_done; -+extern unsigned long iee_si_tcr; ++++ b/arch/arm64/include/asm/haoc/iee-pgtable.h +@@ -0,0 +1,1732 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef __ASM_IEE_PGTABLE_H ++#define __ASM_IEE_PGTABLE_H + -+/* The following are __init functions used for iee si initialization. */ -+extern void iee_si_prepare_data(void); ++#include ++#include + -+extern unsigned long __iee_si_text_start[]; -+// Handler function for sensitive inst -+u64 iee_si_handler(int flag, ...); -+/* -+ * TODO: scan a page to check whether it contains sensitive instructions -+ * return 1 when finding sensitive inst, 0 on safe page. -+ */ -+extern int iee_si_scan_page(unsigned long addr); -+ -+ -+#define DBG_MDSCR_SS (1 << 0) -+#define DBG_MDSCR_MDE (1 << 15) -+ -+#define IEE_SI_TEST 0 -+#define IEE_WRITE_SCTLR 1 -+#define IEE_WRITE_TTBR0 2 -+#define IEE_WRITE_VBAR 3 -+#define IEE_WRITE_TCR 4 -+#define IEE_WRITE_MDSCR 5 -+#define IEE_CONTEXT_SWITCH 6 -+// #define IEE_WRITE_AFSR0 10 -+/* Provide ttbr1 switch gate for KOI */ -+/* MASK modify-permitted bits on IEE protected sys registers */ -+#define IEE_SCTLR_MASK (SCTLR_EL1_CP15BEN | SCTLR_EL1_SED | SCTLR_EL1_UCT | SCTLR_EL1_UCI |\ -+ SCTLR_EL1_BT0 | SCTLR_EL1_BT1 | SCTLR_EL1_TCF0_MASK | SCTLR_ELx_DSSBS |\ -+ SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB|\ -+ SCTLR_EL1_SPINTMASK | SCTLR_EL1_NMI | SCTLR_EL1_TIDCP | SCTLR_EL1_MSCEn|\ -+ SCTLR_ELx_ENTP2 | SCTLR_EL1_TCF_MASK) -+#define IEE_TTBR0_MASK ~0 -+#define IEE_TTBR1_MASK ~0 -+#define IEE_TCR_MASK (TCR_HD | TCR_T0SZ_MASK | TCR_E0PD1) -+#define IEE_MDSCR_MASK (DBG_MDSCR_SS | DBG_MDSCR_MDE) -+ -+#define IEE_DBGBCR_BT 0b0000 << 20 -+#define IEE_DBGBCR_SSC 0b00 << 14 -+#define IEE_DBGBCR_HMC 0b1 << 13 -+#define IEE_DBGBCR_BAS 0b1111 << 5 -+#define IEE_DBGBCR_PMC 0b11 << 1 -+#define IEE_DBGBCR_E 0b1 -+#define IEE_DBGBCR IEE_DBGBCR_BT | IEE_DBGBCR_SSC | IEE_DBGBCR_HMC | IEE_DBGBCR_BAS \ -+ | IEE_DBGBCR_PMC | IEE_DBGBCR_E ++#include ++#include ++#include ++#include ++#include + -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-slab.h b/arch/arm64/include/asm/iee-slab.h -new file mode 100644 -index 000000000000..4f3c17c7da00 ---- /dev/null -+++ b/arch/arm64/include/asm/iee-slab.h -@@ -0,0 +1,23 @@ -+#ifndef _LINUX_IEE_SLAB_H -+#define _LINUX_IEE_SLAB_H +/* -+ * Tracking user of a slab. ++ * VMALLOC range. ++ * ++ * VMALLOC_START: beginning of the kernel vmalloc space ++ * VMALLOC_END: extends to the available space below vmemmap, PCI I/O space ++ * and fixed mappings + */ -+#include ++#define VMALLOC_START (MODULES_END) ++#define VMALLOC_END (VMEMMAP_START - SZ_256M) + -+#define TRACK_ADDRS_COUNT 16 -+struct track { -+ unsigned long addr; /* Called from address */ -+#ifdef CONFIG_STACKDEPOT -+ depot_stack_handle_t handle; -+#endif -+ int cpu; /* Was running on cpu */ -+ int pid; /* Pid context */ -+ unsigned long when; /* When did the operation occur */ -+}; ++#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) + -+enum track_item { TRACK_ALLOC, TRACK_FREE }; ++#ifndef __ASSEMBLY__ + -+typedef struct { unsigned long v; } freeptr_t; ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee-token.h b/arch/arm64/include/asm/iee-token.h -new file mode 100644 -index 000000000000..25ebf08faf8f ---- /dev/null -+++ b/arch/arm64/include/asm/iee-token.h -@@ -0,0 +1,33 @@ -+#ifndef _LINUX_IEE_TOKEN_H -+#define _LINUX_IEE_TOKEN_H ++extern pgd_t tramp_pg_dir[]; ++extern pgd_t idmap_pg_dir[]; + -+#include ++extern int __pmdp_set_access_flags(struct vm_area_struct *vma, ++ unsigned long address, pmd_t *pmdp, ++ pmd_t entry, int dirty); + -+extern unsigned long long iee_rw_gate(int flag, ...); -+struct task_token; -+struct task_struct; -+struct mm_struct; ++static inline bool in_swapper_pgdir(void *addr); ++static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd); + -+#ifdef CONFIG_IEE -+static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) ++static inline bool in_tramp_pgdir(void *addr) +{ -+ iee_rw_gate(IEE_SET_TOKEN_PGD, tsk, pgd); ++ return ((unsigned long)addr & PAGE_MASK) == ++ ((unsigned long)tramp_pg_dir & PAGE_MASK); +} + -+static inline void iee_init_token(struct task_struct *tsk, void *iee_stack, void *tmp_page) ++static inline bool in_idmap_pgdir(void *addr) +{ -+ iee_rw_gate(IEE_INIT_TOKEN, tsk, iee_stack, tmp_page); ++ return ((unsigned long)addr & PAGE_MASK) == ++ ((unsigned long)idmap_pg_dir & PAGE_MASK); +} + -+static inline void iee_invalidate_token(struct task_struct *tsk) ++static inline void iee_set_pte_pre_init(pte_t *ptep, pte_t pte) +{ -+ iee_rw_gate(IEE_INVALIDATE_TOKEN, tsk); ++ WRITE_ONCE(*ptep, pte); ++ ++ /* ++ * Only if the new pte is valid and kernel, otherwise TLB maintenance ++ * or update_mmu_cache() have the necessary barriers. ++ */ ++ dsb(ishst); ++ isb(); +} + -+static inline void iee_validate_token(struct task_struct *tsk) ++static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) +{ -+ iee_rw_gate(IEE_VALIDATE_TOKEN, tsk); ++#ifdef __PAGETABLE_PMD_FOLDED ++ if (in_swapper_pgdir(pmdp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd))); ++ return; ++ } ++#endif /* __PAGETABLE_PMD_FOLDED */ ++ ++ WRITE_ONCE(*pmdp, pmd); ++ ++ dsb(ishst); ++ isb(); +} -+#endif + -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/iee.h b/arch/arm64/include/asm/iee.h -new file mode 100644 -index 000000000000..598f6d0b2626 ---- /dev/null -+++ b/arch/arm64/include/asm/iee.h -@@ -0,0 +1,10 @@ -+#ifndef _LINUX_IEE_H -+#define _LINUX_IEE_H -+#define __iee_code __section(".iee.text") -+#define __iee_header __section(".iee.text.header") ++static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud) ++{ ++ #ifdef __PAGETABLE_PUD_FOLDED ++ if (in_swapper_pgdir(pudp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud))); ++ return; ++ } ++ #endif + -+u64 iee_dispatch(int flag, ...); ++ WRITE_ONCE(*pudp, pud); + -+#include ++ dsb(ishst); ++ isb(); ++} + -+#endif -diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h -index 85d26143faa5..e7a3081ce285 100644 ---- a/arch/arm64/include/asm/kernel-pgtable.h -+++ b/arch/arm64/include/asm/kernel-pgtable.h -@@ -118,4 +118,25 @@ - #define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY) - #endif - -+#ifdef CONFIG_IEE ++static inline void __maybe_unused iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) ++{ ++ if (in_swapper_pgdir(p4dp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } + -+#ifdef CONFIG_ARM64_4K_PAGES // zgcXXX: it has been deleted in 6.6. -+#define ARM64_SWAPPER_USES_SECTION_MAPS 1 -+#else -+#define ARM64_SWAPPER_USES_SECTION_MAPS 0 -+#endif ++ WRITE_ONCE(*p4dp, p4d); ++ dsb(ishst); ++ isb(); ++} + -+#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) // zgcXXX: warning: 6.6 delete this macro. should delete this line later. ++static inline void iee_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++{ ++ if (in_swapper_pgdir(pgdp)) { ++ iee_set_swapper_pgd_pre_init(pgdp, __pgd(pgd_val(pgd))); ++ return; ++ } + -+#define SWAPPER_PTE_FLAGS_IDMAP (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED | PTE_RDONLY) -+#define SWAPPER_PMD_FLAGS_IDMAP (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S | PMD_SECT_RDONLY) ++ WRITE_ONCE(*pgdp, pgd); ++ dsb(ishst); ++ isb(); ++} + -+#if ARM64_SWAPPER_USES_SECTION_MAPS -+#define SWAPPER_MM_MMUFLAGS_IDMAP (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS_IDMAP) -+#else -+#define SWAPPER_MM_MMUFLAGS_IDMAP (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS_IDMAP) -+#endif ++#define pte_set_fixmap_pre_init(addr) \ ++ ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr)) ++#define pte_set_fixmap_offset_pre_init(pmd, addr) \ ++ pte_set_fixmap_pre_init(pte_offset_phys(pmd, addr)) ++#define pte_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PTE) + -+#endif ++#define pmd_set_fixmap_pre_init(addr) \ ++ ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr)) ++#define pmd_set_fixmap_offset_pre_init(pud, addr) \ ++ pmd_set_fixmap_pre_init(pmd_offset_phys(pud, addr)) ++#define pmd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PMD) + - #endif /* __ASM_KERNEL_PGTABLE_H */ -diff --git a/arch/arm64/include/asm/koi.h b/arch/arm64/include/asm/koi.h -new file mode 100644 -index 000000000000..c9afe886bef8 ---- /dev/null -+++ b/arch/arm64/include/asm/koi.h -@@ -0,0 +1,532 @@ -+#include "linux/mm.h" -+#include "asm/current.h" -+#include "asm/pgtable-hwdef.h" -+#include "asm/pgtable-types.h" -+#include "asm/pgtable.h" -+#include "linux/mm_types.h" -+#include "linux/pgtable.h" -+#include "linux/printk.h" -+#include "linux/slab.h" -+#include "linux/string.h" -+#include -+#include -+#include "linux/hashtable.h" -+#include "linux/module.h" -+#include "linux/vmalloc.h" -+#include "stacktrace.h" -+#include "asm/mmu.h" -+#ifdef CONFIG_IEE -+#include "asm/iee-si.h" -+#include "asm/iee-def.h" -+#include -+#endif ++#define pud_set_fixmap_pre_init(addr) \ ++ ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr)) ++#define pud_set_fixmap_offset_pre_init(p4d, addr)\ ++ pud_set_fixmap_pre_init(pud_offset_phys(p4d, addr)) ++#define pud_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PUD) + -+#ifdef CONFIG_KOI ++#define pgd_set_fixmap_pre_init(addr) \ ++ ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr)) ++#define pgd_clear_fixmap_pre_init() clear_fixmap_pre_init(FIX_PGD) + -+#define HASH_TABLE_BIT 10 -+#define HASH_TABLE_LEN (1 << HASH_TABLE_BIT) -+#define HASH_KEY_MASK ((1 << HASH_TABLE_BIT) - 1) ++static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++{ ++ WRITE_ONCE(*pgdp, pgd); ++ dsb(ishst); ++ isb(); ++} + -+#define MAX_VAR_NAME 64 -+#define DRIVER_ISOLATION_VAR_ARRAY_SIZE 32 -+#define DRIVER_ISOLATION_MAX_VAL 256 ++static inline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) ++{ ++ pteval_t ret; + -+extern struct hlist_head koi_mem_htbl[1024]; -+extern spinlock_t koi_mem_htbl_spin_lock; -+extern unsigned long koi_swapper_ttbr1; -+extern s64 koi_offset; ++ ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval); ++ return (pteval_t)ret; ++} + -+#ifdef CONFIG_IEE -+extern unsigned long long iee_rw_gate(int flag, ...); -+#endif ++static inline pmdval_t iee_set_pmd_xchg_relaxed(pmd_t *pmdp, pmdval_t pmdval) ++{ ++ pmdval_t ret; + -+DECLARE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)], -+ koi_irq_current_ttbr1); -+ -+/** -+* struct koi_mem_hash_node - -+*@mod:pointer to driver module -+*@mem_list_head:free memory list head -+*@ko_mm: mm_struct in each driver -+*@pgdp:entry to Page Global Directory :pgd -+*@node:hash linked list node -+*@addr_htbl[1 << (HASH_TABLE_BIT)]: -+*@rcu: -+*/ -+struct koi_mem_hash_node { -+ struct module *mod; -+ struct list_head mem_list_head; -+ struct mm_struct *ko_mm; -+ pgd_t *pgdp; -+ unsigned long ko_ttbr1; -+ struct hlist_node node; -+ struct hlist_head addr_htbl[1 << (HASH_TABLE_BIT)]; -+ struct rcu_head rcu; -+ // used to protect free mem list -+ spinlock_t spin_lock; -+ // used to protect addr hashtable -+ spinlock_t addr_htbl_spin_lock; -+ bool is_valid; -+ spinlock_t mod_lock; -+}; -+//describe the global shared var -+struct shared_variable_descriptor { -+ unsigned int id; -+ unsigned int type; -+ char name[MAX_VAR_NAME]; -+ unsigned long offset; -+ unsigned int size; -+ unsigned int self_ptr_ids[DRIVER_ISOLATION_VAR_ARRAY_SIZE]; -+}; ++ ret = iee_rw_gate(IEE_OP_SET_PMD_XCHG, pmdp, pmdval); ++ return (pmdval_t)ret; ++} + -+int koi_do_switch_to_kernel_pgtbl(void); ++static inline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, ++ pteval_t new_pteval) ++{ ++ pteval_t ret; + -+int koi_share_kstack(struct module *mod); ++ ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval); ++ return ret; ++} + -+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, -+ unsigned long addr, unsigned long end, pteval_t prot); ++static inline pmdval_t iee_set_pmd_cmpxchg_relaxed(pmd_t *pmdp, pmdval_t old_pmdval, ++ pmdval_t new_pmdval) ++{ ++ pmdval_t ret; + -+void koi_create_pagetable(struct module *mod); -+void koi_destroy_pagetable(struct module *mod); ++ ret = iee_rw_gate(IEE_OP_SET_PMD_CMPXCHG, pmdp, old_pmdval, new_pmdval); ++ return ret; ++} + -+void koi_map_kostack(struct module *mod); -+unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr, -+ unsigned long size); -+void koi_mem_free(struct module *mod, unsigned long addr, unsigned long size, -+ bool is_const, int count, ...); -+void *koi_mem_lookup(struct module *mod, unsigned long addr); -+void koi_mem_free_callback(struct module *mod, unsigned long addr, -+ unsigned long size, void (*func)(void *)); -+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size); -+void koi_unmap_mem(struct module *mod, unsigned long addr, unsigned long size); -+void koi_mem_free_to_user(struct module *mod, unsigned long addr, -+ unsigned long size); ++static inline void iee_set_static_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd); ++} + -+unsigned long koi_ttbr_ctor(struct module *mod); ++static inline void iee_set_bm_pte(pte_t *ptep, pte_t pte) ++{ ++ iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte); + -+// unsigned long koi_get_token_addr(struct task_struct *tsk); ++ /* ++ * Only if the new pte is valid and kernel, otherwise TLB maintenance ++ * or update_mmu_cache() have the necessary barriers. ++ */ ++ dsb(ishst); ++ isb(); ++} + -+#define switch_pgtable(ttbr1) \ -+ do { \ -+ write_sysreg((ttbr1), ttbr1_el1); \ -+ isb(); \ -+ asm volatile(ALTERNATIVE("nop; nop; nop", \ -+ "ic iallu; dsb nsh; isb", \ -+ ARM64_WORKAROUND_CAVIUM_27456)); \ -+ } while (0); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE + -+#ifndef CONFIG_IEE -+#define koi_switch_to_ko() \ -+ do { \ -+ unsigned long flags, ko_ttbr1, cur_sp; \ -+ unsigned long *ptr; \ -+ struct task_token *token; \ -+ asm volatile("mrs %0, daif\n" \ -+ "msr daifset, #2\n" \ -+ "isb\n" \ -+ "mov %1, sp\n" \ -+ : "=r"(flags), "=r"(cur_sp) \ -+ :); \ -+ if (!on_irq_stack(cur_sp, (unsigned long)NULL)) { \ -+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ -+ token = (struct task_token *)((unsigned long)current + \ -+ (unsigned long) \ -+ koi_offset); \ -+ token->current_ttbr1 = ko_ttbr1 & (~TTBR_ASID_MASK); \ -+ } else { \ -+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ -+ __kern_my_cpu_offset()); \ -+ *ptr = ko_ttbr1 & ~(TTBR_ASID_MASK); \ -+ } \ -+ switch_pgtable(ko_ttbr1); \ -+ asm volatile("msr daif, %0\n" \ -+ "isb\n" \ -+ : \ -+ : "r"(flags)); \ -+ } while (0); -+ -+#define koi_switch_to_kernel() \ -+ do { \ -+ unsigned long cur_sp, flags, asid; \ -+ unsigned long *ptr; \ -+ struct task_token *token; \ -+ asm volatile("mrs %0, daif\n" \ -+ "msr daifset, #2\n" \ -+ "isb\n" \ -+ "mov %1, sp\n" \ -+ : "=r"(flags), "=r"(cur_sp) \ -+ :); \ -+ asid = read_sysreg(ttbr0_el1) & (~USER_ASID_FLAG); \ -+ asid &= TTBR_ASID_MASK; \ -+ switch_pgtable((koi_swapper_ttbr1 | asid)); \ -+ if (!on_irq_stack(cur_sp, (unsigned long)NULL)) { \ -+ token = (struct task_token *)((unsigned long)current + \ -+ (unsigned long)koi_offset); \ -+ token->current_ttbr1 = koi_swapper_ttbr1; \ -+ } else { \ -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ -+ __kern_my_cpu_offset()); \ -+ *ptr = koi_swapper_ttbr1; \ -+ } \ -+ asm volatile("msr daif, %0\n" \ -+ "isb\n" \ -+ : \ -+ : "r"(flags)); \ -+ } while (0); -+#else -+#define koi_switch_to_ko() \ -+ do { \ -+ unsigned long cur_sp, flags, ko_ttbr1; \ -+ unsigned long *ptr; \ -+ asm volatile("mrs %0, daif\n" \ -+ "msr daifset, #2\n" \ -+ "isb\n" \ -+ "mov %1, sp\n" \ -+ : "=r"(flags), "=r"(cur_sp) \ -+ :); \ -+ if (!on_irq_stack(cur_sp, (unsigned long)NULL)) { \ -+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ -+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \ -+ ko_ttbr1 &(~TTBR_ASID_MASK)); \ -+ } else { \ -+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ -+ __kern_my_cpu_offset()); \ -+ *ptr = ko_ttbr1 & (~TTBR_ASID_MASK); \ -+ } \ -+ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, ko_ttbr1); \ -+ asm volatile("msr daif, %0\n" \ -+ "isb\n" \ -+ : \ -+ : "r"(flags)); \ -+ } while (0); -+ -+#define koi_switch_to_kernel() \ -+ do { \ -+ unsigned long flags, cur_sp; \ -+ unsigned long *ptr; \ -+ asm volatile("mrs %0, daif\n" \ -+ "msr daifset, #2\n" \ -+ "isb\n" \ -+ "mov %1, sp\n" \ -+ : "=r"(flags), "=r"(cur_sp) \ -+ :); \ -+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); \ -+ if (!on_irq_stack(cur_sp, (unsigned long)NULL)) { \ -+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \ -+ koi_swapper_ttbr1); \ -+ } else { \ -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ -+ __kern_my_cpu_offset()); \ -+ *ptr = koi_swapper_ttbr1; \ -+ } \ -+ asm volatile("msr daif, %0\n" \ -+ "isb\n" \ -+ : \ -+ : "r"(flags)); \ -+ } while (0); -+#endif -+//kzalloc function in driver space -+static __maybe_unused noinline void * -+koi_kzalloc_wrapper(struct module *mod, size_t size, gfp_t flags) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk("mem node for module: %s not found\n", mod->name); -+ return NULL; -+ } ++/* Set stride and tlb_level in flush_*_tlb_range */ ++#define flush_pmd_tlb_range(vma, addr, end) \ ++ __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) ++#define flush_pud_tlb_range(vma, addr, end) \ ++ __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + -+ addr = kzalloc(size, flags); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ koi_switch_to_ko(); -+ return addr; -+} -+ -+static __maybe_unused noinline void *koi_kzalloc_node_wrapper(struct module *mod, size_t size, gfp_t flags, int node) { -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr = NULL; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu(koi_mem_htbl, target, node, (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", mod->name); -+ goto ret; -+ } -+ addr = kzalloc_node(cnt * PAGE_SIZE, flags, node); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ret: -+ koi_switch_to_ko(); -+ return (void *)addr; -+} -+ -+//kmalloc function in driver space -+static __maybe_unused void * -+koi_kmalloc_wrapper(struct module *mod, size_t size, gfp_t flags) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr = NULL; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR"mem node for module: %s not found\n", mod->name); -+ goto ret; -+ } ++/* ++ * Outside of a few very special situations (e.g. hibernation), we always ++ * use broadcast TLB invalidation instructions, therefore a spurious page ++ * fault on one CPU which has been handled concurrently by another CPU ++ * does not need to perform additional invalidation. ++ */ ++#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) + -+ addr = kmalloc(cnt * PAGE_SIZE, flags); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ret: -+ koi_switch_to_ko(); -+ return (void *)addr; -+} -+//vmalloc function in driver space -+static __maybe_unused void *koi_vmalloc_wrapper(struct module *mod, -+ unsigned long size) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk("mem node for module: %s not found\n", mod->name); -+ koi_switch_to_ko(); -+ return 0; -+ } -+ addr = vmalloc(cnt * PAGE_SIZE); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ koi_switch_to_ko(); -+ return addr; -+} -+//kmalloc_array function in driver space -+static __maybe_unused void *koi_kmalloc_array_wrapper(struct module *mod, -+ size_t n, size_t size, -+ gfp_t flags) -+{ -+ int kpage; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk("mem node for module: %s not found\n", mod->name); -+ koi_switch_to_ko(); -+ return 0; -+ } -+ kpage = (n * size + PAGE_SIZE - 1) / PAGE_SIZE; -+ n = (kpage * PAGE_SIZE) / size; -+ addr = kmalloc_array(n, size, flags); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * kpage, 0); -+ koi_switch_to_ko(); -+ return addr; -+} ++/* ++ * ZERO_PAGE is a global shared page that is always zero: used ++ * for zero-mapped memory areas etc.. ++ */ ++extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; ++#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page)) + -+static __maybe_unused noinline void *koi_kcalloc_wrapper(struct module *mod, size_t n, size_t size, gfp_t flags) { -+ return koi_kmalloc_array_wrapper(mod, n, size, flags | __GFP_ZERO); ++#define pte_ERROR(e) \ ++ pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) ++ ++/* ++ * Macros to convert between a physical address and its placement in a ++ * page table entry, taking care of 52-bit addresses. ++ */ ++#ifdef CONFIG_ARM64_PA_BITS_52 ++static inline phys_addr_t __pte_to_phys(pte_t pte) ++{ ++ return (pte_val(pte) & PTE_ADDR_LOW) | ++ ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT); ++} ++static inline pteval_t __phys_to_pte_val(phys_addr_t phys) ++{ ++ return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK; +} ++#else ++#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK) ++#define __phys_to_pte_val(phys) (phys) +#endif + -+#ifdef CONFIG_KOI ++#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT) ++#define pfn_pte(pfn, prot) \ ++ __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) ++ ++#define pte_none(pte) (!pte_val(pte)) ++#define __pte_clear(mm, addr, ptep) \ ++ __set_pte(ptep, __pte(0)) ++#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) + -+#define koi_copy_to_user_wrapper(to, from, n) \ -+({ \ -+ koi_switch_to_kernel(); \ -+ long long ret = copy_to_user(to, from, n); \ -+ koi_switch_to_ko(); \ -+ ret; \ ++/* ++ * The following only work if pte_present(). Undefined behaviour otherwise. ++ */ ++#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE))) ++#define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) ++#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) ++#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) ++#define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY)) ++#define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) ++#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) ++#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) ++#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP)) ++#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ ++ PTE_ATTRINDX(MT_NORMAL_TAGGED)) ++ ++#define pte_cont_addr_end(addr, end) \ ++({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ ++ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + -+#define koi_copy_from_user_wrapper(to, from, n) \ -+({ \ -+ koi_switch_to_kernel(); \ -+ long long ret = copy_from_user(to, from, n); \ -+ koi_switch_to_ko(); \ -+ ret; \ ++#define pmd_cont_addr_end(addr, end) \ ++({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \ ++ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + -+#define koi_kasprintf_wrapper(gfp, fmt, args...)\ -+ ({ \ -+ koi_switch_to_kernel(); \ -+ void *ret = kasprintf(gfp, fmt, ##args); \ -+ koi_map_mem(THIS_MODULE, (unsigned long)ret, sizeof(void *)); \ -+ koi_switch_to_ko(); \ -+ ret;\ -+ }) -+ -+#define koi_scnprintf_wrapper(buf, size, fmt, args...) \ -+ ({ \ -+ int ret; \ -+ koi_switch_to_kernel(); \ -+ ret = scnprintf(buf, size, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+ }) -+ -+#define koi_sscanf_wrapper(buf, fmt, args...) \ -+ ({ \ -+ int ret; \ -+ koi_switch_to_kernel(); \ -+ ret = sscanf(buf, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+ }) -+ -+#define koi_rcu_read_lock_wrapper() \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ rcu_read_lock(); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_rcu_read_unlock_wrapper() \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ rcu_read_unlock(); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_lock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_lock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_unlock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_unlock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_init_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_init(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_unlock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_irq_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_irq(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+ -+#define koi_spin_unlock_irq_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock_irq(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_irqsave_wrapper(lock, flags) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_irqsave(lock, flags); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+ -+#define koi_spin_lock_irqrestore_wrapper(lock, flags) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_irqrestore(lock, flags); \ -+ koi_switch_to_ko(); \ -+ } while(0); ++#define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte)) ++#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) ++#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) + -+#else ++#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) ++/* ++ * Execute-only user mappings do not have the PTE_USER bit set. All valid ++ * kernel mappings have the PTE_UXN bit set. ++ */ ++#define pte_valid_not_user(pte) \ ++ ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN)) ++/* ++ * Returns true if the pte is valid and has the contiguous bit set. ++ */ ++#define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) ++/* ++ * Could the pte be present in the TLB? We must check mm_tlb_flush_pending ++ * so that we don't erroneously return false for pages that have been ++ * remapped as PROT_NONE but are yet to be flushed from the TLB. ++ * Note that we can't make any assumptions based on the state of the access ++ * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the ++ * TLB. ++ */ ++#define pte_accessible(mm, pte) \ ++ (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) + -+#define koi_copy_to_user_wrapper copy_to_user ++/* ++ * p??_access_permitted() is true for valid user mappings (PTE_USER ++ * bit set, subject to the write permission check). For execute-only ++ * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits ++ * not set) must return false. PROT_NONE mappings do not have the ++ * PTE_VALID bit set. ++ */ ++#define pte_access_permitted(pte, write) \ ++ (((pte_val(pte) & (PTE_VALID | PTE_USER)) ==\ ++ (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte))) ++#define pmd_access_permitted(pmd, write) \ ++ (pte_access_permitted(pmd_pte(pmd), (write))) ++#define pud_access_permitted(pud, write) \ ++ (pte_access_permitted(pud_pte(pud), (write))) + -+#define koi_copy_from_user_wrapper copy_from_user ++static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) ++{ ++ pte_val(pte) &= ~pgprot_val(prot); ++ return pte; ++} + -+#define koi_kasprintf_wrapper kasprintf ++static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) ++{ ++ pte_val(pte) |= pgprot_val(prot); ++ return pte; ++} + -+#define koi_scnprintf_wrapper scnprintf ++static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot) ++{ ++ pmd_val(pmd) &= ~pgprot_val(prot); ++ return pmd; ++} + -+#define koi_sscanf_wrapper sscanf ++static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) ++{ ++ pmd_val(pmd) |= pgprot_val(prot); ++ return pmd; ++} + -+#define koi_rcu_read_lock_wrapper rcu_read_lock ++static inline pte_t pte_mkwrite_novma(pte_t pte) ++{ ++ pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); ++ pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); ++ return pte; ++} + -+#define koi_rcu_read_unlock_wrapper rcu_read_unlock ++static inline pte_t pte_mkclean(pte_t pte) ++{ ++ pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); ++ pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + -+#define koi_mutex_lock_wrapper mutex_lock ++ return pte; ++} + -+#define koi_mutex_unlock_wrapper mutex_unlock ++static inline pte_t pte_mkdirty(pte_t pte) ++{ ++ pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + -+#define koi_mutex_init_wrapper mutex_init ++ if (pte_write(pte)) ++ pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + -+#define koi_spin_lock_irq_wrapper spin_lock_irq ++ return pte; ++} + -+#define koi_spin_unlock_irq_wrapper spin_unlock_irq ++static inline pte_t pte_wrprotect(pte_t pte) ++{ ++ /* ++ * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY ++ * clear), set the PTE_DIRTY bit. ++ */ ++ if (pte_hw_dirty(pte)) ++ pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); + -+#define koi_spin_lock_wrapper spin_lock ++ pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); ++ pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); ++ return pte; ++} + -+#define koi_spin_unlock_wrapper spin_unlock ++static inline pte_t pte_mkold(pte_t pte) ++{ ++ return clear_pte_bit(pte, __pgprot(PTE_AF)); ++} + -+#define koi_spin_lock_irqsave_wrapper spin_lock_irqsave ++static inline pte_t pte_mkyoung(pte_t pte) ++{ ++ return set_pte_bit(pte, __pgprot(PTE_AF)); ++} + -+#define koi_spin_lock_irqrestore_wrapper spin_lock_irqrestore ++static inline pte_t pte_mkspecial(pte_t pte) ++{ ++ return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); ++} + -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h -index fde4186cc387..c9e9a5a288bc 100644 ---- a/arch/arm64/include/asm/memory.h -+++ b/arch/arm64/include/asm/memory.h -@@ -190,6 +190,13 @@ extern u64 vabits_actual; - #endif - - extern s64 memstart_addr; ++static inline pte_t pte_mkcont(pte_t pte) ++{ ++ pte = set_pte_bit(pte, __pgprot(PTE_CONT)); ++ return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE)); ++} + -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+extern s64 memstart_addr_init; -+extern s64 iee_offset; -+#define LOGICAL_RANDOM (long long int)((long unsigned int)__va(memstart_addr_init) & (~PAGE_OFFSET)) -+#endif ++static inline pte_t pte_mknoncont(pte_t pte) ++{ ++ return clear_pte_bit(pte, __pgprot(PTE_CONT)); ++} + - /* PHYS_OFFSET - the physical address of the start of memory. */ - #define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; }) - -@@ -310,6 +317,27 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); - #define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET) - #define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset)) - -+#ifdef CONFIG_KOI -+#define KOI_OFFSET (((unsigned long)BIT(vabits_actual - 2)) - LOGICAL_RANDOM) -+#endif ++static inline pte_t pte_mkpresent(pte_t pte) ++{ ++ return set_pte_bit(pte, __pgprot(PTE_VALID)); ++} + -+#ifdef CONFIG_IEE -+#ifdef CONFIG_IEE_OFFSET -+#define IEE_OFFSET ((CONFIG_IEE_OFFSET) - LOGICAL_RANDOM) -+#else -+#define IEE_OFFSET (((unsigned long)BIT(vabits_actual - 2)) - LOGICAL_RANDOM) -+#endif -+#define __phys_to_iee(x) (__phys_to_virt(x) + IEE_OFFSET) -+#define SET_UPAGE(x) __pgprot(pgprot_val(x) | PTE_USER) -+#define SET_PPAGE(x) __pgprot(pgprot_val(x) & (~PTE_USER)) -+#define SET_INVALID(x) __pgprot(pgprot_val(x) & (~PTE_VALID)) -+#define SET_NG(x) __pgprot(pgprot_val(x) | PTE_NG) -+#endif ++static inline pmd_t pmd_mkcont(pmd_t pmd) ++{ ++ return __pmd(pmd_val(pmd) | PMD_SECT_CONT); ++} + -+#if defined(CONFIG_IEE) || defined (CONFIG_KOI) -+#define SET_NG(x) __pgprot(pgprot_val(x) | PTE_NG) -+#define SET_INVALID(x) __pgprot(pgprot_val(x) & (~PTE_VALID)) -+#endif - /* - * Convert a page to/from a physical address - */ -diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h -index a6fb325424e7..cca5994dabfb 100644 ---- a/arch/arm64/include/asm/mmu_context.h -+++ b/arch/arm64/include/asm/mmu_context.h -@@ -24,6 +24,9 @@ - #include - #include - #include -+#ifdef CONFIG_IEE -+#define INIT_ASID 0x2 -+#endif - - extern bool rodata_full; - -@@ -43,7 +46,12 @@ static inline void cpu_set_reserved_ttbr0_nosync(void) - { - unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); - -+#ifdef CONFIG_IEE -+ ttbr |= FIELD_PREP(TTBR_ASID_MASK, 1); -+ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr); -+#else - write_sysreg(ttbr, ttbr0_el1); -+#endif - } - - static inline void cpu_set_reserved_ttbr0(void) -@@ -79,7 +87,11 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) - - tcr &= ~TCR_T0SZ_MASK; - tcr |= t0sz << TCR_T0SZ_OFFSET; -+#ifdef CONFIG_IEE -+ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr); -+#else - write_sysreg(tcr, tcr_el1); -+#endif - isb(); - } - -@@ -144,7 +156,11 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) - __cpu_set_tcr_t0sz(t0sz); - - /* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */ -+ #ifdef CONFIG_IEE -+ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr0); -+ #else - write_sysreg(ttbr0, ttbr0_el1); -+ #endif - isb(); - } - -@@ -174,6 +190,10 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) - ttbr1 |= TTBR_CNP_BIT; - } - -+ #ifdef CONFIG_IEE -+ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, ASID(current->active_mm)); -+ #endif ++static inline pte_t pte_mkdevmap(pte_t pte) ++{ ++ return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL)); ++} + - replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); - - __cpu_install_idmap(idmap); -diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h -index 661964e99b9d..6e40b14db920 100644 ---- a/arch/arm64/include/asm/pgalloc.h -+++ b/arch/arm64/include/asm/pgalloc.h -@@ -63,6 +63,10 @@ static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) - extern pgd_t *pgd_alloc(struct mm_struct *mm); - extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp); - -+#ifdef CONFIG_KOI -+pgd_t *koi_pgd_alloc(void); -+#endif ++static inline void __set_pte(pte_t *ptep, pte_t pte) ++{ ++ iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); ++ dsb(ishst); ++ isb(); ++} + - static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, - pmdval_t prot) - { -diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h -index f736a4222190..babc366acc51 100644 ---- a/arch/arm64/include/asm/pgtable-hwdef.h -+++ b/arch/arm64/include/asm/pgtable-hwdef.h -@@ -84,6 +84,13 @@ - #define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) - #define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) - -+#ifdef CONFIG_IEE -+#define PGD_APT_RO (_AT(pudval_t, 1) << 62) -+#endif -+#define PGD_APT (_AT(pudval_t, 1) << 61) -+#define PGD_PXN (_AT(pudval_t, 1) << 59) -+#define PGD_UXN (_AT(pudval_t, 1) << 60) ++static inline pte_t __ptep_get(pte_t *ptep) ++{ ++ return READ_ONCE(*ptep); ++} + - /* - * Hardware page table definitions. - * -@@ -157,6 +164,9 @@ - #define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */ - #define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */ - #define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */ -+#ifdef CONFIG_HIVE -+#define PTE_BPF_SFI_GP (_AT(pteval_t, 1) << 55) /* BPF_SFI guarded */ -+#endif - - #define PTE_ADDR_LOW (((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT) - #ifdef CONFIG_ARM64_PA_BITS_52 -@@ -288,6 +298,10 @@ - #define TCR_TCMA0 (UL(1) << 57) - #define TCR_TCMA1 (UL(1) << 58) - -+#ifdef CONFIG_IEE -+#define TCR_HPD1 (UL(1) << 42) -+#endif ++extern void __sync_icache_dcache(pte_t pteval); ++bool pgattr_change_is_safe(u64 old, u64 new); + - /* - * TTBR. - */ -diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h -index d457dd74f534..89206b1a517b 100644 ---- a/arch/arm64/include/asm/pgtable.h -+++ b/arch/arm64/include/asm/pgtable.h -@@ -34,6 +34,9 @@ - #include - #include - #include -+#ifdef CONFIG_PTP -+#include -+#endif - - #ifdef CONFIG_TRANSPARENT_HUGEPAGE - #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE -@@ -156,6 +159,14 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) - #define pud_access_permitted(pud, write) \ - (pte_access_permitted(pud_pte(pud), (write))) - -+#ifdef CONFIG_PTP -+extern bool in_tramp_pgdir(void *addr); -+extern unsigned long long iee_rw_gate(int flag, ...); -+extern void iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd); -+extern pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval); -+extern pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval); -+#endif ++/* ++ * PTE bits configuration in the presence of hardware Dirty Bit Management ++ * (PTE_WRITE == PTE_DBM): ++ * ++ * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) ++ * 0 0 | 1 0 0 ++ * 0 1 | 1 1 0 ++ * 1 0 | 1 0 1 ++ * 1 1 | 0 1 x ++ * ++ * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via ++ * the page fault mechanism. Checking the dirty status of a pte becomes: ++ * ++ * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) ++ */ + - static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) - { - pte_val(pte) &= ~pgprot_val(prot); -@@ -260,19 +271,35 @@ static inline pte_t pte_mkdevmap(pte_t pte) - return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL)); - } - -+#ifdef CONFIG_PTP -+extern void iee_set_bm_pte(pte_t *ptep, pte_t pte); -+extern void iee_set_fixmap_pte_pre_init(pte_t *ptep, pte_t pte); -+#endif ++static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, ++ pte_t pte) ++{ ++ pte_t old_pte; + -+#ifdef CONFIG_IEE -+extern void __set_pte(pte_t *ptep, pte_t pte); -+#else - static inline void __set_pte(pte_t *ptep, pte_t pte) - { -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) { -+ pte = __pte(pte_val(pte) | PTE_NG); -+ } -+#endif - WRITE_ONCE(*ptep, pte); -- -- /* -- * Only if the new pte is valid and kernel, otherwise TLB maintenance -- * or update_mmu_cache() have the necessary barriers. -- */ - if (pte_valid_not_user(pte)) { - dsb(ishst); - isb(); - } - } -+#endif ++ if (!IS_ENABLED(CONFIG_DEBUG_VM)) ++ return; + -+#ifdef CONFIG_IEE -+extern void iee_set_stack_pte(pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr); -+extern void iee_unset_stack_pte(pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr); -+extern void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); -+extern void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); -+#endif - - static inline pte_t __ptep_get(pte_t *ptep) - { -@@ -361,8 +388,15 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) - return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); - } - --static inline void __set_ptes(struct mm_struct *mm, -- unsigned long __always_unused addr, -+#ifdef CONFIG_PTP -+extern void iee_set_pte_pre_init(pte_t *ptep, pte_t pte); -+extern void iee_set_pte_at(struct mm_struct *mm, unsigned long addr, -+ pte_t *ptep, pte_t pte); -+extern void iee_set_pte_at_delayed(struct mm_struct *mm, unsigned long addr, -+ pte_t *ptep, pte_t pte); -+#endif ++ old_pte = __ptep_get(ptep); + -+static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, unsigned int nr) - { - page_table_check_ptes_set(mm, ptep, pte, nr); -@@ -546,6 +580,44 @@ static inline void __set_pte_at(struct mm_struct *mm, - __set_pte(ptep, pte); - } - -+#ifdef CONFIG_PTP -+extern void set_pmd(pmd_t *pmdp, pmd_t pmd); -+extern void set_pud(pud_t *pudp, pud_t pud); ++ if (!pte_valid(old_pte) || !pte_valid(pte)) ++ return; ++ if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1) ++ return; + -+static inline void __set_pmd_at(struct mm_struct *mm, -+ unsigned long __always_unused addr, -+ pmd_t *pmdp, pmd_t pmd, unsigned int nr) ++ /* ++ * Check for potential race with hardware updates of the pte ++ * (__ptep_set_access_flags safely changes valid ptes without going ++ * through an invalid entry). ++ */ ++ VM_WARN_ONCE(!pte_young(pte), ++ "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", ++ __func__, pte_val(old_pte), pte_val(pte)); ++ VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), ++ "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", ++ __func__, pte_val(old_pte), pte_val(pte)); ++ VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), ++ "%s: unsafe attribute change: 0x%016llx -> 0x%016llx", ++ __func__, pte_val(old_pte), pte_val(pte)); ++} ++ ++static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages) +{ -+ __sync_cache_and_tags(pmd_pte(pmd), nr); -+ __check_safe_pte_update(mm, (pte_t *)pmdp, pmd_pte(pmd)); -+ set_pmd(pmdp, pmd); ++ if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) ++ __sync_icache_dcache(pte); ++ ++ /* ++ * If the PTE would provide user space access to the tags associated ++ * with it then ensure that the MTE tags are synchronised. Although ++ * pte_access_permitted() returns false for exec only mappings, they ++ * don't expose tags (instruction fetches don't check tags). ++ */ ++ if (system_supports_mte() && pte_access_permitted(pte, false) && ++ !pte_special(pte) && pte_tagged(pte)) ++ mte_sync_tags(pte, nr_pages); +} + -+static inline void __set_pud_at(struct mm_struct *mm, -+ unsigned long __always_unused addr, -+ pud_t *pudp, pud_t pud, unsigned int nr) ++/* ++ * Select all bits except the pfn ++ */ ++static inline pgprot_t pte_pgprot(pte_t pte) +{ -+ __sync_cache_and_tags(pud_pte(pud), nr); -+ __check_safe_pte_update(mm, (pte_t *)pudp, pud_pte(pud)); -+ set_pud(pudp, pud); ++ unsigned long pfn = pte_pfn(pte); ++ ++ return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + -+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, -+ pmd_t *pmdp, pmd_t pmd) ++#define pte_advance_pfn pte_advance_pfn ++static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) +{ -+ page_table_check_pmd_set(mm, pmdp, pmd); -+ return __set_pmd_at(mm, addr, pmdp, pmd, -+ PMD_SIZE >> PAGE_SHIFT); ++ return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); +} + -+static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, -+ pud_t *pudp, pud_t pud) ++static inline void __set_ptes(struct mm_struct *mm, ++ unsigned long __always_unused addr, ++ pte_t *ptep, pte_t pte, unsigned int nr) +{ -+ page_table_check_pud_set(mm, pudp, pud); -+ return __set_pud_at(mm, addr, pudp, pud, -+ PUD_SIZE >> PAGE_SHIFT); ++ page_table_check_ptes_set(mm, ptep, pte, nr); ++ __sync_cache_and_tags(pte, nr); ++ ++ for (;;) { ++ __check_safe_pte_update(mm, ptep, pte); ++ __set_pte(ptep, pte); ++ if (--nr == 0) ++ break; ++ ptep++; ++ pte = pte_advance_pfn(pte, 1); ++ } +} -+#else - static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp, pmd_t pmd) - { -@@ -561,6 +633,7 @@ static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, - return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud), - PUD_SIZE >> PAGE_SHIFT); - } -+#endif - - #define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d)) - #define __phys_to_p4d_val(phys) __phys_to_pte_val(phys) -@@ -640,6 +713,14 @@ static inline bool in_swapper_pgdir(void *addr) - ((unsigned long)swapper_pg_dir & PAGE_MASK); - } - -+#ifdef CONFIG_PTP -+extern bool in_tramp_pgdir(void *addr); -+extern void iee_set_fixmap_pmd_pre_init(pmd_t *pmdp, pmd_t pmd); -+#endif + -+#ifdef CONFIG_IEE -+extern void set_pmd(pmd_t *pmdp, pmd_t pmd); -+#else - static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) - { - #ifdef __PAGETABLE_PMD_FOLDED -@@ -648,14 +729,19 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) - return; - } - #endif /* __PAGETABLE_PMD_FOLDED */ -- -+#ifdef CONFIG_KOI -+ pmdval_t val = pmd_val(pmd); -+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { -+ pmd = __pmd(val | PMD_SECT_NG); -+ } -+#endif - WRITE_ONCE(*pmdp, pmd); -- - if (pmd_valid(pmd)) { - dsb(ishst); - isb(); - } - } -+#endif - - static inline void pmd_clear(pmd_t *pmdp) - { -@@ -675,6 +761,12 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) - /* Find an entry in the third-level page table. */ - #define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t)) - -+#ifdef CONFIG_PTP -+#define pte_set_fixmap_init(addr) ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr)) -+#define pte_set_fixmap_offset_init(pmd, addr) pte_set_fixmap_init(pte_offset_phys(pmd, addr)) -+#define pte_clear_fixmap_init() clear_fixmap_init(FIX_PTE) -+#endif ++/* ++ * Huge pte definitions. ++ */ ++#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT)) + - #define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) - #define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) - #define pte_clear_fixmap() clear_fixmap(FIX_PTE) -@@ -703,6 +795,14 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) - #define pud_user(pud) pte_user(pud_pte(pud)) - #define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) - ++/* ++ * Hugetlb definitions. ++ */ ++#define HUGE_MAX_HSTATE 4 ++#define HPAGE_SHIFT PMD_SHIFT ++#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) ++#define HPAGE_MASK (~(HPAGE_SIZE - 1)) ++#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + -+#ifdef CONFIG_PTP -+extern void iee_set_fixmap_pud_pre_init(pud_t *pudp, pud_t pud); -+#endif ++static inline pte_t pgd_pte(pgd_t pgd) ++{ ++ return __pte(pgd_val(pgd)); ++} + -+#ifdef CONFIG_IEE -+extern void set_pud(pud_t *pudp, pud_t pud); -+#else - static inline void set_pud(pud_t *pudp, pud_t pud) - { - #ifdef __PAGETABLE_PUD_FOLDED -@@ -711,14 +811,20 @@ static inline void set_pud(pud_t *pudp, pud_t pud) - return; - } - #endif /* __PAGETABLE_PUD_FOLDED */ -- -+#ifdef CONFIG_KOI -+ pudval_t val = pud_val(pud); -+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { -+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. -+ pud = __pud(val | PMD_SECT_NG); -+ } -+#endif - WRITE_ONCE(*pudp, pud); -- - if (pud_valid(pud)) { - dsb(ishst); - isb(); - } - } -+#endif - - static inline void pud_clear(pud_t *pudp) - { -@@ -738,6 +844,12 @@ static inline pmd_t *pud_pgtable(pud_t pud) - /* Find an entry in the second-level page table. */ - #define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t)) - -+#ifdef CONFIG_PTP -+#define pmd_set_fixmap_init(addr) ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr)) -+#define pmd_set_fixmap_offset_init(pud, addr) pmd_set_fixmap_init(pmd_offset_phys(pud, addr)) -+#define pmd_clear_fixmap_init() clear_fixmap_init(FIX_PMD) -+#endif ++static inline pte_t p4d_pte(p4d_t p4d) ++{ ++ return __pte(p4d_val(p4d)); ++} + - #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) - #define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) - #define pmd_clear_fixmap() clear_fixmap(FIX_PMD) -@@ -769,10 +881,15 @@ static inline pmd_t *pud_pgtable(pud_t pud) - #define p4d_none(p4d) (!p4d_val(p4d)) - #define p4d_bad(p4d) (!(p4d_val(p4d) & 2)) - #define p4d_present(p4d) (p4d_val(p4d)) -+#define p4d_valid(p4d) pte_valid(p4d_pte(p4d)) - -+#ifdef CONFIG_IEE -+extern void set_p4d(p4d_t *p4dp, p4d_t p4d); -+#else - static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) - { -- if (in_swapper_pgdir(p4dp)) { -+ if (in_swapper_pgdir(p4dp)) -+ { - set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); - return; - } -@@ -781,6 +898,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) - dsb(ishst); - isb(); - } -+#endif - - static inline void p4d_clear(p4d_t *p4dp) - { -@@ -800,6 +918,12 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) - /* Find an entry in the first-level page table. */ - #define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t)) - -+#ifdef CONFIG_PTP -+#define pud_set_fixmap_init(addr) ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr)) -+#define pud_set_fixmap_offset_init(p4d, addr) pud_set_fixmap_init(pud_offset_phys(p4d, addr)) -+#define pud_clear_fixmap_init() clear_fixmap_init(FIX_PUD) -+#endif ++static inline pte_t pud_pte(pud_t pud) ++{ ++ return __pte(pud_val(pud)); ++} + - #define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) - #define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr)) - #define pud_clear_fixmap() clear_fixmap(FIX_PUD) -@@ -826,6 +950,10 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) - #define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) - -+#ifdef CONFIG_PTP -+#define pgd_set_fixmap_init(addr) ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr)) -+#define pgd_clear_fixmap_init() clear_fixmap_init(FIX_PGD) -+#endif - #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) - #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) - -@@ -910,10 +1038,19 @@ static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, - - pte = __ptep_get(ptep); - do { -+ #ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+ #endif - old_pte = pte; - pte = pte_mkold(pte); -+ #ifdef CONFIG_PTP -+ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, -+ pte_val(old_pte), pte_val(pte)); -+ #else - pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), - pte_val(old_pte), pte_val(pte)); -+ #endif - } while (pte_val(pte) != pte_val(old_pte)); - - return pte_young(pte); -@@ -954,8 +1091,12 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - static inline pte_t __ptep_get_and_clear(struct mm_struct *mm, - unsigned long address, pte_t *ptep) - { -+ #ifdef CONFIG_PTP -+ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0); -+ pte_t pte = __pte(pteval); -+ #else - pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0)); -- -+ #endif - page_table_check_pte_clear(mm, pte); - - return pte; -@@ -997,7 +1138,12 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, - static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp) - { -+ #ifdef CONFIG_PTP -+ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0); -+ pmd_t pmd = __pmd(pteval); -+ #else - pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0)); -+ #endif - - page_table_check_pmd_clear(mm, pmd); - -@@ -1012,10 +1158,19 @@ static inline void ___ptep_set_wrprotect(struct mm_struct *mm, - pte_t old_pte; - - do { -+ #ifdef CONFIG_KOI -+ if (pte_valid(pte)) { -+ pte = __pte(pte_val(pte) | PTE_NG); -+ } -+ #endif - old_pte = pte; - pte = pte_wrprotect(pte); -+ #ifdef CONFIG_PTP -+ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep,pte_val(old_pte), pte_val(pte)); -+ #else - pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), - pte_val(old_pte), pte_val(pte)); -+ #endif - } while (pte_val(pte) != pte_val(old_pte)); - } - -@@ -1091,7 +1246,17 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp, pmd_t pmd) - { - page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); -+ #ifdef CONFIG_KOI -+ pmdval_t val = pmd_val(pmd); -+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { -+ pmd = __pmd(val | PMD_SECT_NG); -+ } -+ #endif -+ #ifdef CONFIG_PTP -+ return __pmd((pmdval_t)iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pmdval_t)pmd_val(pmd))); -+ #else - return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd))); -+ #endif - } - #endif - -diff --git a/arch/arm64/include/asm/pgtable_slab.h b/arch/arm64/include/asm/pgtable_slab.h -new file mode 100644 -index 000000000000..0674582a1948 ---- /dev/null -+++ b/arch/arm64/include/asm/pgtable_slab.h -@@ -0,0 +1,8 @@ -+#ifndef _LINUX_PGTABLE_SLAB_H -+#define _LINUX_PGTABLE_SLAB_H ++static inline pud_t pte_pud(pte_t pte) ++{ ++ return __pud(pte_val(pte)); ++} + -+extern void __init iee_pgtable_init(void); -+extern void *get_iee_pgtable_page(gfp_t gfpflags); -+extern void free_iee_pgtable_page(void *obj); ++static inline pmd_t pud_pmd(pud_t pud) ++{ ++ return __pmd(pud_val(pud)); ++} + -+#endif -\ No newline at end of file -diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h -index d2e0306e65d3..8352e92d4536 100644 ---- a/arch/arm64/include/asm/pointer_auth.h -+++ b/arch/arm64/include/asm/pointer_auth.h -@@ -108,8 +108,13 @@ static __always_inline void ptrauth_enable(void) - { - if (!system_supports_address_auth()) - return; -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | -+ SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)); -+ #else - sysreg_clear_set(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | - SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)); -+ #endif - isb(); - } - -diff --git a/arch/arm64/include/asm/stack_slab.h b/arch/arm64/include/asm/stack_slab.h -new file mode 100644 -index 000000000000..0a478828421e ---- /dev/null -+++ b/arch/arm64/include/asm/stack_slab.h -@@ -0,0 +1,8 @@ -+#ifndef _LINUX_STACK_SLAB_H -+#define _LINUX_STACK_SLAB_H ++static inline pte_t pmd_pte(pmd_t pmd) ++{ ++ return __pte(pmd_val(pmd)); ++} + -+extern void __init iee_stack_init(void); -+extern void *get_iee_stack(void); -+extern void free_iee_stack(void *obj); ++static inline pmd_t pte_pmd(pte_t pte) ++{ ++ return __pmd(pte_val(pte)); ++} + -+#endif -diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h -index 435634a703c6..c214643777a6 100644 ---- a/arch/arm64/include/asm/sysreg.h -+++ b/arch/arm64/include/asm/sysreg.h -@@ -1140,6 +1140,64 @@ - write_sysreg_s(__scs_new, sysreg); \ - } while (0) - ++static inline pgprot_t mk_pud_sect_prot(pgprot_t prot) ++{ ++ return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT); ++} + -+#ifdef CONFIG_IEE ++static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot) ++{ ++ return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT); ++} + -+#define SYS_TCR_IEE_SI TCR_HPD1 | TCR_A1 ++static inline pte_t pte_swp_mkexclusive(pte_t pte) ++{ ++ return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); ++} + -+extern void iee_rwx_gate_entry(int flag, ...); -+#define IEE_SI_TEST 0 -+#define IEE_WRITE_sctlr_el1 1 -+#define IEE_WRITE_ttbr0_el1 2 -+#define IEE_WRITE_vbar_el1 3 -+#define IEE_WRITE_tcr_el1 4 -+#define IEE_WRITE_mdscr_el1 5 -+#define IEE_WRITE_AFSR0 10 ++static inline int pte_swp_exclusive(pte_t pte) ++{ ++ return pte_val(pte) & PTE_SWP_EXCLUSIVE; ++} + -+#define sysreg_clear_set_iee_si(sysreg, clear, set) do { \ -+ u64 __scs_val = read_sysreg(sysreg); \ -+ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \ -+ if (__scs_new != __scs_val) \ -+ iee_rwx_gate_entry(IEE_WRITE_##sysreg, __scs_new); \ -+} while (0) ++static inline pte_t pte_swp_clear_exclusive(pte_t pte) ++{ ++ return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); ++} + -+#define IEE_SI_WRITE_DAIF_SEL "msr daifclr, #0xf\n\t" \ -+ "tbnz %x0, #6, 114221f\n\t" \ -+ "tbnz %x0, #7, 114210f\n\t" \ -+ "tbnz %x0, #8, 114100f\n\t" \ -+ "msr daifset, #0b000\n\t" \ -+ "b 114514f\n\t" \ -+"114221:\n\t" \ -+ "tbnz %x0, #7, 114211f\n\t" \ -+ "tbnz %x0, #8, 114101f\n\t" \ -+ "msr daifset, #0b001\n\t" \ -+ "b 114514f\n\t" \ -+"114211:\n\t" \ -+ "tbnz %x0, #8, 114111f\n\t" \ -+ "msr daifset, #0b011\n\t" \ -+ "b 114514f\n\t" \ -+"114210:\n\t" \ -+ "tbnz %x0, #8, 114110f\n\t" \ -+ "msr daifset, #0b010\n\t" \ -+ "b 114514f\n\t" \ -+"114100:\n\t" \ -+ "msr daifset, #0b100\n\t" \ -+ "b 114514f\n\t" \ -+"114101:\n\t" \ -+ "msr daifset, #0b101\n\t" \ -+ "b 114514f\n\t" \ -+"114110:\n\t" \ -+ "msr daifset, #0b110\n\t" \ -+ "b 114514f\n\t" \ -+"114111:\n\t" \ -+ "msr daifset, #0b111\n\t" \ -+"114514:\n\t" -+ -+#define iee_si_write_daif(v) do { \ -+ u64 __val = (u64)(v); \ -+ asm volatile(IEE_SI_WRITE_DAIF_SEL: : "rZ" (__val));} while (0) -+#endif -+ - #define read_sysreg_par() ({ \ - u64 par; \ - asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); \ -diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h -index 2c29239d05c3..a68ca9e784cf 100644 ---- a/arch/arm64/include/asm/tlb.h -+++ b/arch/arm64/include/asm/tlb.h -@@ -11,11 +11,31 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#include "pgtable_slab.h" ++#ifdef CONFIG_NUMA_BALANCING ++/* ++ * See the comment in include/linux/pgtable.h ++ */ ++static inline int pte_protnone(pte_t pte) ++{ ++ return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE; ++} ++ ++static inline int pmd_protnone(pmd_t pmd) ++{ ++ return pte_protnone(pmd_pte(pmd)); ++} +#endif + - static inline void __tlb_remove_table(void *_table) - { - free_page_and_swap_cache((struct page *)_table); - } - -+#ifdef CONFIG_PTP -+static inline void __iee_tlb_remove_table(void *_table) ++#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID)) ++ ++static inline int pmd_present(pmd_t pmd) +{ -+ struct page *page = (struct page *)_table; ++ return pte_present(pmd_pte(pmd)) || pmd_present_invalid(pmd); ++} + -+ // if (!is_huge_zero_page(page)) -+ // { -+ // if (page_ref_dec_return(page) == 1) -+ // { -+ free_iee_pgtable_page((void *)page_to_virt(page)); -+ // } -+ // } ++/* ++ * THP definitions. ++ */ ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++static inline int pmd_trans_huge(pmd_t pmd) ++{ ++ return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); +} -+#endif ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + - #define tlb_flush tlb_flush - static void tlb_flush(struct mmu_gather *tlb); - -diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h -index 4bbd9ed591f2..ecd8e35ab777 100644 ---- a/arch/arm64/include/asm/tlbflush.h -+++ b/arch/arm64/include/asm/tlbflush.h -@@ -49,6 +49,7 @@ - - #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) - -+ - #define __tlbi_user(op, arg) do { \ - if (arm64_kernel_unmapped_at_el0()) \ - __tlbi(op, (arg) | USER_ASID_FLAG); \ -@@ -258,6 +259,10 @@ static inline void flush_tlb_mm(struct mm_struct *mm) - asid = __TLBI_VADDR(0, ASID(mm)); - __tlbi(aside1is, asid); - __tlbi_user(aside1is, asid); -+ #if defined(CONFIG_IEE) || defined (CONFIG_KOI) -+ if (!arm64_kernel_unmapped_at_el0()) -+ __tlbi(aside1is, asid | USER_ASID_FLAG); -+ #endif - dsb(ish); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); - } -@@ -273,6 +278,10 @@ static inline void __flush_tlb_page_nosync(struct mm_struct *mm, - __tlbi_user(vale1is, addr); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ if (!arm64_kernel_unmapped_at_el0()) -+ __tlbi(vale1is, addr | USER_ASID_FLAG); -+ #endif - } - - static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, -@@ -366,6 +375,7 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) - * 2. If there is 1 page remaining, flush it through non-range operations. Range - * operations can only span an even number of pages. - */ -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) - #define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, tlbi_user) \ - do { \ -@@ -378,6 +388,8 @@ do { \ - pages == 1) { \ - addr = __TLBI_VADDR(start, asid); \ - __tlbi_level(op, addr, tlb_level); \ -+ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \ -+ __tlbi_level(op, addr | USER_ASID_FLAG, tlb_level); \ - if (tlbi_user) \ - __tlbi_user_level(op, addr, tlb_level); \ - start += stride; \ -@@ -390,6 +402,8 @@ do { \ - addr = __TLBI_VADDR_RANGE(start, asid, scale, \ - num, tlb_level); \ - __tlbi(r##op, addr); \ -+ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \ -+ __tlbi(r##op, addr | USER_ASID_FLAG); \ - if (tlbi_user) \ - __tlbi_user(r##op, addr); \ - start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ -@@ -399,6 +413,42 @@ do { \ - } \ - } while (0) - -+#else -+#define __flush_tlb_range_op(op, start, pages, stride, \ -+ asid, tlb_level, tlbi_user) \ -+do { \ -+ int num = 0; \ -+ int scale = 0; \ -+ unsigned long addr; \ -+ \ -+ while (pages > 0) { \ -+ if (!system_supports_tlb_range() || \ -+ pages % 2 == 1) { \ -+ addr = __TLBI_VADDR(start, asid); \ -+ __tlbi_level(op, addr, tlb_level); \ -+ if (tlbi_user) \ -+ __tlbi_user_level(op, addr, tlb_level); \ -+ start += stride; \ -+ pages -= stride >> PAGE_SHIFT; \ -+ continue; \ -+ } \ -+ \ -+ num = __TLBI_RANGE_NUM(pages, scale); \ -+ if (num >= 0) { \ -+ addr = __TLBI_VADDR_RANGE(start, asid, scale, \ -+ num, tlb_level); \ -+ __tlbi(r##op, addr); \ -+ if (tlbi_user) \ -+ __tlbi_user(r##op, addr); \ -+ start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ -+ pages -= __TLBI_RANGE_PAGES(num, scale); \ -+ } \ -+ scale++; \ -+ } \ -+} while (0) ++#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) ++#define pmd_young(pmd) pte_young(pmd_pte(pmd)) ++#define pmd_valid(pmd) pte_valid(pmd_pte(pmd)) ++#define pmd_user(pmd) pte_user(pmd_pte(pmd)) ++#define pmd_user_exec(pmd) pte_user_exec(pmd_pte(pmd)) ++#define pmd_cont(pmd) pte_cont(pmd_pte(pmd)) ++#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) ++#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) ++#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))) ++#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) ++#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) ++#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) + -+#endif //if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++static inline pmd_t pmd_mkinvalid(pmd_t pmd) ++{ ++ pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID)); ++ pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID)); + - #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false) - -@@ -493,9 +543,9 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end - */ - static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) - { -- unsigned long addr = __TLBI_VADDR(kaddr, 0); -- -- dsb(ishst); -+ unsigned long addr = __TLBI_VADDR(kaddr, 0); -+ -+ dsb(ishst); - __tlbi(vaae1is, addr); - dsb(ish); - isb(); -diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile -index 4ce58887302a..579b1f713849 100644 ---- a/arch/arm64/kernel/Makefile -+++ b/arch/arm64/kernel/Makefile -@@ -36,6 +36,8 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ - syscall.o proton-pack.o idreg-override.o idle.o \ - patching.o - -+obj-y += iee/ -+obj-$(CONFIG_KOI) += koi/ - obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \ - sys_compat.o - obj-$(CONFIG_AARCH32_EL0) += sigreturn32.o -@@ -84,6 +86,7 @@ obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o - obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o - obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o - CFLAGS_patch-scs.o += -mbranch-protection=none -+obj-$(CONFIG_HIVE) += sfi_bpf_arch.o - - # Force dependency (vdso*-wrap.S includes vdso.so through incbin) - $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so -diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c -index 87ac0b9c0b4f..f135db1d9965 100644 ---- a/arch/arm64/kernel/armv8_deprecated.c -+++ b/arch/arm64/kernel/armv8_deprecated.c -@@ -306,11 +306,19 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) - - static int cp15_barrier_set_hw_mode(bool enable) - { -+#ifdef CONFIG_IEE -+ if (enable) -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN); -+ else -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0); -+ return 0; -+#else - if (enable) - sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN); - else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0); - return 0; ++ return pmd; ++} ++ ++#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) ++ ++#define pmd_write(pmd) pte_write(pmd_pte(pmd)) ++ ++#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd)) +#endif - } - - static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn) -@@ -341,11 +349,19 @@ static int setend_set_hw_mode(bool enable) - if (!cpu_supports_mixed_endian_el0()) - return -EINVAL; - -+#ifdef CONFIG_IEE -+ if (enable) -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN); -+ else -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0); -+ return 0; ++static inline pmd_t pmd_mkdevmap(pmd_t pmd) ++{ ++ return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP))); ++} ++ ++#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd)) ++#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys) ++#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT) ++#define pfn_pmd(pfn, prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ ++ | pgprot_val(prot)) ++#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) ++ ++#define pud_young(pud) pte_young(pud_pte(pud)) ++#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) ++#define pud_write(pud) pte_write(pud_pte(pud)) ++ ++#define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT)) ++ ++#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud)) ++#define __phys_to_pud_val(phys) __phys_to_pte_val(phys) ++#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT) ++#define pfn_pud(pfn, prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT)\ ++ | pgprot_val(prot)) ++ ++static inline void __set_pte_at(struct mm_struct *mm, ++ unsigned long __always_unused addr, ++ pte_t *ptep, pte_t pte, unsigned int nr) ++{ ++ __sync_cache_and_tags(pte, nr); ++ __check_safe_pte_update(mm, ptep, pte); ++ __set_pte(ptep, pte); ++} ++ ++static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, ++ pmd_t *pmdp, pmd_t pmd) ++{ ++ page_table_check_pmd_set(mm, pmdp, pmd); ++ return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd), ++ PMD_SIZE >> PAGE_SHIFT); ++} ++ ++static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, ++ pud_t *pudp, pud_t pud) ++{ ++ page_table_check_pud_set(mm, pudp, pud); ++ return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud), ++ PUD_SIZE >> PAGE_SHIFT); ++} ++ ++#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d)) ++#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys) ++ ++#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd)) ++#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys) ++ ++#define __pgprot_modify(prot, mask, bits) \ ++ __pgprot((pgprot_val(prot) & ~(mask)) | (bits)) ++ ++#define pgprot_nx(prot) \ ++ __pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN) ++ ++/* ++ * Mark the prot value as uncacheable and unbufferable. ++ */ ++#define pgprot_noncached(prot) \ ++ __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN) ++#define pgprot_writecombine(prot) \ ++ __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) ++#define pgprot_device(prot) \ ++ __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN) ++#define pgprot_tagged(prot) \ ++ __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED)) ++#define pgprot_mhp pgprot_tagged ++/* ++ * DMA allocations for non-coherent devices use what the Arm architecture calls ++ * "Normal non-cacheable" memory, which permits speculation, unaligned accesses ++ * and merging of writes. This is different from "Device-nGnR[nE]" memory which ++ * is intended for MMIO and thus forbids speculation, preserves access size, ++ * requires strict alignment and can also force write responses to come from the ++ * endpoint. ++ */ ++#define pgprot_dmacoherent(prot) \ ++ __pgprot_modify(prot, PTE_ATTRINDX_MASK, \ ++ PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) ++ ++#define __HAVE_PHYS_MEM_ACCESS_PROT ++struct file; ++extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, ++ unsigned long size, pgprot_t vma_prot); ++ ++#define pmd_none(pmd) (!pmd_val(pmd)) ++ ++#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ ++ PMD_TYPE_TABLE) ++#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ ++ PMD_TYPE_SECT) ++#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd)) ++#define pmd_bad(pmd) (!pmd_table(pmd)) ++ ++#define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE) ++#define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) ++ ++#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3 ++static inline bool pud_sect(pud_t pud) { return false; } ++static inline bool pud_table(pud_t pud) { return true; } +#else - if (enable) - sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0); - else - sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED); - return 0; -+#endif - } - - static int __a32_setend_handler(struct pt_regs *regs, u32 big_endian) -diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c -index f20918eb36bc..68c08adc86c9 100644 ---- a/arch/arm64/kernel/asm-offsets.c -+++ b/arch/arm64/kernel/asm-offsets.c -@@ -102,6 +102,19 @@ int main(void) - DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs)); - BLANK(); - #endif -+#ifdef CONFIG_IEE -+ DEFINE(iee_from_token_offset, offsetof(struct task_token, iee_stack)); -+ DEFINE(tmp_page_from_token_offset, offsetof(struct task_token, tmp_page)); -+ DEFINE(kernel_from_token_offset, offsetof(struct task_token, kernel_stack)); -+ DEFINE(mm_from_task_offset, offsetof(struct task_struct, mm)); -+#endif -+#ifdef CONFIG_KOI -+ DEFINE(koi_kernel_from_token_offset, offsetof(struct task_token, koi_kernel_stack)); -+ DEFINE(koi_from_token_offset, offsetof(struct task_token, koi_stack)); -+ DEFINE(ttbr1_from_token_offset, offsetof(struct task_token, current_ttbr1)); -+ DEFINE(koi_stack_base_from_token_offset, offsetof(struct task_token, koi_stack_base)); -+#endif -+ BLANK(); - #ifdef CONFIG_AARCH32_EL0 - DEFINE(COMPAT_SIGFRAME_REGS_OFFSET, offsetof(struct a32_sigframe, uc.uc_mcontext.arm_r0)); - DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET, offsetof(struct a32_rt_sigframe, sig.uc.uc_mcontext.arm_r0)); -diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c -index 2e5e4052a182..d27d11d7b7bb 100644 ---- a/arch/arm64/kernel/cpu_errata.c -+++ b/arch/arm64/kernel/cpu_errata.c -@@ -80,7 +80,11 @@ hisilicon_1980005_enable(const struct arm64_cpu_capabilities *__unused) - __set_bit(ARM64_HAS_CACHE_IDC, system_cpucaps); - arm64_ftr_reg_ctrel0.sys_val |= BIT(CTR_EL0_IDC_SHIFT); - arm64_ftr_reg_ctrel0.strict_mask &= ~BIT(CTR_EL0_IDC_SHIFT); -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); -+#else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); -+#endif - } - #endif - -@@ -132,7 +136,11 @@ cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) - enable_uct_trap = true; - - if (enable_uct_trap) -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); -+#else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); -+#endif - } - - #ifdef CONFIG_ARM64_ERRATUM_1463225 -@@ -147,7 +155,11 @@ has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry, - static void __maybe_unused - cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) - { -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCI, 0); -+#else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); -+#endif - } - - #ifdef CONFIG_HISILICON_ERRATUM_HIP08_RU_PREFETCH -diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c -index a1736e9044da..81aaca96478c 100644 ---- a/arch/arm64/kernel/cpufeature.c -+++ b/arch/arm64/kernel/cpufeature.c -@@ -94,6 +94,11 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#include ++#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ ++ PUD_TYPE_SECT) ++#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ ++ PUD_TYPE_TABLE) +#endif + - /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ - static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly; - -@@ -1616,7 +1621,11 @@ static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unu - * value. - */ - if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT))) -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); -+#else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); -+#endif - } - - static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, -@@ -1877,7 +1886,11 @@ static inline void __cpu_enable_hw_dbm(void) - { - u64 tcr = read_sysreg(tcr_el1) | TCR_HD; - -+#ifdef CONFIG_IEE -+ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr); -+#else - write_sysreg(tcr, tcr_el1); -+#endif - isb(); - local_flush_tlb_all(); - } -@@ -2060,7 +2073,9 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) - */ - WARN_ON_ONCE(in_interrupt()); - -+ #ifndef CONFIG_IEE - sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); -+ #endif - set_pstate_pan(1); - } - #endif /* CONFIG_ARM64_PAN */ -@@ -2125,7 +2140,11 @@ static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, - static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap) - { - if (this_cpu_has_cap(ARM64_HAS_E0PD)) -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(tcr_el1, 0, TCR_E0PD1); -+#else - sysreg_clear_set(tcr_el1, 0, TCR_E0PD1); -+#endif - } - #endif /* CONFIG_ARM64_E0PD */ - -@@ -2220,7 +2239,11 @@ static void nmi_enable(const struct arm64_cpu_capabilities *__unused) - * avoid leaving things masked. - */ - _allint_clear(); -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); -+ #else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); -+ #endif - isb(); - } - #endif -@@ -2235,7 +2258,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) - * So, be strict and forbid other BRs using other registers to - * jump onto a PACIxSP instruction: - */ -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); -+#else - sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); -+#endif - isb(); - } - #endif /* CONFIG_ARM64_BTI */ -@@ -2243,7 +2270,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) - #ifdef CONFIG_ARM64_MTE - static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) - { -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); -+ #else - sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); -+ #endif - - mte_cpu_setup(); - -@@ -2288,7 +2319,11 @@ static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, in - - static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) - { -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_TIDCP); -+ #else - sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP); -+ #endif - } - - static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) -@@ -2298,7 +2333,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) - - static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused) - { -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_MSCEn); -+ #else - sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn); -+ #endif - } - - /* Internal helper functions to match cpu capability type */ -@@ -3593,6 +3632,43 @@ static void __init setup_system_capabilities(void) - enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU); - } - -+#ifdef CONFIG_IEE ++extern pgd_t init_pg_dir[PTRS_PER_PGD]; ++extern pgd_t init_pg_end[]; ++extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; ++extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; ++extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; ++extern pgd_t reserved_pg_dir[PTRS_PER_PGD]; + -+static void iee_si_test_end(void) -+{ -+ pr_info("IEE: testing iee_exec_entry sctlr...\n"); -+ iee_rwx_gate_entry(IEE_WRITE_SCTLR, read_sysreg(sctlr_el1)& ~SCTLR_ELx_M); -+ pr_info("IEE: testing iee_exec_entry ttbr0_el1...\n"); -+ iee_rwx_gate_entry(IEE_WRITE_TTBR0, read_sysreg(ttbr0_el1)); -+ pr_info("IEE: testing iee_exec_entry vbar...\n"); -+ iee_rwx_gate_entry(IEE_WRITE_VBAR, read_sysreg(vbar_el1)); -+ pr_info("IEE: testing iee_exec_entry tcr...\n"); -+ iee_rwx_gate_entry(IEE_WRITE_TCR, read_sysreg(tcr_el1)); -+ // pr_info("IEE: testing iee_exec_entry mdscr...\n"); -+ // iee_rwx_gate_entry(IEE_WRITE_MDSCR, read_sysreg(mdscr_el1)); -+ // pr_info("IEE: testing iee_exec_entry afsr0...\n"); -+ // iee_rwx_gate_entry(IEE_WRITE_AFSR0); -+ #ifdef CONFIG_KOI -+ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1)); -+ pr_info("IEE: testing iee_exec_entry switch to koi...\n"); -+ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, phys_to_ttbr(__pa_symbol(swapper_pg_dir)) | 3UL << 48); -+ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1)); -+ pr_info("IEE: testing iee_exec_entry switch to kernel...\n"); -+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); -+ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1)); -+ #endif ++extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); ++ ++static inline bool in_swapper_pgdir(void *addr) ++{ ++ return ((unsigned long)addr & PAGE_MASK) == ++ ((unsigned long)swapper_pg_dir & PAGE_MASK); +} + -+/* Finish iee rwx gate initializations. */ -+static void __init iee_si_init_done(void) ++static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ -+ // Prepare data for iee rwx gate -+ iee_si_prepare_data(); -+ // All initialization is done. Do some simple tests. -+ iee_si_test_end(); ++#ifdef __PAGETABLE_PMD_FOLDED ++ if (in_swapper_pgdir(pmdp)) { ++ set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); ++ return; ++ } ++#endif /* __PAGETABLE_PMD_FOLDED */ ++ ++ iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); ++ ++ if (pmd_valid(pmd)) { ++ dsb(ishst); ++ isb(); ++ } +} -+#endif + - void __init setup_cpu_features(void) - { - u32 cwg; -@@ -3621,6 +3697,10 @@ void __init setup_cpu_features(void) - if (!cwg) - pr_warn("No Cache Writeback Granule information, assuming %d\n", - ARCH_DMA_MINALIGN); -+ -+ #ifdef CONFIG_IEE -+ iee_si_init_done(); -+ #endif - } - - static int enable_mismatched_32bit_el0(unsigned int cpu) -diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c -index 745aefddd9a3..265417e0ad81 100644 ---- a/arch/arm64/kernel/debug-monitors.c -+++ b/arch/arm64/kernel/debug-monitors.c -@@ -36,10 +36,14 @@ u8 debug_monitors_arch(void) - */ - static void mdscr_write(u32 mdscr) - { -+// #ifdef CONFIG_IEE -+// iee_rwx_gate_entry(IEE_WRITE_mdscr_el1, mdscr); -+// #else - unsigned long flags; - flags = local_daif_save(); - write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); -+// #endif - } - NOKPROBE_SYMBOL(mdscr_write); - -diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c -index 4602c107c40a..73aa0aad07b1 100644 ---- a/arch/arm64/kernel/entry-common.c -+++ b/arch/arm64/kernel/entry-common.c -@@ -212,7 +212,11 @@ static __always_inline void fast_enter_from_user_mode(struct pt_regs *regs) - * mode. Before this function is called it is not safe to call regular kernel - * code, instrumentable code, or any code which may trigger an exception. - */ -+#ifdef CONFIG_IEE -+void noinstr arm64_enter_nmi(struct pt_regs *regs) -+#else - static void noinstr arm64_enter_nmi(struct pt_regs *regs) -+#endif - { - regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); - -diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S -index da3809632f0f..40c279f562f0 100644 ---- a/arch/arm64/kernel/entry.S -+++ b/arch/arm64/kernel/entry.S -@@ -29,12 +29,399 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#define BAD_SP_EL0 0 -+#define BAD_ELR_EL1 1 -+#define BAD_TCR_EL1 2 -+#define BAD_IEE_SI 4 -+#endif ++static inline void pmd_clear(pmd_t *pmdp) ++{ ++ set_pmd(pmdp, __pmd(0)); ++} + - .macro clear_gp_regs - .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29 - mov x\n, xzr - .endr - .endm - ++static inline phys_addr_t pmd_page_paddr(pmd_t pmd) ++{ ++ return __pmd_to_phys(pmd); ++} + -+#ifdef CONFIG_KOI -+#ifdef CONFIG_IEE -+/* -+ * This function is used to switch to ko stack in glue code -+ */ -+SYM_FUNC_START(koi_do_switch_to_ko_stack) -+ sub sp, sp, #48 -+ stp x29, x30, [sp] -+ str x2, [sp, #16] -+ stp x0, x1, [sp, #32] -+ -+ // iee_rw_gate(IEE_WRITE_KERNEL_STACK, current, sp) -+ mov x0, #IEE_WRITE_KOI_KERNEL_STACK -+ mrs x1, sp_el0 -+ add x2, sp, #48 -+ -+ bl iee_rw_gate -+ -+ // iee_rw_gate(IEE_READ_KOI_STACK, current) -+ mov x0, #IEE_READ_KOI_STACK -+ mrs x1, sp_el0 -+ bl iee_rw_gate -+ -+ ldp x29, x30, [sp] -+ ldr x2, [sp, #16] -+ add x1, sp, #32 -+ mov sp, x0 -+ ldp x0, x1, [x1] ++static inline unsigned long pmd_page_vaddr(pmd_t pmd) ++{ ++ return (unsigned long)__va(pmd_page_paddr(pmd)); ++} + -+ isb -+ ret -+SYM_FUNC_END(koi_do_switch_to_ko_stack) ++/* Find an entry in the third-level page table. */ ++#define pte_offset_phys(dir, addr) (pmd_page_paddr(READ_ONCE(*(dir))) \ ++ + pte_index(addr) * sizeof(pte_t)) + -+/* -+ * This fucntion is used to switch to kernel stack in glue code -+ */ -+SYM_FUNC_START(koi_do_switch_to_kernel_stack) -+ sub sp, sp, #48 -+ stp x29, x30, [sp] -+ str x2, [sp, #16] -+ stp x0, x1, [sp, #32] -+ // iee_rw_gate(IEE_WRITE_KOI_STACK, current, sp) -+ mov x0, #IEE_WRITE_KOI_STACK -+ mrs x1, sp_el0 -+ add x2, sp, #48 -+ bl iee_rw_gate -+ -+ // iee_rw_gate(IEE_READ_KOI_KERNEL_STACK, current) -+ mov x0, #IEE_READ_KOI_KERNEL_STACK -+ mrs x1, sp_el0 -+ bl iee_rw_gate -+ -+ ldp x29, x30, [sp] -+ ldr x2, [sp, #16] -+ add x1, sp, #32 -+ mov sp, x0 -+ ldp x0, x1, [x1] -+ isb -+ ret -+SYM_FUNC_END(koi_do_switch_to_kernel_stack) ++#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) ++#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) ++#define pte_clear_fixmap() clear_fixmap(FIX_PTE) + -+/* -+ * Before switch to ko's pgtable, we must switch current stack to ko's stack. -+ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching, -+ * so we need to copy from kernel stack to ko stack -+ * the memory region to copy is [sp, stack_top) -+ * void koi_switch_to_ko_stack(void); -+ */ -+SYM_FUNC_START(koi_switch_to_ko_stack) -+ mrs x17, pan -+ msr pan, 0x0 -+ -+ sub sp, sp, #32 -+ str x17, [sp, #16] -+ stp x30, x29, [sp] -+ -+ // current sp stores in x2 -+ add x2, x1, #176 -+ // current sp_el0 stores in x0 -+ mov x1, x0 -+ adrp x0, iee_offset -+ ldr x0, [x0, #:lo12:iee_offset] -+ bl _iee_write_koi_kernel_stack -+ -+ adrp x0, iee_offset -+ ldr x0, [x0, #:lo12:iee_offset] -+ mrs x1, sp_el0 -+ bl _iee_read_koi_stack -+ -+ ldr x17, [sp, #16] -+ ldp x30, x29, [sp] -+ add sp, sp, #32 -+ -+ msr pan, x17 -+ -+ sub x0, x0, #176 -+ mov x1, sp -+ mov x2, #176 -+ -+ // memcpy(current->driver_stack, current->kernel_stack, 176) -+ mov x16, lr -+ bl memcpy -+ mov lr, x16 -+ -+ mov sp, x0 -+ isb -+ ret -+SYM_FUNC_END(koi_switch_to_ko_stack) -+ -+SYM_FUNC_START(koi_switch_to_kernel_stack) -+ /* -+ * current sp belongs to driver stack, and the bottom 160 bytes saves registers when exception occurred, -+ * so we should add 160 to current sp, and store it in task_struct -+ * also, fetch kernel sp from task_struct, copy the bottom 160 bytes from driver stack to kernel stack -+ */ -+ mrs x17, pan -+ msr pan, 0x0 -+ -+ sub sp, sp, #32 -+ stp x30, x29, [sp] -+ str x17, [sp, #16] -+ -+ adrp x0, iee_offset -+ ldr x0, [x0, #:lo12:iee_offset] -+ mrs x1, sp_el0 -+ add x2, sp, #192 -+ bl _iee_write_koi_stack -+ -+ adrp x0, iee_offset -+ ldr x0, [x0, #:lo12:iee_offset] -+ mrs x1, sp_el0 -+ bl _iee_read_koi_kernel_stack -+ -+ ldr x17, [sp, #16] -+ ldp x30, x29, [sp] -+ add sp, sp, #32 -+ -+ msr pan, x17 -+ -+ // x0 = kernel_stack -+ sub x0, x0, #160 -+ mov x1, sp -+ // x2 = 160 -+ mov x2, #160 -+ -+ mov x16, lr -+ bl memcpy -+ mov lr, x16 -+ -+ mov sp, x0 -+ isb -+ ret -+SYM_FUNC_END(koi_switch_to_kernel_stack) -+#else -+/* -+ * This function is used to switch to ko stack in glue code -+ */ -+SYM_FUNC_START(koi_do_switch_to_ko_stack) -+ sub sp, sp, #16 -+ stp x16, x17, [sp] -+ mrs x17, sp_el0 -+ adrp x16, koi_offset -+ ldr x16, [x16, #:lo12:koi_offset] -+ add x17, x17, x16 -+ add x16, sp, #16 -+ str x16, [x17, #koi_kernel_from_token_offset] -+ ldr x16, [x17, #koi_from_token_offset] -+ mov x17, sp -+ mov sp, x16 -+ ldp x16, x17, [x17] -+ isb -+ ret -+SYM_FUNC_END(koi_do_switch_to_ko_stack) ++#define pmd_page(pmd) phys_to_page(__pmd_to_phys(pmd)) + -+/* -+ * This fucntion is used to switch to kernel stack in glue code -+ */ -+SYM_FUNC_START(koi_do_switch_to_kernel_stack) -+ sub sp, sp, #16 -+ stp x16, x17, [sp] -+ mrs x17, sp_el0 -+ adrp x16, koi_offset -+ ldr x16, [x16, #:lo12:koi_offset] -+ add x17, x17, x16 -+ add x16, sp, #16 -+ str x16, [x17, #koi_from_token_offset] -+ ldr x16, [x17, #koi_kernel_from_token_offset] -+ mov x17, sp -+ mov sp, x16 -+ ldp x16, x17, [x17] -+ isb -+ ret -+SYM_FUNC_END(koi_do_switch_to_kernel_stack) ++/* use ONLY for statically allocated translation tables */ ++#define pte_offset_kimg(dir, addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr)))) + +/* -+ * Before switch to ko's pgtable, we must switch current stack to ko's stack. -+ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching, -+ * so we need to copy from kernel stack to ko stack -+ * the memory region to copy is [sp, stack_top) -+ * void koi_switch_to_ko_stack(unsigned long stack_top); ++ * Conversion functions: convert a page and protection to a page entry, ++ * and a page entry and page directory to the page they refer to. + */ -+SYM_FUNC_START(koi_switch_to_ko_stack) -+ // current sp stores in x1 -+ add x3, x1, #176 -+ adrp x4, koi_offset -+ ldr x4, [x4, #:lo12:koi_offset] -+ add x4, x0, x4 -+ // current sp_el0 stores in x0 -+ str x3, [x4, #koi_kernel_from_token_offset] -+ ldr x0, [x4, #koi_from_token_offset] -+ sub x0, x0, #176 -+ mov x2, #176 -+ -+ // memcpy(current->driver_stack, current->kernel_stack, 176) -+ mov x16, lr -+ bl memcpy -+ mov lr, x16 -+ -+ mov sp, x0 -+ isb -+ ret -+SYM_FUNC_END(koi_switch_to_ko_stack) -+ -+SYM_FUNC_START(koi_switch_to_kernel_stack) -+ /* -+ * current sp belongs to driver stack, and the bottom 176 bytes saves registers when exception occurred, -+ * so we should add 176 to current sp, and store it in task_struct -+ * also, fetch kernel sp from task_struct, copy the bottom 176 bytes from driver stack to kernel stack -+ */ -+ mov x1, sp -+ add x3, sp, #160 -+ -+ mrs x16, sp_el0 -+ adrp x2, koi_offset -+ ldr x2, [x2, #:lo12:koi_offset] -+ add x16, x16, x2 -+ str x3, [x16, #koi_from_token_offset] -+ // sp points to kernel_stack -+ ldr x0, [x16, #koi_kernel_from_token_offset] -+ -+ // x0 = kernel_stack -+ sub x0, x0, #160 -+ // x2 = 160 -+ mov x2, #160 -+ mov x16, lr -+ // memcpy(kernel_stack, driver_stack, 160) -+ bl memcpy -+ mov lr, x16 -+ mov sp, x0 -+ isb -+ ret -+SYM_FUNC_END(koi_switch_to_kernel_stack) -+#endif -+.pushsection ".koi.text", "ax" -+SYM_FUNC_START(koi_switch_to_ko_pgtbl) -+ stp x0, x1, [sp, #16 * 1] -+ stp x2, x3, [sp, #16 * 2] -+ stp x4, x5, [sp, #16 * 3] -+ stp x6, x7, [sp, #16 * 4] -+ stp x8, x9, [sp, #16 * 5] -+ stp x10, x11, [sp, #16 * 6] -+ stp x12, x13, [sp, #16 * 7] -+ stp x14, x15, [sp, #16 * 8] -+ stp x16, x17, [sp, #16 * 9] -+ stp x18, x30, [sp, #16 * 10] -+ -+ adrp x0, koi_swapper_ttbr1 -+ ldr x0, [x0, #:lo12:koi_swapper_ttbr1] -+ cbz x0, 0f -+ bl koi_do_switch_to_ko_pgtbl -+ // if x0 == 0, don't need to switch pgtable and stack, jump to 0 -+ cbz x0, 0f -+ mov x19, x0 -+ // if current on task's kernel stack， switch to ko stack -+ mrs x0, sp_el0 -+ mov x1, sp -+ ldr x2, [x0, TSK_STACK] -+ eor x2, x2, x1 -+ and x2, x2, #~(THREAD_SIZE - 1) -+ cbnz x2, 1f ++#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) + -+1: -+#ifndef CONFIG_IEE -+ msr ttbr1_el1, x19 -+ isb -+ nop -+ nop -+ nop -+#else -+ mov x0, #IEE_SWITCH_TO_KOI -+ mov x1, x19 -+ bl iee_rwx_gate_entry -+#endif -+0: -+ -+ ldp x0, x1, [sp, #16 * 1] -+ ldp x2, x3, [sp, #16 * 2] -+ ldp x4, x5, [sp, #16 * 3] -+ ldp x6, x7, [sp, #16 * 4] -+ ldp x8, x9, [sp, #16 * 5] -+ ldp x10, x11, [sp, #16 * 6] -+ ldp x12, x13, [sp, #16 * 7] -+ ldp x14, x15, [sp, #16 * 8] -+ ldp x16, x17, [sp, #16 * 9] -+ ldp x18, x30, [sp, #16 * 10] -+ ret -+SYM_FUNC_END(koi_switch_to_ko_pgtbl) -+ -+SYM_FUNC_START(koi_switch_to_kernel_pgtbl) -+ sub sp, sp, #160 -+ stp x0, x1, [sp, #16 * 0] -+ stp x2, x3, [sp, #16 * 1] -+ stp x4, x5, [sp, #16 * 2] -+ stp x6, x7, [sp, #16 * 3] -+ stp x8, x9, [sp, #16 * 4] -+ stp x10, x11, [sp, #16 * 5] -+ -+ stp x12, x13, [sp, #16 * 6] -+ stp x14, x15, [sp, #16 * 7] -+ stp x16, x17, [sp, #16 * 8] -+ stp x18, x30, [sp, #16 * 9] -+ // check whether paging init finished -+ adrp x0, koi_swapper_ttbr1 -+ ldr x0, [x0, #:lo12:koi_swapper_ttbr1] -+ cbz x0, 0f -+ -+ bl koi_do_switch_to_kernel_pgtbl -+ /* -+ * koi_do_switch_to_kernel_pgtbl return 0 indicates -+ * that when exception occurred, the isolated ko is executing under koi pgtbl, -+ * so we need to switch stack to kernel stack after switch pgtbl back to koi_swapper_ttbr1. -+ */ -+ cbz x0, 0f -+#ifndef CONFIG_IEE -+ mrs x0, sp_el0 -+ adrp x1, koi_offset -+ ldr x1, [x1, #:lo12:koi_offset] -+ add x0, x0, x1 -+ mov x16, sp -+ ldr x17, [x0, #koi_stack_base_from_token_offset] -+ eor x17, x17, x16 -+ and x17, x17, #~(THREAD_SIZE - 1) -+ cbnz x17, 0f -+#else -+ // save current pan -+ mrs x17, pan -+ // disable pan -+ msr pan, 0x0 -+ adrp x0, iee_offset -+ ldr x0, [x0, #:lo12:iee_offset] -+ mrs x1, sp_el0 -+ bl _iee_read_koi_stack_base -+ // restore pan -+ msr pan, x17 -+ -+ mov x16, sp -+ eor x0, x0, x16 -+ and x0, x0, #~(THREAD_SIZE - 1) -+ cbnz x0, 0f -+#endif -+0: -+ -+ ldp x0, x1, [sp, #16 * 0] -+ ldp x2, x3, [sp, #16 * 1] -+ ldp x4, x5, [sp, #16 * 2] -+ ldp x6, x7, [sp, #16 * 3] -+ ldp x8, x9, [sp, #16 * 4] -+ ldp x10, x11, [sp, #16 * 5] -+ ldp x12, x13, [sp, #16 * 6] -+ ldp x14, x15, [sp, #16 * 7] -+ ldp x16, x17, [sp, #16 * 8] -+ ldp x18, x30, [sp, #16 * 9] -+ add sp, sp, #160 -+ ret -+SYM_FUNC_END(koi_switch_to_kernel_pgtbl) -+.popsection -+#endif ++#if CONFIG_PGTABLE_LEVELS > 2 + - .macro kernel_ventry, el:req, ht:req, regsize:req, label:req - .align 7 - .Lventry_start\@: -@@ -53,6 +440,15 @@ - .Lskip_tramp_vectors_cleanup\@: - .endif - -+#ifdef CONFIG_KOI -+ .if \el == 1 -+ msr tpidrro_el0, x30 -+ bl koi_switch_to_kernel_pgtbl -+ mrs x30, tpidrro_el0 -+ msr tpidrro_el0, xzr -+ .endif -+#endif ++#define pmd_ERROR(e) \ ++ pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) + - sub sp, sp, #PT_REGS_SIZE - #ifdef CONFIG_VMAP_STACK - /* -@@ -332,6 +728,17 @@ alternative_else_nop_endif - msr_s SYS_ICC_PMR_EL1, x20 - - .Lskip_pmr_save\@: -+#endif ++#define pud_none(pud) (!pud_val(pud)) ++#define pud_bad(pud) (!pud_table(pud)) ++#define pud_present(pud) pte_present(pud_pte(pud)) ++#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud)) ++#define pud_valid(pud) pte_valid(pud_pte(pud)) ++#define pud_user(pud) pte_user(pud_pte(pud)) ++#define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) + -+#ifndef CONFIG_IEE -+#ifdef CONFIG_KOI -+ // set tcr_el1 to choose asid from ttbr1_el1 or ttbr0_el1 -+ .if \el == 0 -+ mrs x0, tcr_el1 -+ orr x0, x0 ,#0x0000000000400000 -+ msr tcr_el1,x0 -+ .endif -+#endif - #endif - - /* -@@ -345,9 +752,11 @@ alternative_else_nop_endif - .endm - - .macro kernel_exit, el, fast_mode = std -+ #ifndef CONFIG_IEE - .if \el != 0 - disable_daif - .endif -+ #endif - - #ifdef CONFIG_ARM64_PSEUDO_NMI - alternative_if_not ARM64_HAS_GIC_PRIO_MASKING -@@ -435,6 +844,40 @@ alternative_else_nop_endif - - msr elr_el1, x21 // set up the return data - msr spsr_el1, x22 ++static inline void set_pud(pud_t *pudp, pud_t pud) ++{ ++#ifdef __PAGETABLE_PUD_FOLDED ++ if (in_swapper_pgdir(pudp)) { ++ set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); ++ return; ++ } ++#endif /* __PAGETABLE_PUD_FOLDED */ + -+#ifdef CONFIG_IEE ++ iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); ++ ++ if (pud_valid(pud)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++ ++static inline void pud_clear(pud_t *pudp) ++{ ++ set_pud(pudp, __pud(0)); ++} ++ ++static inline phys_addr_t pud_page_paddr(pud_t pud) ++{ ++ return __pud_to_phys(pud); ++} ++ ++static inline pmd_t *pud_pgtable(pud_t pud) ++{ ++ return (pmd_t *)__va(pud_page_paddr(pud)); ++} + -+ .if \el == 0 ++/* Find an entry in the second-level page table. */ ++#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) \ ++ + pmd_index(addr) * sizeof(pmd_t)) + -+ /* Skip TCR settings if User\Kernel isolation is already enforced by KPTI.*/ -+ alternative_insn nop, "b 6f", ARM64_UNMAP_KERNEL_AT_EL0 -+ // SET hpd1 = 0 start -+ mrs x0, tcr_el1 -+ and x0, x0, #0xFFFFFBFFFFFFFFFF -+ and x0, x0, #0xFFFFFFFFFFBFFFFF -+ msr tcr_el1, x0 -+ // SET hpd1 = 0 end ++#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) ++#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) ++#define pmd_clear_fixmap() clear_fixmap(FIX_PMD) + -+ // Check ELR_EL1 -+ mrs x0, elr_el1 -+ lsr x0, x0, #48 -+ tst x0, #0xffff -+ b.ne 5f -+6: ++#define pud_page(pud) phys_to_page(__pud_to_phys(pud)) + -+ .endif ++/* use ONLY for statically allocated translation tables */ ++#define pmd_offset_kimg(dir, addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr)))) + +#else -+#ifdef CONFIG_KOI -+ .if \el==0 -+ mrs x0, tcr_el1 -+ and x0, x0, #0xFFFFFFFFFFBFFFFF -+ msr tcr_el1,x0 -+ .endif -+#endif + -+#endif ++#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; }) ++#define pud_user_exec(pud) pud_user(pud) /* Always 0 with folding */ + - ldp x0, x1, [sp, #16 * 0] - ldp x2, x3, [sp, #16 * 1] - ldp x4, x5, [sp, #16 * 2] -@@ -480,6 +923,17 @@ alternative_if ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD - alternative_else_nop_endif - .endif - eret ++/* Match pmd_offset folding in */ ++#define pmd_set_fixmap(addr) NULL ++#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp) ++#define pmd_clear_fixmap() + -+#ifdef CONFIG_IEE -+5: -+ // ELR_EL1 check fail -+ mov x0, sp -+ mov x1, #BAD_ELR_EL1 -+ mrs x2, esr_el1 -+ bl iee_bad_mode -+ ASM_BUG() -+#endif ++#define pmd_offset_kimg(dir, addr) ((pmd_t *)dir) + - .else - ldr lr, [sp, #S_LR] - add sp, sp, #PT_REGS_SIZE // restore sp -@@ -488,7 +942,13 @@ alternative_else_nop_endif - .if \fast_mode == std - alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412 - .endif -- -+#ifdef CONFIG_KOI -+ sub sp, sp, #176 -+ stp x30, x19, [sp, #16 * 0] -+ bl koi_switch_to_ko_pgtbl -+ ldp x30, x19, [sp, #16 * 0] -+ add sp, sp, #176 -+#endif - eret - .endif - sb -@@ -600,6 +1060,151 @@ SYM_CODE_START_LOCAL(__bad_stack) - SYM_CODE_END(__bad_stack) - #endif /* CONFIG_VMAP_STACK */ - -+/* -+ * iee exception entry -+ */ -+ .macro iee_exception_entry, el -+ -+ /* Check whether exception is permmited. */ -+ ldr x1, =__iee_si_no_irq -+ cmp x1, x22 -+ b.hi 1148f -+ ldr x1, =__iee_si_text_end -+ cmp x1, x22 -+ b.lo 1148f -+ /* ELR check fail */ -+ mov x0, sp -+ mov x1, #BAD_IEE_SI -+ mrs x2, esr_el1 -+ bl iee_bad_mode -+ ASM_BUG() -+1148: -+ -+ /* el0 set hpds */ -+ .if \el == 0 -+ -+ /* Skip TCR settings if User\Kernel isolation is already enforced by KPTI.*/ -+ alternative_insn nop, "b 6f", ARM64_UNMAP_KERNEL_AT_EL0 -+ /* SET hpd1 = 1 start */ -+ mrs x0, tcr_el1 -+ orr x0, x0, #0x0000040000000000 -+ orr x0, x0, #0x0000000000400000 -+ msr tcr_el1, x0 -+ /* SET hpd1 = 1 end */ -+ -+ disable_daif -+ -+ /* Check TCR_EL1 */ -+ mrs x0, tcr_el1 -+ tst x0, #0x0000040000000000 -+ b.eq 5f -+ tst x0, #0x0000000000400000 -+ b.ne 6f -+ -+5: -+ /* TCR_EL1 check fail */ -+ mov x0, sp -+ mov x1, #BAD_TCR_EL1 -+ mrs x2, esr_el1 -+ bl iee_bad_mode -+ ASM_BUG() ++#endif /* CONFIG_PGTABLE_LEVELS > 2 */ + -+6: -+ nop ++#if CONFIG_PGTABLE_LEVELS > 3 + -+ .else -+#ifdef CONFIG_IEE_INTERRUPTABLE -+ /* el1 save elr_el1 and set pan */ -+ /* Check ELR_EL1 */ -+ ldr x1, =__iee_code_start -+ cmp x1, x22 -+ b.hi 7f -+ ldr x1, =__iee_code_end -+ cmp x1, x22 -+ b.lo 7f -+ /* Exception from iee code */ -+ /* Switch to kernel stack */ -+ mrs x0, sp_el0 /* x0 -> task_struct(VA) */ -+ adrp x2, iee_offset -+ ldr x2, [x2, #:lo12:iee_offset] -+ add x1, x0, x2 /* x1 -> task_token(IEE) */ -+ // store iee stack -+ mov x3, sp -+ str x3, [x1, #iee_from_token_offset] -+ // load kernel stack -+ ldr x3, [x1, #kernel_from_token_offset] -+ mov sp, x3 -+ sub sp, sp, #PT_REGS_SIZE -+ /* Enable PAN */ -+ msr pan, #0x1 -+ -+7: -+ /* Exception from kernel code */ -+ mov x0, #0x0 -+ mov x1, #0x0 -+ mov x2, #0x0 -+ mov x3, #0x0 -+#endif -+ .endif -+ .endm ++#define pud_ERROR(e) \ ++ pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e)) + -+/* -+ * iee exception exit -+ */ -+ .macro iee_exception_exit, el -+ // Disable daif -+ disable_daif -+ -+ .if \el == 1 -+#ifdef CONFIG_IEE_INTERRUPTABLE -+ /* el1 pop elr_el1 and set pan */ -+ /* Check ELR_EL1 */ -+ ldr x1, =__iee_code_start -+ cmp x1, x22 -+ b.hi 9f -+ ldr x1, =__iee_code_end -+ cmp x1, x22 -+ b.lo 9f -+ /* Eret iee code */ -+ /* Disable PAN */ -+ msr pan, #0x0 -+ /* Switch to iee stack */ -+ add sp, sp, #PT_REGS_SIZE -+ mrs x0, sp_el0 /* x0 -> task_struct */ -+ adrp x2, iee_offset -+ ldr x2, [x2, #:lo12:iee_offset] -+ add x1, x0, x2 /* x1 -> task_token(IEE) */ -+ // store kernel stack -+ mov x3, sp -+ str x3, [x1, #kernel_from_token_offset] -+ // load iee stack -+ ldr x2, [x1, #iee_from_token_offset] -+ mov sp, x2 -+ /* Load ELR_EL1 from iee stack */ -+ ldr x21, [sp, #S_PC] -+ /* Check the modify of ELR_EL1 */ -+ cmp x21, x22 -+ b.ne 8f -+ /* ELR_EL1 not modified */ -+ b 9f -+ -+8: -+ // ELR_EL1 modified -+ mov x0, sp -+ mov x1, #BAD_ELR_EL1 -+ mrs x2, esr_el1 -+ bl iee_bad_mode -+ ASM_BUG() ++#define p4d_none(p4d) (!p4d_val(p4d)) ++#define p4d_bad(p4d) (!(p4d_val(p4d) & 2)) ++#define p4d_present(p4d) (p4d_val(p4d)) ++extern bool check_addr_in_iee_valid(unsigned long addr); ++static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) ++{ ++ if (in_swapper_pgdir(p4dp)) { ++ set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } + -+9: -+ // Eret kernel code -+ mov x0, #0x0 -+ mov x1, #0x0 -+ mov x2, #0x0 -+ mov x3, #0x0 -+#endif -+ .endif -+ .endm - #ifdef CONFIG_FAST_SYSCALL - .macro check_esr_el1_ec_svc64 - /* Only support SVC64 for now */ -@@ -731,8 +1336,18 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) - .endif - #endif - kernel_entry \el, \regsize ++ if (in_tramp_pgdir(p4dp)) { ++ iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } + -+ #ifdef CONFIG_IEE -+ iee_exception_entry \el -+ #endif ++ if (in_idmap_pgdir(p4dp)) { ++ iee_set_static_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } + - mov x0, sp - bl el\el\ht\()_\regsize\()_\label\()_handler ++ iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); ++ dsb(ishst); ++ isb(); ++} + -+ #ifdef CONFIG_IEE -+ iee_exception_exit \el -+ #endif ++static inline void p4d_clear(p4d_t *p4dp) ++{ ++ set_p4d(p4dp, __p4d(0)); ++} + - .if \el == 0 - b ret_to_user - .else -@@ -764,9 +1379,15 @@ SYM_CODE_END(el\el\ht\()_\regsize\()_\label) - entry_handler 0, t, 32, fiq - entry_handler 0, t, 32, error - -+#ifdef CONFIG_KOI -+.pushsection ".koi.text", "ax" -+#endif - SYM_CODE_START_LOCAL(ret_to_kernel) - kernel_exit 1 - SYM_CODE_END(ret_to_kernel) -+#ifdef CONFIG_KOI -+.popsection -+#endif - - SYM_CODE_START_LOCAL(ret_to_user) - ldr x19, [tsk, #TSK_TI_FLAGS] // re-check for single-step -diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c -index 0137d987631e..fbb543bcdb4a 100644 ---- a/arch/arm64/kernel/fpsimd.c -+++ b/arch/arm64/kernel/fpsimd.c -@@ -1309,7 +1309,11 @@ void sme_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) - isb(); - - /* Allow EL0 to access TPIDR2 */ -+ #ifdef CONFIG_IEE -+ iee_rwx_gate_entry(IEE_WRITE_sctlr_el1, read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2); -+ #else - write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1); -+ #endif - isb(); - } - -diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S -index 6517bf2644a0..3d0716d59c0b 100644 ---- a/arch/arm64/kernel/head.S -+++ b/arch/arm64/kernel/head.S -@@ -661,6 +661,7 @@ SYM_FUNC_START_LOCAL(secondary_startup) - SYM_FUNC_END(secondary_startup) - - .text -+ - SYM_FUNC_START_LOCAL(__secondary_switched) - mov x0, x20 - bl set_cpu_boot_mode_flag -@@ -746,6 +747,10 @@ SYM_FUNC_START(__enable_mmu) - cmp x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX - b.gt __no_granule_support - phys_to_ttbr x2, x2 -+#ifdef CONFIG_IEE -+ mov x3, #1 -+ bfi x2, x3, #48, #16 // ASID 1 is used by IEE rwx gate. -+#endif - msr ttbr0_el1, x2 // load TTBR0 - load_ttbr1 x1, x1, x3 - -diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c -index 02870beb271e..76d86b3d71b1 100644 ---- a/arch/arm64/kernel/hibernate.c -+++ b/arch/arm64/kernel/hibernate.c -@@ -34,6 +34,10 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif ++static inline phys_addr_t p4d_page_paddr(p4d_t p4d) ++{ ++ return __p4d_to_phys(p4d); ++} + - /* - * Hibernate core relies on this value being 0 on resume, and marks it - * __nosavedata assuming it will keep the resume kernel's '0' value. This -@@ -203,6 +207,11 @@ static int create_safe_exec_page(void *src_start, size_t length, - - memcpy(page, src_start, length); - caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length); ++static inline pud_t *p4d_pgtable(p4d_t p4d) ++{ ++ return (pud_t *)__va(p4d_page_paddr(p4d)); ++} + -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)page_address(page),0); -+ #endif ++/* Find an entry in the first-level page table. */ ++#define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) \ ++ + pud_index(addr) * sizeof(pud_t)) + - rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page); - if (rc) - return rc; -diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c -index d39a8787edf2..b5ac4b7670bc 100644 ---- a/arch/arm64/kernel/hw_breakpoint.c -+++ b/arch/arm64/kernel/hw_breakpoint.c -@@ -26,6 +26,10 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#endif ++#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) ++#define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr)) ++#define pud_clear_fixmap() clear_fixmap(FIX_PUD) + - /* Breakpoint currently in use for each BRP. */ - static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); - -@@ -102,13 +106,68 @@ int hw_breakpoint_slots(int type) - WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ - WRITE_WB_REG_CASE(OFF, 15, REG, VAL) - -+#ifdef CONFIG_IEE ++#define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d))) + -+#define IEE_SI_READ_WB_REG_CASE(OFF, N, REG, VAL) \ -+ case (OFF + N): \ -+ IEE_SI_AARCH64_DBG_READ(N, REG, VAL); \ -+ break -+ -+#define IEE_SI_WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ -+ case (OFF + N): \ -+ IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL); \ -+ break -+ -+#define IEE_SI_GEN_READ_REG_CASES(OFF, REG, VAL) \ -+ IEE_SI_READ_WB_REG_CASE(OFF, 0, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 15, REG, VAL) -+ -+#define IEE_SI_GEN_WRITE_REG_CASES(OFF, REG, VAL) \ -+ IEE_SI_WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ -+ WRITE_WB_REG_CASE(OFF, 15, REG, VAL) -+ -+#endif -+ - static u64 read_wb_reg(int reg, int n) - { - u64 val = 0; - - switch (reg + n) { -+// #ifdef CONFIG_IEE -+// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); -+// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); -+// #else - GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); - GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); -+// #endif - GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); - GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); - default: -@@ -122,8 +181,13 @@ NOKPROBE_SYMBOL(read_wb_reg); - static void write_wb_reg(int reg, int n, u64 val) - { - switch (reg + n) { -+// #ifdef CONFIG_IEE -+// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); -+// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); -+// #else - GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); - GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); -+// #endif - GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); - GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); - default: -@@ -171,6 +235,10 @@ static int is_a32_compat_bp(struct perf_event *bp) - return tsk && is_a32_compat_thread(task_thread_info(tsk)); - } - -+#ifdef CONFIG_IEE -+int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); -+#endif ++/* use ONLY for statically allocated translation tables */ ++#define pud_offset_kimg(dir, addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr)))) + - /** - * hw_breakpoint_slot_setup - Find and setup a perf slot according to - * operations -@@ -191,6 +259,37 @@ static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, - { - int i; - struct perf_event **slot; -+// reserve hw breakpoint 0 for iee rwx gate in kernel sapce. -+// #ifdef CONFIG_IEE -+// struct arch_hw_breakpoint *info = counter_arch_bp(bp); -+// if (arch_check_bp_in_kernelspace(info)){ -+// for (i = 1; i < max_slots; ++i) { // search from hw breakpoint 1 -+// slot = &slots[i]; -+// switch (ops) { -+// case HW_BREAKPOINT_INSTALL: -+// if (!*slot) { -+// *slot = bp; -+// return i; -+// } -+// break; -+// case HW_BREAKPOINT_UNINSTALL: -+// if (*slot == bp) { -+// *slot = NULL; -+// return i; -+// } -+// break; -+// case HW_BREAKPOINT_RESTORE: -+// if (*slot == bp) -+// return i; -+// break; -+// default: -+// pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); -+// return -EINVAL; -+// } -+// } -+// return -ENOSPC; -+// } -+// #endif - - for (i = 0; i < max_slots; ++i) { - slot = &slots[i]; -diff --git a/arch/arm64/kernel/iee/Makefile b/arch/arm64/kernel/iee/Makefile -new file mode 100644 -index 000000000000..c62a1cc4f03b ---- /dev/null -+++ b/arch/arm64/kernel/iee/Makefile -@@ -0,0 +1,5 @@ -+ccflags-$(CONFIG_IEE_SELINUX_P) := -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include ++#else + -+obj-$(CONFIG_IEE) += iee.o iee-gate.o iee-func.o iee-pgtable.o stack_slab.o pgtable_slab.o ++#define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0; }) ++#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0; }) + -+obj-$(CONFIG_IEE_SELINUX_P) += iee-selinuxp.o -\ No newline at end of file -diff --git a/arch/arm64/kernel/iee/iee-func.c b/arch/arm64/kernel/iee/iee-func.c -new file mode 100644 -index 000000000000..29035c96a4f2 ---- /dev/null -+++ b/arch/arm64/kernel/iee/iee-func.c -@@ -0,0 +1,722 @@ -+#include "asm/pgtable.h" -+#include -+#include -+#include -+#include -+#include -+#include ++/* Match pud_offset folding in */ ++#define pud_set_fixmap(addr) NULL ++#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) ++#define pud_clear_fixmap() + -+extern s64 iee_offset; -+extern void iee_split_huge_pmd(pmd_t *pmdp, pte_t *pgtable); -+#ifdef CONFIG_PTP -+extern phys_addr_t __init early_pgtable_alloc(int shift); -+#endif ++#define pud_offset_kimg(dir, addr) ((pud_t *)dir) + -+static inline void iee_set_token(pte_t *ptep, void *new, unsigned long order) -+{ -+#ifdef CONFIG_PTP -+ pgd_t *pgdir; -+ pgd_t *pgdp; -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ int use_block_pmd = 0; ++#endif /* CONFIG_PGTABLE_LEVELS > 3 */ + -+ pgdir = swapper_pg_dir; -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)new); -+ p4dp = p4d_offset(pgdp, (unsigned long)new); -+ pudp = pud_offset(p4dp, (unsigned long)new); -+ pmdp = pmd_offset(pudp, (unsigned long)new); ++#define pgd_ERROR(e) \ ++ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) + -+ // Handling cont mapping. -+ if(pmd_val(*pmdp) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)new & CONT_PMD_MASK); -+ for(i = 0; i < CONT_PMDS; i++) -+ { -+ set_pmd(pmdp,__pmd(pmd_val(*pmdp) & ~PTE_CONT)); -+ pmdp++; -+ } -+ } ++#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) ++#define pgd_clear_fixmap() clear_fixmap(FIX_PGD) + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ #ifndef CONFIG_PTP -+ struct page *page = pmd_page(*pmdp); -+ #endif -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); ++static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) ++{ ++ /* ++ * Normal and Normal-Tagged are two different memory types and indices ++ * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK. ++ */ ++ const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | ++ PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP | ++ PTE_ATTRINDX_MASK; ++ /* preserve the hardware dirty information */ ++ if (pte_hw_dirty(pte)) ++ pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); ++ ++ pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); ++ /* ++ * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware ++ * dirtiness again. ++ */ ++ if (pte_sw_dirty(pte)) ++ pte = pte_mkdirty(pte); ++ return pte; ++} + -+ if (!pgtable) -+ panic("Alloc pgtable error.\n"); ++static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) ++{ ++ return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); ++} + -+ iee_split_huge_pmd(pmdp, pgtable); ++extern int __ptep_set_access_flags(struct vm_area_struct *vma, ++ unsigned long address, pte_t *ptep, ++ pte_t entry, int dirty); + -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if(pmd_leaf(READ_ONCE(*pmdp))) -+ { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS ++static inline int pmdp_set_access_flags(struct vm_area_struct *vma, ++ unsigned long address, pmd_t *pmdp, ++ pmd_t entry, int dirty) ++{ ++ return __pmdp_set_access_flags(vma, address, pmdp, ++ entry, dirty); ++} + -+ if(pgtable) -+ { -+ #ifdef CONFIG_PTP -+ iee_memset(pgtable, 0, PAGE_SIZE); -+ #endif -+ pte_free_kernel(&init_mm, pgtable); -+ } -+ } -+ else if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ } ++static inline int pud_devmap(pud_t pud) ++{ ++ return 0; ++} + -+ iee_rw_gate(IEE_SET_TOKEN, ptep, new, order, use_block_pmd); -+#else -+ int i; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, new); -+ p4d_t *p4dp = p4d_offset(pgdp, new); -+ pud_t *pudp = pud_offset(p4dp, new); -+ pmd_t *pmdp; -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ int use_block_pmd = 0; ++static inline int pgd_devmap(pgd_t pgd) ++{ ++ return 0; ++} ++#endif + -+ pmdp = pmd_offset(pudp, new); ++#ifdef CONFIG_PAGE_TABLE_CHECK ++static inline bool pte_user_accessible_page(pte_t pte) ++{ ++ return pte_present(pte) && (pte_user(pte) || pte_user_exec(pte)); ++} + -+ // Handling cont mapping. -+ if(pmd_val(*pmdp) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)new & CONT_PMD_MASK); -+ for(i = 0; i < CONT_PMDS; i++) -+ { -+ set_pmd(pmdp,__pmd(pmd_val(*pmdp) & ~PTE_CONT)); -+ pmdp++; -+ } -+ } ++static inline bool pmd_user_accessible_page(pmd_t pmd) ++{ ++ return pmd_leaf(pmd) && !pmd_present_invalid(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd)); ++} + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp) && order < 9) -+ { -+ struct page *page = pmd_page(*pmdp); -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); -+ int i; -+ pte_t *ptep = pgtable; ++static inline bool pud_user_accessible_page(pud_t pud) ++{ ++ return pud_leaf(pud) && (pud_user(pud) || pud_user_exec(pud)); ++} ++#endif + -+ if (!pgtable) -+ panic("Alloc pgtable error.\n"); ++/* ++ * Atomic pte/pmd modifications. ++ */ ++static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, ++ unsigned long address, ++ pte_t *ptep) ++{ ++ pte_t old_pte, pte; + -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = PAGE_KERNEL; -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); ++ pte = __ptep_get(ptep); ++ do { ++ old_pte = pte; ++ pte = pte_mkold(pte); ++ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, ++ pte_val(old_pte), pte_val(pte)); ++ } while (pte_val(pte) != pte_val(old_pte)); + -+ entry = mk_pte(page + i, pgprot); -+ set_pte(ptep, entry); -+ } ++ return pte_young(pte); ++} + -+ spin_lock(&init_mm.page_table_lock); -+ if(pmd_leaf(READ_ONCE(*pmdp))) -+ { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(&init_mm.page_table_lock); ++static inline int __ptep_clear_flush_young(struct vm_area_struct *vma, ++ unsigned long address, pte_t *ptep) ++{ ++ int young = __ptep_test_and_clear_young(vma, address, ptep); + -+ if(pgtable) -+ { -+ pte_free_kernel(&init_mm, pgtable); -+ } -+ } -+ else if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; ++ if (young) { ++ /* ++ * We can elide the trailing DSB here since the worst that can ++ * happen is that a CPU continues to use the young entry in its ++ * TLB and we mistakenly reclaim the associated page. The ++ * window for such an event is bounded by the next ++ * context-switch, which provides a DSB to complete the TLB ++ * invalidation. ++ */ ++ flush_tlb_page_nosync(vma, address); + } + -+ if(use_block_pmd) -+ lm_ptep = (pte_t *)pmdp; -+ else -+ lm_ptep = pte_offset_kernel(pmdp, new); ++ return young; ++} + -+ // Handling cont mapping. -+ if(pte_val(*lm_ptep) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)new & CONT_PTE_MASK); -+ if(order < CONFIG_ARM64_CONT_PTE_SHIFT) -+ { -+ for(i = 0; i < CONT_PTES; i++) -+ { -+ set_pte(ptep,__pte(pte_val(*ptep) & ~PTE_CONT)); -+ ptep++; -+ } -+ } -+ } -+ -+ iee_addr = ((unsigned long)new + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); ++static inline int __pmdp_test_and_clear_young(struct vm_area_struct *vma, ++ unsigned long address, ++ pmd_t *pmdp) ++{ ++ pmd_t old_pmd, pmd; + -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)lm_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*pmdp, pmd); -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | 0x1); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ iee_ptep++; -+ new += PAGE_SIZE; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | 0x1); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ lm_ptep++; -+ iee_ptep++; -+ new += PAGE_SIZE; -+ } -+ } -+#endif -+ dsb(ishst); -+ isb(); ++ pmd = READ_ONCE(*pmdp); ++ do { ++ old_pmd = pmd; ++ pmd = __pmd(pmd_val(pmd) & ~PTE_AF); ++ pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, ++ pmd_val(old_pmd), pmd_val(pmd)); ++ } while (pmd_val(pmd) != pmd_val(old_pmd)); ++ ++ return pmd_young(pmd); +} + -+static inline void iee_unset_token(pte_t *ptep, void *token_addr, void *token_page, unsigned long order) ++static inline void __pmdp_set_wrprotect(struct mm_struct *mm, ++ unsigned long address, pmd_t *pmdp, ++ pmd_t pmd) +{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_UNSET_TOKEN, ptep, token_addr, token_page, order); -+#else -+ int i; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, token_page); -+ p4d_t *p4dp = p4d_offset(pgdp, token_page); -+ pud_t *pudp = pud_offset(p4dp, token_page); -+ pmd_t *pmdp = pmd_offset(pudp, token_page); -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ int use_block_pmd = 0; ++ pmd_t old_pmd; + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ lm_ptep = (pte_t *)pmdp; -+ } -+ else -+ lm_ptep = pte_offset_kernel(pmdp, token_page); -+ -+ iee_addr = ((unsigned long)token_page + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); ++ do { ++ old_pmd = pmd; ++ pmd = pmd_wrprotect(pmd); ++ pmd_val(pmd) = iee_set_pmd_cmpxchg_relaxed(pmdp, pmd_val(old_pmd), pmd_val(pmd)); ++ } while (pmd_val(pmd) != pmd_val(old_pmd)); ++} + -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)lm_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | PTE_DBM); -+ WRITE_ONCE(*pmdp, pmd); -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~((unsigned long)0x1)) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token_addr - IEE_OFFSET))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~0x1); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ iee_ptep++; -+ token_addr += PAGE_SIZE; -+ token_page += PAGE_SIZE; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~((unsigned long)0x1)) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token_addr - IEE_OFFSET))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~0x1); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ lm_ptep++; -+ iee_ptep++; -+ token_addr += PAGE_SIZE; -+ token_page += PAGE_SIZE; -+ } -+ } -+#endif -+ dsb(ishst); -+ isb(); ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) ++#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG ++static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, ++ unsigned long address, ++ pmd_t *pmdp) ++{ ++ /* Operation applies to PMD table entry only if FEAT_HAFT is enabled */ ++ VM_WARN_ON(pmd_table(READ_ONCE(*pmdp)) && !system_supports_haft()); ++ return __pmdp_test_and_clear_young(vma, address, pmdp); +} ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ + -+// Input is the lm vaddr of sensitive data. -+void set_iee_page(unsigned long addr, int order) ++static inline pte_t __ptep_get_and_clear(struct mm_struct *mm, ++ unsigned long address, pte_t *ptep) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp; -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ int use_block_pmd = 0; ++ pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0); ++ pte_t pte = __pte(pteval); + -+ pmdp = pmd_offset(pudp, addr); ++ page_table_check_pte_clear(mm, pte); + -+ // Handling cont mapping. -+ if(pmd_val(*pmdp) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pmd_t *pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); -+ for(i = 0; i < CONT_PMDS; i++) -+ { -+ set_pmd(pmdp,__pmd(pmd_val(*pmdp) & ~PTE_CONT)); -+ pmdp++; -+ } ++ return pte; ++} ++ ++static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, unsigned int nr, int full) ++{ ++ for (;;) { ++ __ptep_get_and_clear(mm, addr, ptep); ++ if (--nr == 0) ++ break; ++ ptep++; ++ addr += PAGE_SIZE; + } ++} + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp) && order < 9) -+ { -+ #ifndef CONFIG_PTP -+ struct page *page = pmd_page(*pmdp); -+ #endif -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); ++static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, int full) ++{ ++ pte_t pte, tmp_pte; + -+ if (!pgtable) -+ panic("Alloc pgtable error.\n"); ++ pte = __ptep_get_and_clear(mm, addr, ptep); ++ while (--nr) { ++ ptep++; ++ addr += PAGE_SIZE; ++ tmp_pte = __ptep_get_and_clear(mm, addr, ptep); ++ if (pte_dirty(tmp_pte)) ++ pte = pte_mkdirty(pte); ++ if (pte_young(tmp_pte)) ++ pte = pte_mkyoung(pte); ++ } ++ return pte; ++} + -+ #ifdef CONFIG_PTP -+ iee_split_huge_pmd(pmdp, pgtable); -+ #else -+ { -+ int i; -+ pte_t *ptep = pgtable; -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = PAGE_KERNEL; -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR ++static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, ++ unsigned long address, pmd_t *pmdp) ++{ ++ pteval_t pteval = iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0); ++ pmd_t pmd = __pmd(pteval); + -+ entry = mk_pte(page + i, pgprot); -+ set_pte(ptep, entry); -+ } -+ } -+ #endif ++ page_table_check_pmd_clear(mm, pmd); + -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if(pmd_leaf(READ_ONCE(*pmdp))) -+ { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); ++ return pmd; ++} ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + -+ if(pgtable) -+ { -+ #ifdef CONFIG_PTP -+ iee_memset(pgtable, 0, PAGE_SIZE); -+ #endif -+ pte_free_kernel(&init_mm, pgtable); -+ } -+ } -+ else if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ } ++static inline void ___ptep_set_wrprotect(struct mm_struct *mm, ++ unsigned long address, pte_t *ptep, ++ pte_t pte) ++{ ++ pte_t old_pte; + -+ if(use_block_pmd) -+ lm_ptep = (pte_t *)pmdp; -+ else -+ lm_ptep = pte_offset_kernel(pmdp, addr); ++ do { ++ old_pte = pte; + -+ // Handling cont mapping. -+ if(pte_val(*lm_ptep) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pte_t *ptep = pte_offset_kernel(pmdp, addr & CONT_PTE_MASK); -+ if(order < CONFIG_ARM64_CONT_PTE_SHIFT) -+ { -+ for(i = 0; i < CONT_PTES; i++) -+ { -+ set_pte(ptep,__pte(pte_val(*ptep) & ~PTE_CONT)); -+ ptep++; -+ } -+ } -+ } -+ -+ iee_addr = ((unsigned long)addr + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); -+ isb(); ++ pte = pte_wrprotect(pte); ++ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, pte_val(old_pte), pte_val(pte)); ++ } while (pte_val(pte) != pte_val(old_pte)); +} + -+// Input is the lm vaddr of sensitive data. -+void unset_iee_page(unsigned long addr, int order) ++/* ++ * __ptep_set_wrprotect - mark read-only while trasferring potential hardware ++ * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. ++ */ ++static inline void __ptep_set_wrprotect(struct mm_struct *mm, ++ unsigned long address, pte_t *ptep) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ int use_block_pmd = 0; -+ -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ lm_ptep = (pte_t *)pmdp; -+ } -+ else -+ lm_ptep = pte_offset_kernel(pmdp, addr); -+ -+ iee_addr = ((unsigned long)addr + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ iee_unset_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); -+ flush_tlb_kernel_range(iee_addr, iee_addr+PAGE_SIZE*(1 << order)); -+ isb(); ++ ___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep)); +} + -+void set_iee_page_valid(unsigned long addr) ++static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, ++ pte_t *ptep, unsigned int nr) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ unsigned int i; + -+ pud_t *pudp = pud_offset(p4dp, addr); ++ for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) ++ __ptep_set_wrprotect(mm, address, ptep); ++} + -+ pmd_t *pmdp = pmd_offset(pudp, addr); ++static inline void __clear_young_dirty_pte(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ pte_t pte, cydp_t flags) ++{ ++ pte_t old_pte; + -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ pte_t pte = READ_ONCE(*ptep); ++ do { ++ old_pte = pte; + -+ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) -+ return; ++ if (flags & CYDP_CLEAR_YOUNG) ++ pte = pte_mkold(pte); ++ if (flags & CYDP_CLEAR_DIRTY) ++ pte = pte_mkclean(pte); + -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ set_pte(ptep, pte); ++ pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), ++ pte_val(old_pte), pte_val(pte)); ++ } while (pte_val(pte) != pte_val(old_pte)); +} + -+void iee_set_logical_mem_ro(unsigned long addr) ++static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, cydp_t flags) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp; -+ pte_t *ptep; + pte_t pte; + -+ pmdp = pmd_offset(pudp, addr); ++ for (;;) { ++ pte = __ptep_get(ptep); + -+ // Handling cont mapping. -+ if(pmd_val(*pmdp) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pmd_t *pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); -+ for(i = 0; i < CONT_PMDS; i++) -+ { -+ set_pmd(pmdp,__pmd(pmd_val(*pmdp) & ~PTE_CONT)); -+ pmdp++; -+ } ++ if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY)) ++ __set_pte(ptep, pte_mkclean(pte_mkold(pte))); ++ else ++ __clear_young_dirty_pte(vma, addr, ptep, pte, flags); ++ ++ if (--nr == 0) ++ break; ++ ptep++; ++ addr += PAGE_SIZE; + } ++} + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ #ifndef CONFIG_PTP -+ struct page *page = pmd_page(*pmdp); -+ #endif -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#define __HAVE_ARCH_PMDP_SET_WRPROTECT ++static inline void pmdp_set_wrprotect(struct mm_struct *mm, ++ unsigned long address, pmd_t *pmdp) ++{ ++ __pmdp_set_wrprotect(mm, address, pmdp, READ_ONCE(*pmdp)); ++} + -+ if (!pgtable) -+ panic("Alloc pgtable error.\n"); ++#define pmdp_establish pmdp_establish ++static inline pmd_t pmdp_establish(struct vm_area_struct *vma, ++ unsigned long address, pmd_t *pmdp, pmd_t pmd) ++{ ++ page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); ++ return __pmd(iee_set_pmd_xchg_relaxed((pmd_t *)&pmd_val(*pmdp), pmd_val(pmd))); ++} ++#endif + -+ #ifdef CONFIG_PTP -+ iee_split_huge_pmd(pmdp, pgtable); -+ #else -+ { -+ int i; -+ pte_t *ptep = pgtable; -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = PAGE_KERNEL; -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); ++/* ++ * Encode and decode a swap entry: ++ * bits 0-1: present (must be zero) ++ * bits 2: remember PG_anon_exclusive ++ * bits 3-7: swap type ++ * bits 8-57: swap offset ++ * bit 58: PTE_PROT_NONE (must be zero) ++ */ ++#define __SWP_TYPE_SHIFT 3 ++#define __SWP_TYPE_BITS 5 ++#define __SWP_OFFSET_BITS 50 ++#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) ++#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) ++#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) ++ ++#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) ++#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK) ++#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) \ ++ | ((offset) << __SWP_OFFSET_SHIFT) }) ++ ++#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) ++#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) ++ ++#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION ++#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) ++#define __swp_entry_to_pmd(swp) __pmd((swp).val) ++#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ + -+ entry = mk_pte(page + i, pgprot); -+ set_pte(ptep, entry); -+ } -+ } -+ #endif ++/* ++ * Ensure that there are not more swap files than can be encoded in the kernel ++ * PTEs. ++ */ ++#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) + -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if(pmd_leaf(READ_ONCE(*pmdp))) -+ { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); ++#ifdef CONFIG_ARM64_MTE + -+ if(pgtable) -+ { -+ #ifdef CONFIG_PTP -+ iee_memset(pgtable, 0, PAGE_SIZE); -+ #endif -+ pte_free_kernel(&init_mm, pgtable); -+ } -+ } ++#define __HAVE_ARCH_PREPARE_TO_SWAP ++extern int arch_prepare_to_swap(struct folio *folio); + -+ ptep = pte_offset_kernel(pmdp, addr); ++#define __HAVE_ARCH_SWAP_INVALIDATE ++static inline void arch_swap_invalidate_page(int type, pgoff_t offset) ++{ ++ if (system_supports_mte()) ++ mte_invalidate_tags(type, offset); ++} + -+ // Handling cont mapping. -+ if(pte_val(*ptep) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pte_t *ptep = pte_offset_kernel(pmdp, addr & CONT_PTE_MASK); -+ for(i = 0; i < CONT_PTES; i++) -+ { -+ set_pte(ptep,__pte(pte_val(*ptep) & ~PTE_CONT)); -+ ptep++; -+ } -+ } ++static inline void arch_swap_invalidate_area(int type) ++{ ++ if (system_supports_mte()) ++ mte_invalidate_tags_area(type); ++} + -+ pte = READ_ONCE(*ptep); ++#define __HAVE_ARCH_SWAP_RESTORE ++extern void arch_swap_restore(swp_entry_t entry, struct folio *folio); + -+ if(addr < PAGE_OFFSET) -+ return; -+ -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ set_pte(ptep, pte); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); -+ isb(); ++#endif /* CONFIG_ARM64_MTE */ ++ ++/* ++ * On AArch64, the cache coherency is handled via the __set_ptes() function. ++ */ ++static inline void update_mmu_cache_range(struct vm_fault *vmf, ++ struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, ++ unsigned int nr) ++{ ++ /* ++ * We don't do anything here, so there's a very small chance of ++ * us retaking a user fault which we just fixed up. The alternative ++ * is doing a dsb(ishst), but that penalises the fastpath. ++ */ ++} ++ ++#define update_mmu_cache(vma, addr, ptep) \ ++ update_mmu_cache_range(NULL, vma, addr, ptep, 1) ++#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) ++ ++#ifdef CONFIG_ARM64_PA_BITS_52 ++#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52) ++#else ++#define phys_to_ttbr(addr) (addr) ++#endif ++ ++/* ++ * On arm64 without hardware Access Flag, copying from user will fail because ++ * the pte is old and cannot be marked young. So we always end up with zeroed ++ * page after fork() + CoW for pfn mappings. We don't always have a ++ * hardware-managed access flag on arm64. ++ */ ++#define arch_has_hw_pte_young cpu_has_hw_af ++ ++#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG ++#define arch_has_hw_nonleaf_pmd_young system_supports_haft ++#endif ++ ++/* ++ * Experimentally, it's cheap to set the access flag in hardware and we ++ * benefit from prefaulting mappings as 'old' to start with. ++ */ ++#define arch_wants_old_prefaulted_pte cpu_has_hw_af ++ ++/* ++ * Request exec memory is read into pagecache in at least 64K folios. The ++ * trade-off here is performance improvement due to storing translations more ++ * effciently in the iTLB vs the potential for read amplification due to reading ++ * data from disk that won't be used. The latter is independent of base page ++ * size, so we set a page-size independent block size of 64K. This size can be ++ * contpte-mapped when 4K base pages are in use (16 pages into 1 iTLB entry), ++ * and HPA can coalesce it (4 pages into 1 TLB entry) when 16K base pages are in ++ * use. ++ */ ++#define arch_wants_exec_folio_order() ilog2(SZ_64K >> PAGE_SHIFT) ++ ++static inline bool pud_sect_supported(void) ++{ ++ return PAGE_SIZE == SZ_4K; +} + -+void iee_set_token_page_valid(void *token, void *new, unsigned int order) ++ ++#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION ++#define ptep_modify_prot_start ptep_modify_prot_start ++extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep); ++ ++#define ptep_modify_prot_commit ptep_modify_prot_commit ++extern void ptep_modify_prot_commit(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ pte_t old_pte, pte_t new_pte); ++ ++#ifdef CONFIG_ARM64_CONTPTE ++ ++/* ++ * The contpte APIs are used to transparently manage the contiguous bit in ptes ++ * where it is possible and makes sense to do so. The PTE_CONT bit is considered ++ * a private implementation detail of the public ptep API (see below). ++ */ ++extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, pte_t pte); ++extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, pte_t pte); ++extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte); ++extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep); ++extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, pte_t pte, unsigned int nr); ++extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, unsigned int nr, int full); ++extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, int full); ++extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep); ++extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep); ++extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, unsigned int nr); ++extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ pte_t entry, int dirty); ++extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, cydp_t flags); ++ ++static __always_inline void contpte_try_fold(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, pte_t pte) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); ++ /* ++ * Only bother trying if both the virtual and physical addresses are ++ * aligned and correspond to the last entry in a contig range. The core ++ * code mostly modifies ranges from low to high, so this is the likely ++ * the last modification in the contig range, so a good time to fold. ++ * We can't fold special mappings, because there is no associated folio. ++ */ + -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); ++ const unsigned long contmask = CONT_PTES - 1; ++ bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); ++ if (unlikely(valign)) { ++ bool palign = (pte_pfn(pte) & contmask) == contmask; + -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); ++ if (unlikely(palign && ++ pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) ++ __contpte_try_fold(mm, addr, ptep, pte); ++ } ++} + -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); -+ iee_set_token(ptep, new, order); ++static __always_inline void contpte_try_unfold(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, pte_t pte) ++{ ++ if (unlikely(pte_valid_cont(pte))) ++ __contpte_try_unfold(mm, addr, ptep, pte); ++} + -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)new, (unsigned long)(new + (PAGE_SIZE * (1 << order)))); ++#define pte_batch_hint pte_batch_hint ++static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) ++{ ++ if (!pte_valid_cont(pte)) ++ return 1; + -+ isb(); ++ return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); +} + -+void iee_set_token_page_invalid(void *token_addr, void *token_page, unsigned long order) ++/* ++ * The below functions constitute the public API that arm64 presents to the ++ * core-mm to manipulate PTE entries within their page tables (or at least this ++ * is the subset of the API that arm64 needs to implement). These public ++ * versions will automatically and transparently apply the contiguous bit where ++ * it makes sense to do so. Therefore any users that are contig-aware (e.g. ++ * hugetlb, kernel mapper) should NOT use these APIs, but instead use the ++ * private versions, which are prefixed with double underscore. All of these ++ * APIs except for ptep_get_lockless() are expected to be called with the PTL ++ * held. Although the contiguous bit is considered private to the ++ * implementation, it is deliberately allowed to leak through the getters (e.g. ++ * ptep_get()), back to core code. This is required so that pte_leaf_size() can ++ * provide an accurate size for perf_get_pgtable_size(). But this leakage means ++ * its possible a pte will be passed to a setter with the contiguous bit set, so ++ * we explicitly clear the contiguous bit in those cases to prevent accidentally ++ * setting it in the pgtable. ++ */ ++ ++#define ptep_get ptep_get ++static inline pte_t ptep_get(pte_t *ptep) +{ -+ unsigned long iee_addr = ((unsigned long)token_page + (unsigned long)iee_offset); -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token_addr); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token_addr); -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token_addr); -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token_addr); ++ pte_t pte = __ptep_get(ptep); + -+ if(token_page == NULL) -+ panic("Token of task_struct was unset.\n"); ++ if (likely(!pte_valid_cont(pte))) ++ return pte; + -+ iee_unset_token(ptep, token_addr, token_page, order); ++ return contpte_ptep_get(ptep, pte); ++} + -+ flush_tlb_kernel_range((unsigned long)token_addr, (unsigned long)(token_addr + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)token_page, (unsigned long)(token_page + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range(iee_addr, iee_addr + (PAGE_SIZE * (1 << order))); -+ isb(); ++#define ptep_get_lockless ptep_get_lockless ++static inline pte_t ptep_get_lockless(pte_t *ptep) ++{ ++ pte_t pte = __ptep_get(ptep); ++ ++ if (likely(!pte_valid_cont(pte))) ++ return pte; ++ ++ return contpte_ptep_get_lockless(ptep); +} + -+void unset_iee_stack_page(unsigned long addr, int order) ++static inline void set_pte(pte_t *ptep, pte_t pte) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ pte_t *ptep; -+ int use_block_pmd = 0; ++ /* ++ * We don't have the mm or vaddr so cannot unfold contig entries (since ++ * it requires tlb maintenance). set_pte() is not used in core code, so ++ * this should never even be called. Regardless do our best to service ++ * any call and emit a warning if there is any attempt to set a pte on ++ * top of an existing contig range. ++ */ ++ pte_t orig_pte = __ptep_get(ptep); + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ ptep = (pte_t *)pmdp; ++ WARN_ON_ONCE(pte_valid_cont(orig_pte)); ++ __set_pte(ptep, pte_mknoncont(pte)); ++} ++ ++#define set_ptes set_ptes ++static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, pte_t pte, unsigned int nr) ++{ ++ pte = pte_mknoncont(pte); ++ ++ if (likely(nr == 1)) { ++ contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); ++ __set_ptes(mm, addr, ptep, pte, 1); ++ contpte_try_fold(mm, addr, ptep, pte); ++ } else { ++ contpte_set_ptes(mm, addr, ptep, pte, nr); + } -+ else -+ ptep = pte_offset_kernel(pmdp, addr); ++} + -+ iee_unset_stack_pte(ptep, order, use_block_pmd, addr); -+ flush_tlb_kernel_range(addr+iee_offset, addr+iee_offset+(1 << order)*PAGE_SIZE); -+ isb(); ++static inline void pte_clear(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep) ++{ ++ contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); ++ __pte_clear(mm, addr, ptep); +} + -+void set_iee_stack_page(unsigned long addr, int order) ++#define clear_full_ptes clear_full_ptes ++static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, unsigned int nr, int full) +{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp; -+ pte_t *ptep; -+ int use_block_pmd = 0; ++ if (likely(nr == 1)) { ++ contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); ++ __clear_full_ptes(mm, addr, ptep, nr, full); ++ } else { ++ contpte_clear_full_ptes(mm, addr, ptep, nr, full); ++ } ++} + -+ pmdp = pmd_offset(pudp, addr); ++#define get_and_clear_full_ptes get_and_clear_full_ptes ++static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, int full) ++{ ++ pte_t pte; + -+ // Handling cont mapping. -+ if(pmd_val(*pmdp) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pmd_t *pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); -+ for(i = 0; i < CONT_PMDS; i++) -+ { -+ set_pmd(pmdp,__pmd(pmd_val(*pmdp) & ~PTE_CONT)); -+ pmdp++; -+ } ++ if (likely(nr == 1)) { ++ contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); ++ pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full); ++ } else { ++ pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full); + } + -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp) && order < 9) -+ { -+ #ifndef CONFIG_PTP -+ struct page *page = pmd_page(*pmdp); -+ #endif -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); ++ return pte; ++} + -+ if (!pgtable) -+ panic("Alloc pgtable error.\n"); ++#define __HAVE_ARCH_PTEP_GET_AND_CLEAR ++static inline pte_t ptep_get_and_clear(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep) ++{ ++ contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); ++ return __ptep_get_and_clear(mm, addr, ptep); ++} + -+ #ifdef CONFIG_PTP -+ iee_split_huge_pmd(pmdp, pgtable); -+ #else -+ { -+ int i; -+ pte_t *ptep = pgtable; -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = PAGE_KERNEL; -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); ++#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG ++static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep) ++{ ++ pte_t orig_pte = __ptep_get(ptep); + -+ entry = mk_pte(page + i, pgprot); -+ set_pte(ptep, entry); -+ } -+ } -+ #endif ++ if (likely(!pte_valid_cont(orig_pte))) ++ return __ptep_test_and_clear_young(vma, addr, ptep); + -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if(pmd_leaf(READ_ONCE(*pmdp))) -+ { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); ++ return contpte_ptep_test_and_clear_young(vma, addr, ptep); ++} + -+ if(pgtable) -+ { -+ #ifdef CONFIG_PTP -+ iee_memset(pgtable, 0, PAGE_SIZE); -+ #endif -+ pte_free_kernel(&init_mm, pgtable); -+ } -+ } -+ else if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ } ++#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH ++static inline int ptep_clear_flush_young(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep) ++{ ++ pte_t orig_pte = __ptep_get(ptep); + -+ if(use_block_pmd) -+ ptep = (pte_t *)pmdp; -+ else -+ ptep = pte_offset_kernel(pmdp, addr); ++ if (likely(!pte_valid_cont(orig_pte))) ++ return __ptep_clear_flush_young(vma, addr, ptep); + -+ // Handling cont mapping. -+ if(pte_val(*ptep) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pte_t *ptep = pte_offset_kernel(pmdp, addr & CONT_PTE_MASK); -+ if(order < CONFIG_ARM64_CONT_PTE_SHIFT) -+ { -+ for(i = 0; i < CONT_PTES; i++) -+ { -+ set_pte(ptep,__pte(pte_val(*ptep) & ~PTE_CONT)); -+ ptep++; -+ } ++ return contpte_ptep_clear_flush_young(vma, addr, ptep); ++} ++ ++#define wrprotect_ptes wrprotect_ptes ++static __always_inline void wrprotect_ptes(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep, unsigned int nr) ++{ ++ if (likely(nr == 1)) { ++ /* ++ * Optimization: wrprotect_ptes() can only be called for present ++ * ptes so we only need to check contig bit as condition for ++ * unfold, and we can remove the contig bit from the pte we read ++ * to avoid re-reading. This speeds up fork() which is sensitive ++ * for order-0 folios. Equivalent to contpte_try_unfold(). ++ */ ++ pte_t orig_pte = __ptep_get(ptep); ++ ++ if (unlikely(pte_cont(orig_pte))) { ++ __contpte_try_unfold(mm, addr, ptep, orig_pte); ++ orig_pte = pte_mknoncont(orig_pte); + } ++ ___ptep_set_wrprotect(mm, addr, ptep, orig_pte); ++ } else { ++ contpte_wrprotect_ptes(mm, addr, ptep, nr); + } ++} + -+ iee_set_stack_pte(ptep, order, use_block_pmd, addr); -+ flush_tlb_kernel_range(addr, addr+(1 << order)*PAGE_SIZE); -+ isb(); ++#define __HAVE_ARCH_PTEP_SET_WRPROTECT ++static inline void ptep_set_wrprotect(struct mm_struct *mm, ++ unsigned long addr, pte_t *ptep) ++{ ++ wrprotect_ptes(mm, addr, ptep, 1); +} + -+void __init iee_rest_init(void) {} -\ No newline at end of file -diff --git a/arch/arm64/kernel/iee/iee-gate.S b/arch/arm64/kernel/iee/iee-gate.S -new file mode 100644 -index 000000000000..3b039e1dfbc4 ---- /dev/null -+++ b/arch/arm64/kernel/iee/iee-gate.S -@@ -0,0 +1,314 @@ -+#include -+#include -+#include -+#include -+#include -+#include ++#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS ++static inline int ptep_set_access_flags(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ pte_t entry, int dirty) ++{ ++ pte_t orig_pte = __ptep_get(ptep); + -+#ifdef CONFIG_IEE ++ entry = pte_mknoncont(entry); + -+SYM_FUNC_START(iee_read_tmp_page) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* disable PAN */ -+ msr pan, #0x0 -+ adrp x12, iee_offset -+ ldr x12, [x12, #:lo12:iee_offset] -+ add x11, x0, x12 /* x11 -> task_token(IEE) */ -+ ldr x0, [x11, #tmp_page_from_token_offset] -+ /* enable PAN */ -+ msr pan, #0x1 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_read_tmp_page) ++ if (likely(!pte_valid_cont(orig_pte))) ++ return __ptep_set_access_flags(vma, addr, ptep, entry, dirty); + -+SYM_FUNC_START(iee_read_freeptr) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* disable PAN */ -+ msr pan, #0x0 -+ adrp x12, iee_offset -+ ldr x12, [x12, #:lo12:iee_offset] -+ add x0, x0, x12 -+ ldr x0, [x0] -+ /* enable PAN */ -+ msr pan, #0x1 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_read_freeptr) ++ return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty); ++} + -+SYM_FUNC_START(iee_read_token_stack) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* disable PAN */ -+ msr pan, #0x0 -+ adrp x12, iee_offset -+ ldr x12, [x12, #:lo12:iee_offset] -+ add x11, x0, x12 /* x11 -> task_token(IEE) */ -+ ldr x0, [x11, #iee_from_token_offset] -+ /* enable PAN */ -+ msr pan, #0x1 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_read_token_stack) ++#define clear_young_dirty_ptes clear_young_dirty_ptes ++static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep, ++ unsigned int nr, cydp_t flags) ++{ ++ if (likely(nr == 1 && !pte_cont(__ptep_get(ptep)))) ++ __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); ++ else ++ contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags); ++} ++ ++#else /* CONFIG_ARM64_CONTPTE */ ++ ++#define ptep_get __ptep_get ++#define set_pte __set_pte ++#define set_ptes __set_ptes ++#define pte_clear __pte_clear ++#define clear_full_ptes __clear_full_ptes ++#define get_and_clear_full_ptes __get_and_clear_full_ptes ++#define __HAVE_ARCH_PTEP_GET_AND_CLEAR ++#define ptep_get_and_clear __ptep_get_and_clear ++#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG ++#define ptep_test_and_clear_young __ptep_test_and_clear_young ++#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH ++#define ptep_clear_flush_young __ptep_clear_flush_young ++#define __HAVE_ARCH_PTEP_SET_WRPROTECT ++#define ptep_set_wrprotect __ptep_set_wrprotect ++#define wrprotect_ptes __wrprotect_ptes ++#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS ++#define ptep_set_access_flags __ptep_set_access_flags ++#define clear_young_dirty_ptes __clear_young_dirty_ptes ++ ++#endif /* CONFIG_ARM64_CONTPTE */ ++ ++#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP ++void vmemmap_update_pmd(unsigned long addr, pmd_t *pmdp, pte_t *ptep); ++#define vmemmap_update_pmd vmemmap_update_pmd ++void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte); ++#define vmemmap_update_pte vmemmap_update_pte ++#endif ++ ++#endif /* !__ASSEMBLY__ */ ++ ++#endif /* __ASM_PGTABLE_H */ +diff --git a/arch/arm64/include/asm/haoc/iee-ptp-init.h b/arch/arm64/include/asm/haoc/iee-ptp-init.h +new file mode 100644 +index 000000000000..bfd554611d2a +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-ptp-init.h +@@ -0,0 +1,56 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_PTP_INIT_H ++#define _LINUX_IEE_PTP_INIT_H + -+SYM_FUNC_START(iee_set_xchg_relaxed) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* disable PAN */ -+ msr pan, #0x0 -+ /* begin */ -+ adrp x2, iee_offset -+ ldr x2, [x2, #:lo12:iee_offset] -+ add x0, x0, x2 -+ prfm pstl1strm, [x0] -+12: -+ ldxr x3, [x0] -+ stxr w4, x1, [x0] -+ cbnz w4, 12b -+ mov x0, x3 -+ /* end */ -+ /* enable PAN */ -+ msr pan, #0x1 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_set_xchg_relaxed) ++#include + -+SYM_FUNC_START(iee_set_cmpxchg_relaxed) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* disable PAN */ -+ msr pan, #0x0 -+ /* begin */ -+ adrp x9, iee_offset -+ ldr x9, [x9, #:lo12:iee_offset] -+ add x3, x0, x9 -+ prfm pstl1strm, [x3] -+11: -+ ldxr x0, [x3] -+ eor x4, x0, x1 -+ cbnz w4, 13f -+ stxr w4, x2, [x3] -+ cbnz w4, 11b -+13: -+ /* end */ -+ /* enable PAN */ -+ msr pan, #0x1 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_set_cmpxchg_relaxed) ++#define NR_BM_PTE_TABLES \ ++ SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) ++#define NR_BM_PMD_TABLES \ ++ SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PUD_SHIFT) + -+SYM_FUNC_START(iee_rw_gate) -+ /* save daif, close irq */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+ /* save lr */ -+ sub sp, sp, #16 -+ stp x29, x30, [sp] -+ bl iee_protected_rw_gate -+ /* restore lr */ -+ ldp x29, x30, [sp] -+ add sp, sp, #16 -+ /* restore daif */ -+ msr daif, x13 -+ ret -+SYM_FUNC_END(iee_rw_gate) -+#if defined(CONFIG_CREDP) || defined(CONFIG_KOI) || (CONFIG_KEYP) -+EXPORT_SYMBOL(iee_rw_gate) -+#endif ++static_assert(NR_BM_PMD_TABLES == 1); + -+ .pushsection ".iee.text.header", "ax" ++#define __BM_TABLE_IDX(addr, shift) \ ++ (((addr) >> (shift)) - (FIXADDR_TOT_START >> (shift))) + -+SYM_FUNC_START(iee_protected_rw_gate) -+ /* disable PAN */ -+ msr pan, #0x0 -+ /* switch to iee stack */ -+ mrs x9, sp_el0 /* x9 -> task_struct */ -+ adrp x7, iee_offset -+ ldr x7, [x7, #:lo12:iee_offset] -+ add x11, x9, x7 /* x11 -> task_token(IEE) */ -+ // store kernel stack -+ mov x10, sp -+ str x10, [x11, #kernel_from_token_offset] -+ // load iee stack -+ ldr x10, [x11, #iee_from_token_offset] -+ mov sp, x10 -+#ifdef CONFIG_IEE_INTERRUPTABLE -+ isb -+ /* restore daif */ -+ msr daif, x13 -+ sub sp, sp, #16 -+ stp x11, x30, [sp] -+#else -+ sub sp, sp, #32 -+ stp x11, x7, [sp, #16] -+ stp x13, x30, [sp] -+#endif -+ /* call iee func */ -+ adrp x12, iee_funcs -+ add x12, x12, x0, lsl #3 -+ ldr x12, [x12, #:lo12:iee_funcs] -+ mov x0, x7 -+ blr x12 -+#ifdef CONFIG_IEE_INTERRUPTABLE -+ ldp x11, x30, [sp] -+ add sp, sp, #16 -+ /* store and disable daif */ -+ mrs x13, daif -+ msr daifset, #0x2 -+ isb -+#else -+ ldp x13, x30, [sp] -+ ldp x11, x7, [sp, #16] -+ add sp, sp, #32 -+#endif -+ /* switch to kernel stack */ -+ // load kernel stack -+ ldr x10, [x11, #kernel_from_token_offset] -+ mov sp, x10 -+ /* enable PAN */ -+ msr pan, #0x1 -+ ret -+SYM_FUNC_END(iee_protected_rw_gate) -+ -+ .popsection ++#define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) + -+#include -+#define BAD_IEE 4 -+#define BAD_IEE_SI 5 ++extern pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); ++extern pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; ++extern pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; + -+#define SYS_TCR_EL1_HPD1 0x40000000000 -+#define SYS_TCR_EL1_A1 0x400000 -+ -+ .pushsection ".iee.exec_entry", "ax" -+ -+SYM_FUNC_START(iee_rwx_gate_entry) -+ /* Disable irq first. */ -+ mrs x15, daif // use x15 to restore daif -+ msr DAIFSet, #0xf -+ isb -+ -+ /* Set HPD1 = 0 to exec follwing codes in U RWX page */ -+ mrs x9, tcr_el1 -+ bic x9, x9, #SYS_TCR_EL1_HPD1 -+ bic x9, x9, #SYS_TCR_EL1_A1 -+ msr tcr_el1, x9 -+ isb -+ -+ b iee_rwx_gate_tramp -+SYM_FUNC_END(iee_rwx_gate_entry) -+#ifdef CONFIG_KOI -+EXPORT_SYMBOL(iee_rwx_gate_entry) -+#endif -+ .popsection ++extern void __iee_pgd_populate_pre_init(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot); ++extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot); ++extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot); ++extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, ++ pmdval_t prot); ++extern int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot); ++extern int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot); + -+#define USER_ASID_FLAG (UL(1) << 48) ++extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, ++ phys_addr_t phys, pgprot_t flags); + -+ .pushsection ".iee.si_text", "awx" ++extern void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content); ++extern void set_iee_address_pre_init(unsigned long addr, bool valid); ++extern void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags); ++extern void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags); + -+SYM_FUNC_START(iee_rwx_gate_tramp) -+ /* Check tcr val. */ -+ mrs x10, tcr_el1 -+ adrp x12, iee_si_tcr // tcr value shall be const after init -+ ldr x12, [x12, #:lo12:iee_si_tcr] -+ cbz x12, 1f -+ cmp x12, x10 -+ b.ne 3f -+1: -+ mov x13, sp -+ /* if called by koi, skip stack switch */ -+#ifdef CONFIG_KOI -+ cmp x0, #IEE_SWITCH_TO_KERNEL -+ b.eq 4f -+ cmp x0, #IEE_SWITCH_TO_KOI -+ b.eq 5f -+#endif -+ -+ /* If iee hasn't been initialized, skip stack switch. */ -+ adrp x11, iee_init_done -+ ldr x10, [x11, #:lo12:iee_init_done] -+ cbz x10, 2f -+ -+ /* Switch to iee stack */ -+ mrs x9, sp_el0 // x9 -> task_struct -+ adrp x12, iee_si_offset -+ ldr x12, [x12, #:lo12:iee_si_offset] -+ add x11, x9, x12 // x11 -> task_token(IEE) -+ // load iee stack -+ ldr x10, [x11, #iee_from_token_offset] -+ mov sp, x10 -+ -+ /* x15 stores daif and x13 stores previous sp */ -+2: -+ stp x15, x13, [sp, #-32]! -+ stp x29, x30, [sp, #16] -+ bl iee_si_handler // enter actual handler -+ ldp x29, x30, [sp, #16] ++extern void __init efi_memmap_unmap_after_init(void); + -+ b iee_rwx_gate_exit // jump to iee exit -+3: -+ mov x0, sp -+ mov x1, #BAD_IEE_SI -+ mrs x2, esr_el1 -+ bl iee_bad_mode -+ ASM_BUG() -+#ifdef CONFIG_KOI -+/* Handle KOI switch gates. */ -+4: -+ /* IEE_SWITCH_TO_KERNEL */ -+ mrs x9, ttbr1_el1 -+ bic x9, x9, #USER_ASID_FLAG // Set new ASID to even -+ adrp x11, iee_base_swapper_pg_dir -+ ldr x11, [x11, #:lo12:iee_base_swapper_pg_dir] -+ ubfx x10, x9, #48, #16 // x10 -> new TTBR1 ASID -+ bfi x11, x10, #48, #16 -+ b 6f -+5: -+ /* IEE_SWITCH_TO_KOI */ -+ tbz x1, #48, 3b // KOI ASID shall be odd. -+ mov x11, x1 -+6: -+ msr ttbr1_el1, x11 -+ isb -+ stp x15, x13, [sp, #-32]! -+ b iee_rwx_gate_exit -+#endif -+SYM_FUNC_END(iee_rwx_gate_tramp) -+ -+ .popsection -+ -+ .pushsection ".iee.exec_exit", "ax" -+ -+SYM_FUNC_START(iee_rwx_gate_exit) -+ ldp x15, x13, [sp], #32 -+ mov sp, x13 // switch to kernel stack -+ mrs x9, tcr_el1 -+ orr x9, x9, #SYS_TCR_EL1_HPD1 -+ orr x9, x9, #SYS_TCR_EL1_A1 -+ msr tcr_el1, x9 -+/* --------Page boundary-------- */ -+ isb -+ msr daif, x15 -+ isb -+ ret -+SYM_FUNC_END(iee_rwx_gate_exit) ++extern int early_ioremap_debug __initdata; ++extern int after_paging_init __initdata; ++extern void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; ++extern unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; ++extern unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; + -+ .popsection ++extern bool pgattr_change_is_safe(u64 old, u64 new); + +#endif -diff --git a/arch/arm64/kernel/iee/iee-pgtable.c b/arch/arm64/kernel/iee/iee-pgtable.c +diff --git a/arch/arm64/include/asm/haoc/iee-selinux.h b/arch/arm64/include/asm/haoc/iee-selinux.h new file mode 100644 -index 000000000000..1369f00f410d +index 000000000000..875c685235c7 --- /dev/null -+++ b/arch/arm64/kernel/iee/iee-pgtable.c -@@ -0,0 +1,402 @@ -+#include -+#include -+#include -+#include ++++ b/arch/arm64/include/asm/haoc/iee-selinux.h +@@ -0,0 +1,53 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_SELINUX_P_H ++#define _LINUX_IEE_SELINUX_P_H + -+#ifdef CONFIG_PTP ++#include ++#include ++#include "security.h" ++#include "ss/services.h" ++ ++extern struct kmem_cache *policy_jar; ++extern unsigned long long iee_rw_gate(int flag, ...); + -+void iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++static inline struct mutex *iee_get_selinux_policy_lock(void) +{ -+ iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd); ++ return (struct mutex *)(selinux_state.policy_mutex.owner.counter); +} + -+inline void iee_set_bm_pte(pte_t *ptep, pte_t pte) ++static inline struct mutex *iee_get_selinux_status_lock(void) +{ -+ // If it is pre init, write once. -+ // Else, write once will cause exception. So it is safe. -+ unsigned long flags; -+ unsigned long res; -+ local_irq_save(flags); -+ asm volatile("at s1e1r, %0"::"r"(__phys_to_iee(__pa_symbol(ptep)))); -+ isb(); -+ res = read_sysreg(par_el1); -+ local_irq_restore(flags); -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+#endif -+ if(res & 0x1) -+ WRITE_ONCE(*ptep,pte); -+ else -+ iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte); -+ -+ /* -+ * Only if the new pte is valid and kernel, otherwise TLB maintenance -+ * or update_mmu_cache() have the necessary barriers. -+ */ -+ if (pte_valid_not_user(pte)) { -+ dsb(ishst); -+ isb(); -+ } ++ return (struct mutex *)(selinux_state.status_lock.owner.counter); +} + -+inline void iee_set_fixmap_pte_pre_init(pte_t *ptep, pte_t pte) ++static inline void iee_set_selinux_status_pg(struct page *new_page) +{ -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) { -+ pte = __pte(pte_val(pte) | PTE_NG); -+ } -+#endif -+ WRITE_ONCE(*ptep, pte); -+ -+ /* -+ * Only if the new pte is valid and kernel, otherwise TLB maintenance -+ * or update_mmu_cache() have the necessary barriers. -+ */ -+ if (pte_valid_not_user(pte)) { -+ dsb(ishst); -+ isb(); -+ } ++ iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); +} + -+inline void iee_set_pte_at(struct mm_struct *mm, unsigned long addr, -+ pte_t *ptep, pte_t pte) ++static inline void enforcing_set(bool value) +{ -+ __sync_cache_and_tags(pte, 1); -+ __check_safe_pte_update(mm, ptep, pte); -+ -+ iee_set_pte_pre_init(ptep, pte); ++ iee_rw_gate(IEE_SEL_SET_ENFORCING, value); +} + -+inline void iee_set_pte_at_delayed(struct mm_struct *mm, unsigned long addr, -+ pte_t *ptep, pte_t pte) ++static inline void selinux_mark_initialized(void) +{ -+ __sync_cache_and_tags(pte, 1); -+ __check_safe_pte_update(mm, ptep, pte); ++ iee_rw_gate(IEE_SEL_SET_INITIALIZED); +} + -+inline bool in_tramp_pgdir(void *addr) ++static inline void iee_set_sel_policy_cap(unsigned int idx, int cap) +{ -+ return ((unsigned long)addr & PAGE_MASK) == -+ ((unsigned long)tramp_pg_dir & PAGE_MASK); ++ iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); +} + -+inline void iee_set_fixmap_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) ++/* ++ * Please make sure param iee_new_policy is from policy_jar memcache. ++ * Need to free new_policy after calling this func as it's only used to ++ * trans data from kernel. ++ */ ++static inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, ++ struct selinux_policy *iee_new_policy) +{ -+#ifdef __PAGETABLE_PMD_FOLDED -+ if (in_swapper_pgdir(pmdp)) { -+ set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); -+ return; -+ } -+#endif /* __PAGETABLE_PMD_FOLDED */ -+#ifdef CONFIG_KOI -+ pmdval_t val = pmd_val(pmd); -+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { -+ pmd = __pmd(val | PMD_SECT_NG); -+ } ++ iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); ++} +#endif -+ WRITE_ONCE(*pmdp, pmd); +diff --git a/arch/arm64/include/asm/haoc/iee-si.h b/arch/arm64/include/asm/haoc/iee-si.h +new file mode 100644 +index 000000000000..cebbb6aa9d01 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-si.h +@@ -0,0 +1,45 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_SI_H ++#define _LINUX_IEE_SI_H + -+ if (pmd_valid(pmd)) { -+ dsb(ishst); -+ isb(); -+ } -+} ++#include + -+inline void iee_set_fixmap_pud_pre_init(pud_t *pudp, pud_t pud) -+{ -+#ifdef __PAGETABLE_PUD_FOLDED -+ if (in_swapper_pgdir(pudp)) { -+ set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); -+ return; -+ } -+#endif /* __PAGETABLE_PUD_FOLDED */ -+#ifdef CONFIG_KOI -+ pudval_t val = pud_val(pud); -+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { -+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. -+ pud = __pud(val | PMD_SECT_NG); -+ } -+#endif -+ WRITE_ONCE(*pudp, pud); ++#define __iee_si_code __section(".iee.si_text") + -+ if (pud_valid(pud)) { -+ dsb(ishst); -+ isb(); -+ } -+} ++enum { ++ IEE_SI_TEST, ++ IEE_SI_SET_SCTLR_EL1, ++ IEE_SI_SET_TCR_EL1, ++ IEE_SI_SET_VBAR, ++ IEE_SI_SET_TTBR0, ++ IEE_SI_CONTEXT_SWITCH, ++ IEE_SI_FLAGS ++}; ++ ++/* Handler function for sensitive instruction. */ ++extern unsigned long iee_rwx_gate(int flag, ...); ++ ++/* Replace sysreg_clear_set in sysreg.h. */ ++#define iee_si_sysreg_clear_set(sysreg, clear, set) do { \ ++ u64 __scs_val = read_sysreg(sysreg); \ ++ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \ ++ if (__scs_new != __scs_val) \ ++ iee_rwx_gate(IEE_SI_SET_##sysreg, __scs_new); \ ++} while (0) ++ ++#define IEE_SI_SCTLR_MASK (SCTLR_EL1_CP15BEN | SCTLR_EL1_SED | SCTLR_EL1_UCT | \ ++ SCTLR_EL1_UCI | SCTLR_EL1_BT0 | SCTLR_EL1_BT1 | SCTLR_EL1_TCF0_MASK | \ ++ SCTLR_ELx_DSSBS | SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA |\ ++ SCTLR_ELx_ENDB | SCTLR_EL1_SPINTMASK | SCTLR_EL1_NMI | SCTLR_EL1_TIDCP |\ ++ SCTLR_EL1_MSCEn | SCTLR_ELx_ENTP2 | SCTLR_EL1_TCF_MASK) + -+#endif //#ifdef CONFIG_PTP ++extern u64 __iee_si_text_start[]; ++extern u64 __iee_si_text_end[]; ++extern u64 iee_si_reserved_pg_dir; + -+inline void __set_pte(pte_t *ptep, pte_t pte) ++static inline void iee_si_setup_data(void) +{ -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) { -+ pte = __pte(pte_val(pte) | PTE_NG); -+ } ++ iee_si_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); ++} ++ +#endif -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); -+ dsb(ishst); -+ isb(); -+#else -+ WRITE_ONCE(*ptep, pte); -+ if (pte_valid_not_user(pte)) { -+ dsb(ishst); -+ isb(); -+ } +diff --git a/arch/arm64/include/asm/haoc/iee-token.h b/arch/arm64/include/asm/haoc/iee-token.h +new file mode 100644 +index 000000000000..6a3d733d103d +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-token.h +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_TOKEN_H ++#define _LINUX_IEE_TOKEN_H ++ ++#include ++#ifdef CONFIG_CREDP ++#include +#endif -+} + -+inline void set_pmd(pmd_t *pmdp, pmd_t pmd) ++extern struct kmem_cache *task_struct_cachep; ++ ++extern void __init iee_prepare_init_task_token(void); ++extern void iee_set_token_page_valid(unsigned long token, unsigned long new, ++ unsigned int order); ++extern void iee_set_token_page_invalid(unsigned long token_addr, ++ unsigned long token_page, unsigned int order); ++extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, ++ struct slab *slab, unsigned int order); ++ ++struct task_token { ++ pgd_t *pgd; /* Logical VA */ ++ bool valid; ++#ifdef CONFIG_CREDP ++ struct cred *new_cred; /* The copied cred by prepare_cred. */ ++ struct cred *curr_cred; /* A backup cred pointer for each process. */ ++#endif ++}; ++ ++#ifndef CONFIG_IEE_SIP ++#include ++static inline void iee_verify_token_pgd(struct task_struct *tsk) +{ -+#ifdef __PAGETABLE_PMD_FOLDED -+ if (in_swapper_pgdir(pmdp)) { -+ set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); ++ struct task_token *token; ++ ++ if (tsk == &init_task) + return; -+ } -+#endif /* __PAGETABLE_PMD_FOLDED */ -+#ifdef CONFIG_KOI -+ pmdval_t val = pmd_val(pmd); -+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { -+ pmd = __pmd(val | PMD_SECT_NG); -+ } -+#endif -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); ++ ++ token = (struct task_token *)__addr_to_iee(tsk); ++ if (token->pgd != tsk->mm->pgd) ++ panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", ++ (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); ++} +#else -+ WRITE_ONCE(*pmdp, pmd); ++static inline void iee_verify_token_pgd(struct task_struct *tsk) ++{ ++ ++} +#endif -+ if (pmd_valid(pmd)) { -+ dsb(ishst); -+ isb(); -+ } ++ ++static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) ++{ ++ iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); +} + -+inline void set_pud(pud_t *pudp, pud_t pud) ++static inline void iee_init_token(struct task_struct *tsk) +{ -+#ifdef __PAGETABLE_PUD_FOLDED -+ if (in_swapper_pgdir(pudp)) { -+ set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); -+ return; -+ } -+#endif /* __PAGETABLE_PUD_FOLDED */ -+#ifdef CONFIG_KOI -+ pudval_t val = pud_val(pud); -+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { -+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. -+ pud = __pud(val | PMD_SECT_NG); -+ } -+#endif -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); -+#else -+ WRITE_ONCE(*pudp, pud); -+#endif -+ if (pud_valid(pud)) { -+ dsb(ishst); -+ isb(); -+ } ++ iee_rw_gate(IEE_OP_INIT_TOKEN, tsk); +} + -+inline void set_p4d(p4d_t *p4dp, p4d_t p4d) ++static inline void iee_invalidate_token(struct task_struct *tsk) +{ -+ if (in_swapper_pgdir(p4dp)) -+ { -+ set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); -+ return; -+ } ++ iee_rw_gate(IEE_OP_INVALIDATE_TOKEN, tsk); ++} + -+#ifdef CONFIG_PTP -+ if(in_tramp_pgdir(p4dp)) -+ { -+ iee_set_tramp_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); -+ return; -+ } -+ iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); ++static inline void iee_validate_token(struct task_struct *tsk) ++{ ++ iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); ++} ++ ++#endif +diff --git a/arch/arm64/include/asm/haoc/iee-vmlinux.h b/arch/arm64/include/asm/haoc/iee-vmlinux.h +new file mode 100644 +index 000000000000..1cdb6d269539 +--- /dev/null ++++ b/arch/arm64/include/asm/haoc/iee-vmlinux.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_VMLINUX_H ++#define _LINUX_IEE_VMLINUX_H ++ ++#ifdef CONFIG_IEE_SIP ++#define IEE_SI_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ . += PMD_SIZE; \ ++ __iee_si_text_start = .; \ ++ *(.iee.si_text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_text_end = .; \ ++ . += PMD_SIZE; +#else -+ WRITE_ONCE(*p4dp, p4d); ++#define IEE_SI_TEXT +#endif -+ dsb(ishst); -+ isb(); -+} + +#ifdef CONFIG_IEE -+inline void iee_set_stack_pte(pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr) -+{ ++#define IEE_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_code_start = .; \ ++ *(.iee.text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_code_end = .; \ ++ IEE_SI_TEXT ++#else ++#define IEE_TEXT ++#endif ++ +#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PTE_U, ptep, order, use_block_pmd, lm_addr); ++#define PTP_DATA \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_ptp_data_start = .; \ ++ *(.iee.ptp) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_ptp_data_end = .; +#else -+ int i; -+ unsigned long iee_addr = lm_addr + iee_offset; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, iee_addr); -+ pud_t *pudp = pud_offset(p4dp, iee_addr); -+ pmd_t *pmdp = pmd_offset(pudp, iee_addr); -+ pte_t *iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } ++#define PTP_DATA ++#endif + -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) & ~PTE_VALID); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*ptep, pte); -+ ptep++; -+ } ++#ifdef CONFIG_IEE ++#define IEE_INIT_DATA \ ++ .iee.data : { \ ++ . = ALIGN(PAGE_SIZE); \ ++ iee_init_data_begin = .; \ ++ PTP_DATA \ ++ . = ALIGN(PAGE_SIZE); \ ++ iee_init_data_end = .; \ + } -+#endif -+ dsb(ishst); -+ isb(); -+} -+ -+inline void iee_unset_stack_pte(pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr) -+{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PTE_P, ptep, order, use_block_pmd, lm_addr); -+#else -+ int i; -+ unsigned long iee_addr = lm_addr + iee_offset; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, iee_addr); -+ pud_t *pudp = pud_offset(p4dp, iee_addr); -+ pmd_t *pmdp = pmd_offset(pudp, iee_addr); -+ pte_t *iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | PTE_VALID); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ #ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+ #endif -+ WRITE_ONCE(*ptep, pte); -+ ptep++; -+ } -+ } -+#endif -+ dsb(ishst); -+ isb(); -+} -+ -+inline void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) -+{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_SET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, use_block_pmd); +#else -+ int i; -+ if(use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ #ifdef CONFIG_KOI -+ pte = __pte(pte_val(pte) | PTE_VALID | PTE_NG); -+ #else -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ #endif -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ #ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+ #endif -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ #ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+ #endif -+ WRITE_ONCE(*iee_ptep, pte); -+ lm_ptep++; -+ iee_ptep++; -+ } -+ } ++#define IEE_INIT_DATA +#endif -+ dsb(ishst); -+ isb(); -+} + -+inline void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) -+{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_UNSET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, use_block_pmd); -+#else -+ int i; -+ if(use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd(pmd_val(pmd) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ lm_ptep++; -+ iee_ptep++; -+ } -+ } +#endif -+ dsb(ishst); -+ isb(); -+} -+#endif //#ifdef CONFIG_IEE -diff --git a/arch/arm64/kernel/iee/iee-selinuxp.c b/arch/arm64/kernel/iee/iee-selinuxp.c +diff --git a/arch/arm64/include/asm/haoc/iee.h b/arch/arm64/include/asm/haoc/iee.h new file mode 100644 -index 000000000000..394f99f3f574 +index 000000000000..9eaa15e98758 --- /dev/null -+++ b/arch/arm64/kernel/iee/iee-selinuxp.c -@@ -0,0 +1,36 @@ -+#include ++++ b/arch/arm64/include/asm/haoc/iee.h +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_H ++#define _LINUX_IEE_H + -+extern unsigned long long iee_rw_gate(int flag, ...); ++#include ++#include ++#include + -+#ifdef CONFIG_IEE_SELINUX_P -+inline void iee_set_selinux_status_pg(struct page* new_page) -+{ -+ iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); -+} ++extern unsigned long iee_tcr; ++extern unsigned long kernel_tcr; ++extern bool iee_init_done; + -+inline void enforcing_set(bool value) -+{ -+ iee_rw_gate(IEE_SEL_SET_ENFORCING, value); -+} ++#define IEE_OFFSET 0x400000000000 ++#define IEE_DATA_ORDER (PMD_SHIFT - PAGE_SHIFT) ++ ++#define __phys_to_iee(x) (__phys_to_virt(x) | IEE_OFFSET) ++#define __virt_to_iee(x) (((u64)x) | IEE_OFFSET) ++#define __kimg_to_iee(x) (__phys_to_iee(__pa_symbol(x))) ++#define __page_to_iee(x) (__phys_to_iee(page_to_phys(x))) ++#define __slab_to_iee(x) (__page_to_iee(folio_page(slab_folio(x), 0))) ++ ++#define __iee_to_virt(x) (((u64)x) & ~IEE_OFFSET) ++#define __iee_to_phys(x) (__pa(__iee_to_virt(x))) ++ ++/* Support conversion from both kernel and linear addresses. */ ++#define __ptr_to_iee(x) ({ \ ++ typeof(x) __val; \ ++ if (__is_lm_address((u64)x)) \ ++ __val = ((typeof(x))(__virt_to_iee((u64)x))); \ ++ else \ ++ __val = ((typeof(x))(__kimg_to_iee((u64)x))); \ ++ __val; \ ++}) + -+inline void selinux_mark_initialized(void) -+{ -+ iee_rw_gate(IEE_SEL_SET_INITIALIZED); -+} ++#define __addr_to_iee(x) ({ \ ++ u64 __val; \ ++ if (__is_lm_address((u64)x)) \ ++ __val = __virt_to_iee((u64)x); \ ++ else \ ++ __val = __kimg_to_iee((u64)x); \ ++ __val; \ ++}) + -+inline void iee_set_sel_policy_cap(unsigned int idx, int cap) -+{ -+ iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); -+} ++#define SET_UPAGE(x) __pgprot(pgprot_val(x) | PTE_USER) ++#define SET_PPAGE(x) __pgprot(pgprot_val(x) & (~PTE_USER)) ++#define SET_INVALID(x) __pgprot(pgprot_val(x) & (~PTE_VALID)) ++#define SET_NG(x) __pgprot(pgprot_val(x) | PTE_NG) + -+/* -+ * Please make sure param iee_new_policy is from policy_jar memcache. -+ * Need to free new_policy after calling this func as it's only used to -+ * trans data from kernel. -+ */ -+inline void iee_sel_rcu_assign_policy(struct selinux_policy* new_policy, -+ struct selinux_policy* iee_new_policy) -+{ -+ iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); -+} -+#endif -diff --git a/arch/arm64/kernel/iee/iee.c b/arch/arm64/kernel/iee/iee.c -new file mode 100644 -index 000000000000..1464c2e947e7 ---- /dev/null -+++ b/arch/arm64/kernel/iee/iee.c -@@ -0,0 +1,1720 @@ -+#include "linux/sched.h" -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++#define PGD_APTABLE_RO (_AT(pudval_t, 1) << 62) ++#define PGD_APTABLE (_AT(pudval_t, 1) << 61) ++#define PGD_PXNTABLE (_AT(pudval_t, 1) << 59) ++#define PGD_UXNTABLE (_AT(pudval_t, 1) << 60) + -+#define is_cow_mapping(flags) (((flags) & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE) ++#define TCR_HPD1 (UL(1) << 42) + -+#ifdef CONFIG_IEE -+extern struct cred init_cred; -+extern s64 memstart_addr; -+extern unsigned long highest_memmap_pfn; ++void iee_init_mappings(pgd_t *pgdp); ++void iee_init_post(void); ++void iee_stack_init(void); ++void iee_init_tcr(void); ++void iee_print_pgtables(u64 addr); + -+struct watch_list; ++#define IEE_STACK_ORDER 0x3 ++#define IEE_STACK_SIZE (PAGE_SIZE << IEE_STACK_ORDER) + -+void __iee_code _iee_set_swapper_pgd(unsigned long iee_offset, pgd_t *pgdp, pgd_t pgd); -+void __iee_code _iee_set_tramp_pgd(unsigned long iee_offset, pgd_t *pgdp, pgd_t pgd); -+void __iee_code _iee_set_pte(unsigned long iee_offset, pte_t *ptep, pte_t pte); -+void __iee_code _iee_set_pmd(unsigned long iee_offset, pmd_t *pmdp, pmd_t pmd); -+void __iee_code _iee_set_pud(unsigned long iee_offset, pud_t *pudp, pud_t pud); -+void __iee_code _iee_set_p4d(unsigned long iee_offset, p4d_t *p4dp, p4d_t p4d); -+void __iee_code _iee_set_bm_pte(unsigned long iee_offset, pte_t *ptep, pte_t pte); -+void __iee_code _iee_write_in_byte(unsigned long iee_offset, void *ptr, __u64 data, int length); -+void __iee_code _iee_set_cred_uid(unsigned long iee_offset, struct cred *cred, kuid_t uid); -+void __iee_code _iee_set_cred_gid(unsigned long iee_offset, struct cred *cred, kgid_t gid); -+void __iee_code _iee_copy_cred(unsigned long iee_offset, struct cred *old, struct cred *new); -+void __iee_code _iee_set_cred_suid(unsigned long iee_offset, struct cred *cred, kuid_t suid); -+void __iee_code _iee_set_cred_sgid(unsigned long iee_offset, struct cred *cred, kgid_t sgid); -+void __iee_code _iee_set_cred_euid(unsigned long iee_offset, struct cred *cred, kuid_t euid); -+void __iee_code _iee_set_cred_egid(unsigned long iee_offset, struct cred *cred, kgid_t egid); -+void __iee_code _iee_set_cred_fsuid(unsigned long iee_offset, struct cred *cred, kuid_t fsuid); -+void __iee_code _iee_set_cred_fsgid(unsigned long iee_offset, struct cred *cred, kgid_t fsgid); -+void __iee_code _iee_set_cred_user(unsigned long iee_offset, struct cred *cred, struct user_struct *user); -+void __iee_code _iee_set_cred_user_ns(unsigned long iee_offset, struct cred *cred, struct user_namespace *user_ns); -+void __iee_code _iee_set_cred_group_info(unsigned long iee_offset, struct cred *cred, struct group_info *group_info); -+void __iee_code _iee_set_cred_securebits(unsigned long iee_offset, struct cred *cred, unsigned securebits); -+void __iee_code _iee_set_cred_cap_inheritable(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_inheritable); -+void __iee_code _iee_set_cred_cap_permitted(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_permitted); -+void __iee_code _iee_set_cred_cap_effective(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_effective); -+void __iee_code _iee_set_cred_cap_bset(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_bset); -+void __iee_code _iee_set_cred_cap_ambient(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_ambient); -+void __iee_code _iee_set_cred_jit_keyring(unsigned long iee_offset, struct cred *cred, unsigned char jit_keyring); -+void __iee_code _iee_set_cred_session_keyring(unsigned long iee_offset, struct cred *cred, struct key *session_keyring); -+void __iee_code _iee_set_cred_process_keyring(unsigned long iee_offset, struct cred *cred, struct key *process_keyring); -+void __iee_code _iee_set_cred_thread_keyring(unsigned long iee_offset, struct cred *cred, struct key *thread_keyring); -+void __iee_code _iee_set_cred_request_key_auth(unsigned long iee_offset, struct cred *cred, struct key *request_key_auth); -+void __iee_code _iee_set_cred_non_rcu(unsigned long iee_offset, struct cred *cred, int non_rcu); -+void __iee_code _iee_set_cred_atomic_set_usage(unsigned long iee_offset, struct cred *cred, int i); -+bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long iee_offset, struct cred *cred, int flag, int nr); -+void __iee_code _iee_set_cred_security(unsigned long iee_offset, struct cred *cred, void *security); -+void __iee_code _iee_set_cred_rcu(unsigned long iee_offset, struct cred *cred, struct rcu_head *rcu); -+void __iee_code _iee_memset(unsigned long iee_offset, void *ptr, int data, size_t n); -+void __iee_code _iee_set_track(unsigned long iee_offset, struct track *ptr, struct track *data); -+void __iee_code _iee_set_freeptr(unsigned long iee_offset, void **pptr, void *ptr); -+void __iee_code _iee_set_stack_pte(unsigned long iee_offset, pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr); -+void __iee_code _iee_unset_stack_pte(unsigned long iee_offset, pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr); -+void __iee_code _iee_set_token_pgd(unsigned long iee_offset, struct task_struct *tsk, pgd_t *pgd); -+void __iee_code _iee_init_token(unsigned long iee_offset, struct task_struct *tsk, void *iee_stack, void *tmp_page); -+void __iee_code _iee_invalidate_token(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_validate_token(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_set_sensitive_pte(unsigned long iee_offset, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); -+void __iee_code _iee_unset_sensitive_pte(unsigned long iee_offset, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd); -+void __iee_code _iee_set_token(unsigned long iee_offset, pte_t *ptep, void *new, unsigned long order, int use_block_pmd); -+void __iee_code _iee_unset_token(unsigned long iee_offset, pte_t *ptep, void *token_addr, void *token_page, unsigned long order); -+void __iee_code _iee_copy_pte_range(unsigned long iee_offset, pte_t *new_dst, pte_t *old_dst, pte_t *src_pte, struct vm_area_struct *src_vma, unsigned long dst_vm_flags, pte_t *end_pte); -+void __iee_code _iee_split_huge_pmd(unsigned long iee_offset, pmd_t *pmdp, pte_t *pgtable); -+void __iee_code _iee_set_cred_ucounts(unsigned long iee_offset, struct cred *cred, struct ucounts *ucounts); -+void __iee_code _iee_set_key_union(unsigned long iee_offset, struct key *key, struct key_union *key_union); -+void __iee_code _iee_set_key_struct(unsigned long iee_offset, struct key *key, struct key_struct *key_struct); -+void __iee_code _iee_set_key_payload(unsigned long iee_offset, struct key *key, union key_payload *key_payload); -+void __iee_code _iee_memcpy(unsigned long iee_offset, void *dst, void *src, size_t n); -+bool __iee_code _iee_set_key_usage(unsigned long iee_offset, struct key *key, int n, int flag); -+void __iee_code _iee_set_key_serial(unsigned long iee_offset, struct key *key, key_serial_t serial); -+void __iee_code _iee_set_key_watchers(unsigned long iee_offset, struct key *key, struct watch_list *watchers); -+void __iee_code _iee_set_key_user(unsigned long iee_offset, struct key *key, struct key_user *user); -+void __iee_code _iee_set_key_security(unsigned long iee_offset, struct key *key, void *security); -+void __iee_code _iee_set_key_expiry(unsigned long iee_offset, struct key *key, time64_t expiry); -+void __iee_code _iee_set_key_revoked_at(unsigned long iee_offset, struct key *key, time64_t revoked_at); -+void __iee_code _iee_set_key_last_used_at(unsigned long iee_offset, struct key *key, time64_t last_used_at); -+void __iee_code _iee_set_key_uid(unsigned long iee_offset, struct key *key, kuid_t uid); -+void __iee_code _iee_set_key_gid(unsigned long iee_offset, struct key *key, kgid_t gid); -+void __iee_code _iee_set_key_perm(unsigned long iee_offset, struct key *key, key_perm_t perm); -+void __iee_code _iee_set_key_quotalen(unsigned long iee_offset, struct key *key, unsigned short quotalen); -+void __iee_code _iee_set_key_datalen(unsigned long iee_offset, struct key *key, unsigned short datalen); -+void __iee_code _iee_set_key_state(unsigned long iee_offset, struct key *key, short state); -+void __iee_code _iee_set_key_magic(unsigned long iee_offset, struct key *key, unsigned magic); -+void __iee_code _iee_set_key_flags(unsigned long iee_offset, struct key *key, unsigned long flags); -+void __iee_code _iee_set_key_index_key(unsigned long iee_offset, struct key *key, struct keyring_index_key *index_key); -+void __iee_code _iee_set_key_hash(unsigned long iee_offset, struct key *key, unsigned long hash); -+void __iee_code _iee_set_key_len_desc(unsigned long iee_offset, struct key *key, unsigned long len_desc); -+void __iee_code _iee_set_key_type(unsigned long iee_offset, struct key *key, struct key_type *type); -+void __iee_code _iee_set_key_domain_tag(unsigned long iee_offset, struct key *key, struct key_tag *domain_tag); -+void __iee_code _iee_set_key_description(unsigned long iee_offset, struct key *key, char *description); -+void __iee_code _iee_set_key_restrict_link(unsigned long iee_offset, struct key *key, struct key_restriction *restrict_link); -+bool __iee_code _iee_set_key_flag_bit(unsigned long iee_offset, struct key *key, long nr, int flag); -+#ifdef CONFIG_KOI -+unsigned long __iee_code _iee_read_koi_stack(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack); -+unsigned long __iee_code _iee_read_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk, unsigned long current_ttbr1); -+unsigned long __iee_code _iee_read_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long kernel_stack); -+unsigned long __iee_code _iee_read_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack_base); -+void __iee_code _iee_set_koi_pgd(unsigned long iee_offset, unsigned long koi_pgd_addr); -+#endif ++#define IEE_CHECK(condition) do { \ ++ if (unlikely(condition)) \ ++ panic("IEE check failed on %s.", __func__); \ ++} while (0) + -+#ifdef CONFIG_IEE_SELINUX_P -+#include -+void __iee_code _iee_set_selinux_status_pg(unsigned long iee_offset, struct page* new_page); -+void __iee_code _iee_set_selinux_enforcing(unsigned long iee_offset, bool value); -+void __iee_code _iee_mark_selinux_initialized(unsigned long iee_offset); -+void __iee_code _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap); -+void __iee_code _iee_sel_rcu_assign_policy(unsigned long iee_offset, -+ struct selinux_policy* new_policy, struct selinux_policy* iee_new_policy); -+#endif -+ -+static void inline _iee_set_pte_single(pte_t *ptep, pte_t pte, unsigned long iee_offset); -+static pteval_t inline _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval, unsigned long iee_offset); -+ -+// Define the function pointer type for wrapper functions. -+// Each function pointer conforms to a standardized calling convention -+// using a variable argument list (va_list) as its parameter. -+// This allows dynamic invocation of different functions with various arguments. -+typedef void (*iee_func)(void); -+iee_func iee_funcs[] = { -+ (iee_func)_iee_write_in_byte, -+ (iee_func)_iee_set_pte, -+ (iee_func)_iee_set_pmd, -+ (iee_func)_iee_set_pud, -+ (iee_func)_iee_set_p4d, -+ (iee_func)_iee_set_bm_pte, -+ (iee_func)_iee_set_swapper_pgd, -+ (iee_func)_iee_set_tramp_pgd, -+ (iee_func)_iee_memset, -+ (iee_func)_iee_set_track, -+ (iee_func)_iee_set_freeptr, -+ (iee_func)_iee_set_stack_pte, -+ (iee_func)_iee_unset_stack_pte, -+ (iee_func)_iee_set_token_pgd, -+ (iee_func)_iee_init_token, -+ (iee_func)_iee_invalidate_token, -+ (iee_func)_iee_set_sensitive_pte, -+ (iee_func)_iee_unset_sensitive_pte, -+ (iee_func)_iee_set_token, -+ (iee_func)_iee_unset_token, -+ (iee_func)_iee_copy_pte_range, -+ (iee_func)_iee_split_huge_pmd, -+ (iee_func)_iee_validate_token, -+ (iee_func)_iee_memcpy, -+#ifdef CONFIG_KOI -+ (iee_func)_iee_read_koi_stack, -+ (iee_func)_iee_write_koi_stack, -+ (iee_func)_iee_read_token_ttbr1, -+ (iee_func)_iee_write_token_ttbr1, -+ (iee_func)_iee_read_koi_kernel_stack, -+ (iee_func)_iee_write_koi_kernel_stack, -+ (iee_func)_iee_read_koi_stack_base, -+ (iee_func)_iee_write_koi_stack_base, -+ (iee_func)_iee_set_koi_pgd, +#endif -+#ifdef CONFIG_CREDP -+ (iee_func)_iee_copy_cred, -+ (iee_func)_iee_set_cred_uid, -+ (iee_func)_iee_set_cred_gid, -+ (iee_func)_iee_set_cred_suid, -+ (iee_func)_iee_set_cred_sgid, -+ (iee_func)_iee_set_cred_euid, -+ (iee_func)_iee_set_cred_egid, -+ (iee_func)_iee_set_cred_fsuid, -+ (iee_func)_iee_set_cred_fsgid, -+ (iee_func)_iee_set_cred_user, -+ (iee_func)_iee_set_cred_user_ns, -+ (iee_func)_iee_set_cred_group_info, -+ (iee_func)_iee_set_cred_securebits, -+ (iee_func)_iee_set_cred_cap_inheritable, -+ (iee_func)_iee_set_cred_cap_permitted, -+ (iee_func)_iee_set_cred_cap_effective, -+ (iee_func)_iee_set_cred_cap_bset, -+ (iee_func)_iee_set_cred_cap_ambient, -+ (iee_func)_iee_set_cred_jit_keyring, -+ (iee_func)_iee_set_cred_session_keyring, -+ (iee_func)_iee_set_cred_process_keyring, -+ (iee_func)_iee_set_cred_thread_keyring, -+ (iee_func)_iee_set_cred_request_key_auth, -+ (iee_func)_iee_set_cred_non_rcu, -+ (iee_func)_iee_set_cred_atomic_set_usage, -+ (iee_func)_iee_set_cred_atomic_op_usage, -+ (iee_func)_iee_set_cred_security, -+ (iee_func)_iee_set_cred_rcu, -+ (iee_func)_iee_set_cred_ucounts, +diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h +index fde4186cc387..d6a43c8e9722 100644 +--- a/arch/arm64/include/asm/memory.h ++++ b/arch/arm64/include/asm/memory.h +@@ -77,7 +77,11 @@ + #define KASAN_THREAD_SHIFT 1 + #else + #define KASAN_THREAD_SHIFT 0 ++#ifdef CONFIG_IEE ++#define PAGE_END ((UL(1) << ((VA_BITS_MIN) - 2)) | PAGE_OFFSET) ++#else + #define PAGE_END (_PAGE_END(VA_BITS_MIN)) +#endif -+#ifdef CONFIG_KEYP -+ (iee_func)_iee_set_key_union, -+ (iee_func)_iee_set_key_struct, -+ (iee_func)_iee_set_key_payload, -+ (iee_func)_iee_set_key_usage, -+ (iee_func)_iee_set_key_serial, -+ (iee_func)_iee_set_key_watchers, -+ (iee_func)_iee_set_key_user, -+ (iee_func)_iee_set_key_security, -+ (iee_func)_iee_set_key_expiry, -+ (iee_func)_iee_set_key_revoked_at, -+ (iee_func)_iee_set_key_last_used_at, -+ (iee_func)_iee_set_key_uid, -+ (iee_func)_iee_set_key_gid, -+ (iee_func)_iee_set_key_perm, -+ (iee_func)_iee_set_key_quotalen, -+ (iee_func)_iee_set_key_datalen, -+ (iee_func)_iee_set_key_state, -+ (iee_func)_iee_set_key_magic, -+ (iee_func)_iee_set_key_flags, -+ (iee_func)_iee_set_key_index_key, -+ (iee_func)_iee_set_key_hash, -+ (iee_func)_iee_set_key_len_desc, -+ (iee_func)_iee_set_key_type, -+ (iee_func)_iee_set_key_domain_tag, -+ (iee_func)_iee_set_key_description, -+ (iee_func)_iee_set_key_restrict_link, -+ (iee_func)_iee_set_key_flag_bit, + #endif /* CONFIG_KASAN */ + + #define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) +diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h +index a6fb325424e7..0a954f2af9fa 100644 +--- a/arch/arm64/include/asm/mmu_context.h ++++ b/arch/arm64/include/asm/mmu_context.h +@@ -24,6 +24,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include +#endif -+#ifdef CONFIG_IEE_SELINUX_P -+ (iee_func)_iee_set_selinux_status_pg, -+ (iee_func)_iee_set_selinux_enforcing, -+ (iee_func)_iee_mark_selinux_initialized, -+ (iee_func)_iee_set_sel_policy_cap, -+ (iee_func)_iee_sel_rcu_assign_policy, + + extern bool rodata_full; + +@@ -43,7 +46,11 @@ static inline void cpu_set_reserved_ttbr0_nosync(void) + { + unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); + ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_SET_TTBR0, ttbr); ++#else + write_sysreg(ttbr, ttbr0_el1); +#endif -+ NULL -+}; -+ -+#ifdef CONFIG_KOI -+unsigned long __iee_code _iee_read_koi_stack(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ return (unsigned long)token->koi_stack; -+} -+ -+void __iee_code _iee_write_koi_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->koi_stack = (void *) koi_stack; -+} -+ -+unsigned long __iee_code _iee_read_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ return token->current_ttbr1; -+} + } + + static inline void cpu_set_reserved_ttbr0(void) +@@ -79,8 +86,12 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) + + tcr &= ~TCR_T0SZ_MASK; + tcr |= t0sz << TCR_T0SZ_OFFSET; ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_SET_TCR_EL1, tcr); ++#else + write_sysreg(tcr, tcr_el1); + isb(); ++#endif + } + + #define cpu_set_default_tcr_t0sz() __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)) +@@ -144,8 +155,12 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) + __cpu_set_tcr_t0sz(t0sz); + + /* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */ ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_SET_TTBR0, ttbr0); ++#else + write_sysreg(ttbr0, ttbr0_el1); + isb(); ++#endif + } + + /* +diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h +index f8afbcacac8f..c1c5888755c9 100644 +--- a/arch/arm64/include/asm/pgtable.h ++++ b/arch/arm64/include/asm/pgtable.h +@@ -5,6 +5,10 @@ + #ifndef __ASM_PGTABLE_H + #define __ASM_PGTABLE_H + ++#ifdef CONFIG_PTP ++#include ++#else + -+void __iee_code _iee_write_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk, unsigned long current_ttbr1) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->current_ttbr1 = current_ttbr1; -+} + #include + #include + +@@ -1560,4 +1564,6 @@ void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte); + + #endif /* !__ASSEMBLY__ */ + ++#endif /* !CONFIG_PTP */ + -+unsigned long __iee_code _iee_read_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ return (unsigned long) token->koi_kernel_stack; -+} -+ -+void __iee_code _iee_write_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long kernel_stack) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->koi_kernel_stack = (void *) kernel_stack; -+} -+ -+unsigned long __iee_code _iee_read_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ return (unsigned long)token->koi_stack_base; -+} -+ -+void __iee_code _iee_write_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack_base) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->koi_stack_base = (void *) koi_stack_base; -+} -+ -+static inline void iee_set_koi_pgd_writeable(unsigned long koi_pgd_addr, unsigned long iee_si_addr) -+{ -+ pgd_t *pgdir = (pgd_t *)koi_pgd_addr; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, iee_si_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, iee_si_addr); -+ pud_t *pudp = pud_offset(p4dp, iee_si_addr); -+ pmd_t *pmdp = pmd_offset(pudp, iee_si_addr); -+ pte_t *ptep = pte_offset_kernel(pmdp, iee_si_addr); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) | PTE_DBM); -+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); -+} -+/* -+ * Set IEE SI codes U RWX here to avoid IEE DEP checking fault. -+ * Mark koi pgd in the same time. -+ */ -+void __iee_code _iee_set_koi_pgd(unsigned long iee_offset, unsigned long koi_pgd_addr) -+{ -+ // IEE SI codes are 2 pages starting at __iee_si_text_start. -+ unsigned long iee_si_addr = (unsigned long)__iee_si_text_start; -+ iee_set_koi_pgd_writeable(koi_pgd_addr, iee_si_addr); -+ iee_si_addr += PAGE_SIZE; -+ iee_set_koi_pgd_writeable(koi_pgd_addr, iee_si_addr); -+ // Use DBM=0, AP[7]=0 to mark this page as a koi pgd in IEE. -+} -+#endif -+ -+#ifdef CONFIG_IEE_SELINUX_P -+void __iee_code _iee_set_selinux_status_pg(unsigned long iee_offset, struct page* new_page) -+{ -+ struct page** iee_addr = (struct page**)__phys_to_iee(__pa_symbol(&(selinux_state.status_page))); -+ *iee_addr = new_page; -+} -+ -+void __iee_code _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) -+{ -+ *(bool*)__phys_to_iee(__pa_symbol(&(selinux_state.enforcing))) = value; -+} -+ -+void __iee_code _iee_mark_selinux_initialized(unsigned long iee_offset) -+{ -+ smp_store_release(((bool*)__phys_to_iee(__pa_symbol(&(selinux_state.initialized)))), true); -+ printk("IEE: Mark selinux initialized."); -+} -+ -+void __iee_code _iee_set_sel_policy_cap(unsigned long iee_offset, unsigned int idx, int cap) -+{ -+ *(bool*)__phys_to_iee(__pa_symbol(&(selinux_state.policycap[idx]))) = cap; -+} -+ -+/* -+ * Please make sure param iee_new_policy is from policy_jar memcache. -+ * Need to free new_policy after calling this func as it's only used to -+ * trans data from kernel. -+ */ -+void __iee_code _iee_sel_rcu_assign_policy(unsigned long iee_offset, struct selinux_policy* new_policy, -+ struct selinux_policy* iee_new_policy) -+{ -+ /* TODO: Verify informations from incoming policy. */ -+ // /* Make sure iee_new_policy is from policy_jar memcache. */ -+ // struct slab* policy_pg = (struct slab*)pfn_to_page(__pa(iee_new_policy) >> PAGE_SHIFT); -+ // if (policy_pg->slab_cache != policy_jar) -+ // printk("IEE SELINUXP ERROR: new policy is not from iee memcache."); -+ /* Copy data from kernel to new allocated policy struct inside iee. */ -+ struct selinux_policy* iee_addr = (struct selinux_policy *)((unsigned long)iee_new_policy + iee_offset); -+ memcpy(iee_addr, new_policy, sizeof(struct selinux_policy)); -+ -+ rcu_assign_pointer(*((struct selinux_policy**)__phys_to_iee(__pa_symbol(&(selinux_state.policy)))), -+ iee_new_policy); -+ printk("IEE: assigned rcu pointer selinux_state.policy."); -+} -+#endif -+ -+bool __iee_code _iee_set_key_flag_bit(unsigned long iee_offset, struct key *key, long nr, int flag) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ switch(flag) -+ { -+ case SET_BIT_OP: -+ { -+ set_bit(nr, &key->flags); -+ break; -+ } -+ case TEST_AND_CLEAR_BIT: -+ { -+ return test_and_clear_bit(nr, &key->flags); -+ } -+ case TEST_AND_SET_BIT: -+ { -+ return test_and_set_bit(nr, &key->flags); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_key_restrict_link(unsigned long iee_offset, struct key *key, struct key_restriction *restrict_link) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->restrict_link = restrict_link; -+} -+ -+void __iee_code _iee_set_key_magic(unsigned long iee_offset, struct key *key, unsigned magic) -+{ -+ #ifdef KEY_DEBUGGING -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->magic = magic; -+ #endif -+} -+ -+void __iee_code _iee_set_key_flags(unsigned long iee_offset, struct key *key, unsigned long flags) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->flags = flags; -+} -+ -+void __iee_code _iee_set_key_index_key(unsigned long iee_offset, struct key *key, struct keyring_index_key *index_key) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->index_key = *index_key; -+} -+ -+void __iee_code _iee_set_key_hash(unsigned long iee_offset, struct key *key, unsigned long hash) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->hash = hash; -+} -+ -+void __iee_code _iee_set_key_len_desc(unsigned long iee_offset, struct key *key, unsigned long len_desc) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->len_desc = len_desc; -+} -+ -+void __iee_code _iee_set_key_type(unsigned long iee_offset, struct key *key, struct key_type *type) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->type = type; -+} -+ -+void __iee_code _iee_set_key_domain_tag(unsigned long iee_offset, struct key *key, struct key_tag *domain_tag) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->domain_tag = domain_tag; -+} -+ -+void __iee_code _iee_set_key_description(unsigned long iee_offset, struct key *key, char *description) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->description = description; -+} -+ -+void __iee_code _iee_set_key_uid(unsigned long iee_offset, struct key *key, kuid_t uid) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->uid = uid; -+} -+ -+void __iee_code _iee_set_key_gid(unsigned long iee_offset, struct key *key, kgid_t gid) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->gid = gid; -+} -+ -+void __iee_code _iee_set_key_perm(unsigned long iee_offset, struct key *key, key_perm_t perm) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->perm = perm; -+} -+ -+void __iee_code _iee_set_key_quotalen(unsigned long iee_offset, struct key *key, unsigned short quotalen) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->quotalen = quotalen; -+} -+ -+void __iee_code _iee_set_key_datalen(unsigned long iee_offset, struct key *key, unsigned short datalen) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->datalen = datalen; -+} -+ -+void __iee_code _iee_set_key_state(unsigned long iee_offset, struct key *key, short state) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ WRITE_ONCE(key->state, state); -+} -+ -+void __iee_code _iee_set_key_user(unsigned long iee_offset, struct key *key, struct key_user *user) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->user = user; -+} -+ -+void __iee_code _iee_set_key_security(unsigned long iee_offset, struct key *key, void *security) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->security = security; -+} -+ -+void __iee_code _iee_set_key_expiry(unsigned long iee_offset, struct key *key, time64_t expiry) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->expiry = expiry; -+} -+ -+void __iee_code _iee_set_key_revoked_at(unsigned long iee_offset, struct key *key, time64_t revoked_at) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->revoked_at = revoked_at; -+} -+ -+void __iee_code _iee_set_key_last_used_at(unsigned long iee_offset, struct key *key, time64_t last_used_at) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->last_used_at = last_used_at; -+} -+ -+bool __iee_code _iee_set_key_usage(unsigned long iee_offset, struct key *key, int n, int flag) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ switch(flag) -+ { -+ case REFCOUNT_INC: -+ { -+ refcount_inc(&key->usage); -+ break; -+ } -+ case REFCOUNT_SET: -+ { -+ refcount_set(&key->usage, n); -+ break; -+ } -+ case REFCOUNT_DEC_AND_TEST: -+ { -+ return refcount_dec_and_test(&key->usage); -+ } -+ case REFCOUNT_INC_NOT_ZERO: -+ { -+ return refcount_inc_not_zero(&key->usage); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_key_serial(unsigned long iee_offset, struct key *key, key_serial_t serial) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->serial = serial; -+} -+ -+void __iee_code _iee_set_key_watchers(unsigned long iee_offset, struct key *key, struct watch_list *watchers) -+{ -+ #ifdef CONFIG_KEY_NOTIFICATIONS -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->watchers = watchers; -+ #endif -+} -+ -+void __iee_code _iee_set_key_union(unsigned long iee_offset, struct key *key, struct key_union *key_union) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->graveyard_link.next = (struct list_head *)key_union; -+} -+ -+void __iee_code _iee_set_key_struct(unsigned long iee_offset, struct key *key, struct key_struct *key_struct) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->name_link.prev = (struct list_head *)key_struct; -+} -+ -+void __iee_code _iee_set_key_payload(unsigned long iee_offset, struct key *key, union key_payload *key_payload) -+{ -+ key = (struct key *)((unsigned long)key + iee_offset); -+ key->name_link.next = (struct list_head *)key_payload; -+} -+ -+void __iee_code _iee_split_huge_pmd(unsigned long iee_offset, pmd_t *pmdp, pte_t *pgtable) -+{ -+ int i; -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)((unsigned long)pgtable + (unsigned long)iee_offset); -+ -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = PAGE_KERNEL; -+ #ifdef CONFIG_KOI -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT | PTE_NG); -+ #else -+ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); -+ #endif -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+} -+ -+void __iee_code _iee_copy_pte_range(unsigned long iee_offset, pte_t *new_dst, pte_t *old_dst, pte_t *src_pte, struct vm_area_struct *src_vma, unsigned long dst_vm_flags, pte_t *end_pte) -+{ -+ pte_t pte, tmp; -+ swp_entry_t entry; -+ tmp = __pte(0); -+ while(src_pte < end_pte) -+ { -+ if (!pte_none(*src_pte) && unlikely(!pte_present(*src_pte))) -+ { -+ pte = *src_pte; -+ entry = pte_to_swp_entry(pte); -+ if (likely(!non_swap_entry(entry))) -+ ; -+ else if(is_migration_entry(entry)) -+ { -+ if (is_readable_migration_entry(entry) && -+ is_cow_mapping(dst_vm_flags)) -+ { -+ entry = make_readable_migration_entry(swp_offset(entry)); -+ pte = swp_entry_to_pte(entry); -+ if (pte_swp_soft_dirty(*src_pte)) -+ pte = pte_swp_mksoft_dirty(pte); -+ if (pte_swp_uffd_wp(*src_pte)) -+ pte = pte_swp_mkuffd_wp(pte); -+ _iee_set_pte_single(src_pte, pte, iee_offset); -+ } -+ } -+ else if (is_device_private_entry(entry)) -+ { -+ if (is_writable_device_private_entry(entry) && -+ is_cow_mapping(dst_vm_flags)) { -+ entry = make_readable_device_private_entry( -+ swp_offset(entry)); -+ pte = swp_entry_to_pte(entry); -+ if (pte_swp_uffd_wp(*src_pte)) -+ pte = pte_swp_mkuffd_wp(pte); -+ _iee_set_pte_single(src_pte, pte, iee_offset); -+ } -+ } -+ } -+ else if(!pte_none(*src_pte) && likely(pte_present(*src_pte))) -+ { -+ struct page *page = NULL; -+ #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL -+ if (likely(!pte_special(*src_pte))) -+ { -+ if (unlikely(pte_pfn(*src_pte) > highest_memmap_pfn)) -+ page = NULL; -+ else -+ page = pte_page(*src_pte); -+ } -+ #else -+ if((!unlikely(src_vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) || ((src_vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pte_pfn(*src_pte))) || (!(src_vma->vm_flags & VM_MIXEDMAP) && !(pfn == src_vma->vm_pgoff + off) && is_cow_mapping(src_vma->vm_flags))) && !is_zero_pfn(pte_pfn(*src_pte)) && !unlikely(pte_pfn(*src_pte) > highest_memmap_pfn)) -+ page = pte_page(*src_pte); -+ #endif -+ if(!(page && folio_test_anon(page_folio(page))) || likely(!PageAnonExclusive(page) || !(likely(!is_device_private_page(page) && unlikely(folio_needs_cow_for_dma(src_vma, page_folio(page))))))) -+ { -+ if (is_cow_mapping(src_vma->vm_flags) && pte_write(*src_pte)) -+ { -+ pte_t old_pte, new_pte; -+ -+ new_pte = READ_ONCE(*src_pte); -+ do { -+ old_pte = new_pte; -+ new_pte = pte_wrprotect(new_pte); -+ _iee_set_cmpxchg_relaxed(src_pte, pte_val(old_pte), pte_val(new_pte), iee_offset); -+ } while (pte_val(new_pte) != pte_val(old_pte)); -+ } -+ } -+ } -+ if(!pte_none(*new_dst)) -+ { -+ _iee_set_pte_single(old_dst, *new_dst, iee_offset); -+ WRITE_ONCE(*new_dst, __pte(0)); -+ } -+ old_dst++; -+ src_pte++; -+ new_dst++; -+ } -+} -+ -+void __iee_code _iee_set_sensitive_pte(unsigned long iee_offset, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) -+{ -+ int i; -+ -+ lm_ptep = (pte_t *)((unsigned long)lm_ptep + iee_offset); -+ iee_ptep = (pte_t *)((unsigned long)iee_ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ lm_ptep++; -+ iee_ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_unset_sensitive_pte(unsigned long iee_offset, pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) -+{ -+ int i; -+ -+ lm_ptep = (pte_t *)((unsigned long)lm_ptep + iee_offset); -+ iee_ptep = (pte_t *)((unsigned long)iee_ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd(pmd_val(pmd) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ lm_ptep++; -+ iee_ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_set_token(unsigned long iee_offset, pte_t *ptep, void *new, unsigned long order, int use_block_pmd) -+{ -+ int i; -+ pgd_t *pgdir; -+ pgd_t *pgdp; -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ -+ pgdir = swapper_pg_dir; -+ -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)new); -+ p4dp = p4d_offset(pgdp, (unsigned long)new); -+ pudp = pud_offset(p4dp, (unsigned long)new); -+ pmdp = pmd_offset(pudp, (unsigned long)new); -+ if(use_block_pmd) -+ lm_ptep = (pte_t *)pmdp; -+ else -+ lm_ptep = pte_offset_kernel(pmdp, (unsigned long)new); -+ // Handling cont mapping. -+ if(pte_val(*lm_ptep) & PTE_CONT) -+ { -+ // The beginning of cont mapping. -+ int i; -+ pte_t *ptep = (pte_t *)((unsigned long)pte_offset_kernel(pmdp, (unsigned long)new & CONT_PTE_MASK) + iee_offset); -+ if(order < CONFIG_ARM64_CONT_PTE_SHIFT) -+ { -+ for(i = 0; i < CONT_PTES; i++) -+ { -+ WRITE_ONCE(*ptep,__pte(pte_val(*ptep) & ~PTE_CONT)); -+ ptep++; -+ } -+ } -+ } -+ -+ iee_addr = ((unsigned long)new + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ -+ ptep = (pte_t *)((unsigned long)ptep + iee_offset); -+ lm_ptep = (pte_t *)((unsigned long)lm_ptep + iee_offset); -+ iee_ptep = (pte_t *)((unsigned long)iee_ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)lm_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*pmdp, pmd); -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | PTE_VALID) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ iee_ptep++; -+ new += PAGE_SIZE; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | PTE_VALID) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ lm_ptep++; -+ iee_ptep++; -+ new += PAGE_SIZE; -+ } -+ } -+} -+ -+void __iee_code _iee_unset_token(unsigned long iee_offset, pte_t *ptep, void *token_addr, void *token_page, unsigned long order) -+{ -+ int i; -+ pgd_t *pgdir; -+ pgd_t *pgdp; -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *lm_ptep; -+ pte_t *iee_ptep; -+ unsigned long iee_addr; -+ int use_block_pmd = 0; -+ -+ pgdir = swapper_pg_dir; -+ -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_page); -+ p4dp = p4d_offset(pgdp, (unsigned long)token_page); -+ pudp = pud_offset(p4dp, (unsigned long)token_page); -+ pmdp = pmd_offset(pudp, (unsigned long)token_page); -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ lm_ptep = (pte_t *)pmdp; -+ } -+ else -+ lm_ptep = pte_offset_kernel(pmdp, (unsigned long)token_page); -+ -+ iee_addr = ((unsigned long)token_page + (unsigned long)iee_offset); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4dp = p4d_offset(pgdp, iee_addr); -+ pudp = pud_offset(p4dp, iee_addr); -+ pmdp = pmd_offset(pudp, iee_addr); -+ iee_ptep = pte_offset_kernel(pmdp, iee_addr); -+ -+ ptep = (pte_t *)((unsigned long)ptep + iee_offset); -+ lm_ptep = (pte_t *)((unsigned long)lm_ptep + iee_offset); -+ iee_ptep = (pte_t *)((unsigned long)iee_ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)lm_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | PTE_DBM); -+ WRITE_ONCE(*pmdp, pmd); -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_VALID) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token_addr - IEE_OFFSET))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ iee_ptep++; -+ token_addr += PAGE_SIZE; -+ token_page += PAGE_SIZE; -+ } -+ } -+ else -+ { -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_VALID) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token_addr - IEE_OFFSET))); -+ WRITE_ONCE(*ptep, pte); -+ pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | PTE_DBM); -+ WRITE_ONCE(*lm_ptep, pte); -+ pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ ptep++; -+ lm_ptep++; -+ iee_ptep++; -+ token_addr += PAGE_SIZE; -+ token_page += PAGE_SIZE; -+ } -+ } -+} -+ -+void __iee_code _iee_invalidate_token(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->pgd = NULL; -+ token->valid = false; -+ token->kernel_stack = NULL; -+} -+ -+void __iee_code _iee_validate_token(unsigned long iee_offset, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->valid = true; -+} -+ -+#ifdef CONFIG_KOI -+extern unsigned long koi_swapper_ttbr1; -+#endif -+void __iee_code _iee_init_token(unsigned long iee_offset, struct task_struct *tsk, void *iee_stack, void *tmp_page) -+{ -+ struct task_token *token; -+ -+ token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->iee_stack = iee_stack; -+ token->tmp_page = tmp_page; -+#ifdef CONFIG_KOI -+ token->koi_kernel_stack = NULL; -+ token->koi_stack = NULL; -+ token->koi_stack_base = NULL; -+ token->current_ttbr1 = 0; -+#endif -+} -+ -+void __iee_code _iee_set_token_pgd(unsigned long iee_offset, struct task_struct *tsk, pgd_t *pgd) -+{ -+ struct task_token *token; -+ -+ token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ token->pgd = pgd; -+} -+ -+void __iee_code _iee_set_freeptr(unsigned long iee_offset, void **pptr, void *ptr) -+{ -+ pptr = (void **)((unsigned long)pptr + (unsigned long)iee_offset); -+ *pptr = ptr; -+} -+ -+#pragma GCC push_options -+#pragma GCC optimize("O0") -+void __iee_code _iee_memset(unsigned long iee_offset, void *ptr, int data, size_t n) -+{ -+ char *_ptr; -+ -+ _ptr = (char *)((unsigned long)ptr + (unsigned long)iee_offset); -+ -+ while (n--) -+ *_ptr++ = data; -+} -+ -+void __iee_code _iee_memcpy(unsigned long iee_offset, void *dst, void *src, size_t n) -+{ -+ char *_dst, *_src; -+ -+ _dst = (char *)((unsigned long)dst + (unsigned long)iee_offset); -+ _src = (char *)src; -+ -+ while(n--) -+ *_dst++ = *_src++; -+} -+#pragma GCC pop_options -+ -+void __iee_code _iee_set_track(unsigned long iee_offset, struct track *ptr, struct track *data) -+{ -+ _iee_memcpy(iee_offset, ptr, data, sizeof(struct track)); -+} -+ -+void __iee_code _iee_set_cred_rcu(unsigned long iee_offset, struct cred *cred, struct rcu_head *rcu) -+{ -+ if(cred == &init_cred) -+ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred)); -+ else -+ cred = (struct cred *)((unsigned long)cred + (unsigned long)iee_offset); -+ #ifdef CONFIG_CREDP -+ *((struct rcu_head **)(&(cred->rcu.func))) = rcu; -+ #endif -+} -+ -+void __iee_code _iee_set_cred_security(unsigned long iee_offset, struct cred *cred, void *security) -+{ -+ if(cred == &init_cred) -+ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred)); -+ else -+ cred = (struct cred *)((unsigned long)cred + (unsigned long)iee_offset); -+ cred->security = security; -+} -+ -+bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long iee_offset, struct cred *cred, int flag, int nr) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ switch (flag) -+ { -+ case AT_ADD: { -+ atomic_long_add(nr, &cred->usage); -+ return 0; -+ } -+ case AT_INC_NOT_ZERO: { -+ return atomic_long_inc_not_zero(&cred->usage); -+ } -+ case AT_SUB_AND_TEST: { -+ return atomic_long_sub_and_test(nr, &cred->usage); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_cred_atomic_set_usage(unsigned long iee_offset, struct cred *cred, int i) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ atomic_long_set(&cred->usage,i); -+} -+ -+void __iee_code _iee_set_cred_non_rcu(unsigned long iee_offset, struct cred *cred, int non_rcu) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->non_rcu = non_rcu; -+} -+ -+void __iee_code _iee_set_cred_session_keyring(unsigned long iee_offset, struct cred *cred, struct key *session_keyring) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->session_keyring = session_keyring; -+} -+ -+void __iee_code _iee_set_cred_process_keyring(unsigned long iee_offset, struct cred *cred, struct key *process_keyring) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->process_keyring = process_keyring; -+} -+ -+void __iee_code _iee_set_cred_thread_keyring(unsigned long iee_offset, struct cred *cred, struct key *thread_keyring) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->thread_keyring = thread_keyring; -+} -+ -+void __iee_code _iee_set_cred_request_key_auth(unsigned long iee_offset, struct cred *cred, struct key *request_key_auth) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->request_key_auth = request_key_auth; -+} -+ -+void __iee_code _iee_set_cred_jit_keyring(unsigned long iee_offset, struct cred *cred, unsigned char jit_keyring) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->jit_keyring = jit_keyring; -+} -+ -+void __iee_code _iee_set_cred_cap_inheritable(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_inheritable) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->cap_inheritable = cap_inheritable; -+} -+ -+void __iee_code _iee_set_cred_cap_permitted(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_permitted) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->cap_permitted = cap_permitted; -+} -+ -+void __iee_code _iee_set_cred_cap_effective(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_effective) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->cap_effective = cap_effective; -+} -+ -+void __iee_code _iee_set_cred_cap_bset(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_bset) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->cap_bset = cap_bset; -+} -+ -+void __iee_code _iee_set_cred_cap_ambient(unsigned long iee_offset, struct cred *cred, kernel_cap_t cap_ambient) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->cap_ambient = cap_ambient; -+} -+ -+void __iee_code _iee_set_cred_securebits(unsigned long iee_offset, struct cred *cred, unsigned securebits) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->securebits = securebits; -+} -+ -+void __iee_code _iee_set_cred_group_info(unsigned long iee_offset, struct cred *cred, struct group_info *group_info) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->group_info = group_info; -+} -+ -+void __iee_code _iee_set_cred_ucounts(unsigned long iee_offset, struct cred *cred, struct ucounts *ucounts) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->ucounts = ucounts; -+} -+ -+void __iee_code _iee_set_cred_user_ns(unsigned long iee_offset, struct cred *cred, struct user_namespace *user_ns) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->user_ns = user_ns; -+} -+ -+void __iee_code _iee_set_cred_user(unsigned long iee_offset, struct cred *cred, struct user_struct *user) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->user = user; -+} -+ -+void __iee_code _iee_set_cred_fsgid(unsigned long iee_offset, struct cred *cred, kgid_t fsgid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->fsgid = fsgid; -+} -+ -+void __iee_code _iee_set_cred_fsuid(unsigned long iee_offset, struct cred *cred, kuid_t fsuid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->fsuid = fsuid; -+} -+ -+void __iee_code _iee_set_cred_egid(unsigned long iee_offset, struct cred *cred, kgid_t egid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->egid = egid; -+} -+ -+void __iee_code _iee_set_cred_euid(unsigned long iee_offset, struct cred *cred, kuid_t euid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->euid = euid; -+} -+ -+void __iee_code _iee_set_cred_sgid(unsigned long iee_offset, struct cred *cred, kgid_t sgid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->sgid = sgid; -+} -+ -+void __iee_code _iee_set_cred_suid(unsigned long iee_offset, struct cred *cred, kuid_t suid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->suid = suid; -+} -+ -+void __iee_code _iee_copy_cred(unsigned long iee_offset, struct cred *old, struct cred *new) -+{ -+ #ifdef CONFIG_CREDP -+ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); -+ struct cred *_new = (struct cred *)__phys_to_iee(__pa(new)); -+ _iee_memcpy(iee_offset, new, old, sizeof(struct cred)); -+ *(struct rcu_head **)(&(_new->rcu.func)) = rcu; -+ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func); -+ #endif -+} -+ -+void __iee_code _iee_set_cred_gid(unsigned long iee_offset, struct cred *cred, kgid_t gid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->gid = gid; -+} -+ -+void __iee_code _iee_set_cred_uid(unsigned long iee_offset, struct cred *cred, kuid_t uid) -+{ -+ cred = (struct cred *)__phys_to_iee(__pa(cred)); -+ cred->uid = uid; -+} -+ -+void __iee_code _iee_write_in_byte(unsigned long iee_offset, void *ptr, __u64 data, int length) -+{ -+ ptr = (void *)((unsigned long)ptr + (unsigned long)iee_offset); -+ switch(length) { -+ case 8: { -+ *(__u64 *)ptr = data; -+ break; -+ } -+ case 4: { -+ *(__u32 *)ptr = (__u32)data; -+ break; -+ } -+ case 2: { -+ *(__u16 *)ptr = (__u16)data; -+ break; -+ } -+ case 1: { -+ *(__u8 *)ptr = (__u8)data; -+ break; -+ } -+ } -+} -+ -+static pteval_t inline _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval, unsigned long iee_offset) -+{ -+ pteval_t pteval = cmpxchg_relaxed((pteval_t *)((unsigned long)ptep + iee_offset), old_pteval, new_pteval); -+ return pteval; -+} -+ -+/* Check if addr is allocated in IEE page */ -+static inline bool check_addr_in_iee_valid(unsigned long addr) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ -+ if(!(p4d_val(READ_ONCE(*p4dp)) & PTE_VALID)) -+ return false; -+ -+ pudp = pud_offset(p4dp, addr); -+ -+ if(!(pud_val(READ_ONCE(*pudp)) & PTE_VALID)) -+ return false; -+ -+ pmdp = pmd_offset(pudp, addr); -+ -+ if(!(pmd_val(READ_ONCE(*pmdp)) & PTE_VALID)) -+ return false; -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ -+ return (pte_val(READ_ONCE(*ptep)) & PTE_VALID); -+} -+ -+void __iee_code _iee_set_tramp_pgd(unsigned long iee_offset, pgd_t *pgdp, pgd_t pgd) -+{ -+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); -+} -+ -+void __iee_code _iee_set_swapper_pgd(unsigned long iee_offset, pgd_t *pgdp, pgd_t pgd) -+{ -+ if(!(pgd_val(pgd) & PMD_SECT_VALID)) -+ { -+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); -+ return; -+ } -+ -+ if ((pgd_val(pgd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) -+ panic("You can't use non-iee-pgtable\n"); -+ -+ if((pgdp >= pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 2))) && (pgdp < pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 1))) && !(pgd_val(pgd) & PGD_APT)) -+ panic("Set IEE pgd U page.\n"); -+ -+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); -+} -+ -+void __iee_code _iee_set_p4d(unsigned long iee_offset, p4d_t *p4dp, p4d_t p4d) -+{ -+ if(!(p4d_val(p4d) & PMD_SECT_VALID)) -+ { -+ WRITE_ONCE(*((p4d_t *)((unsigned long)p4dp + (unsigned long)iee_offset)), p4d); -+ return; -+ } -+ -+ if ((p4d_val(p4d) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) -+ panic("You can't use non-iee-pgtable\n"); -+ -+ WRITE_ONCE(*((p4d_t *)((unsigned long)p4dp + (unsigned long)iee_offset)), p4d); -+} -+ -+void __iee_code _iee_set_pud(unsigned long iee_offset, pud_t *pudp, pud_t pud) -+{ -+ if(!(pud_val(pud) & PMD_SECT_VALID)) -+ { -+ WRITE_ONCE(*((pud_t *)((unsigned long)pudp + (unsigned long)iee_offset)), pud); -+ return; -+ } -+ -+ if ((pud_val(pud) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) -+ panic("You can't use non-iee-pgtable\n"); -+ -+ WRITE_ONCE(*((pud_t *)((unsigned long)pudp + (unsigned long)iee_offset)), pud); -+} -+ -+// Return true if the modify does not break DEP. -+static inline bool check_pmd_dep(char *addr, pmd_t pmd) -+{ -+ // DEP for kernel code and readonly data -+ // _text: .text start addr, __init_begin: .rodata end addr -+ if (addr >= _stext && addr < _etext) -+ { -+ if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable -+ !(PTE_RDONLY & pmd_val(pmd))) // DBM == 0 && AP[2] = 0 --> writable -+ { -+ panic("Can't make kernel's text/readonly page as writable!\n" -+ "addr = 0x%16llx, pmd_val = 0x%16llx", -+ (u64)addr, pmd_val(pmd)); -+ } -+ } -+ return true; -+} -+ -+// Return true if the pmd table is a part of kernel page table. -+// TODO : Optimize to get lower overhead. -+static inline bool is_kernel_pmd_table(pmd_t *pmdp, pmd_t pmd) -+{ -+ int i = 0,j = 0; -+ for(i = 0; i < PAGE_SIZE/sizeof(pgd_t); i++) -+ { -+ pgd_t *pgdp = (pgd_t *)swapper_pg_dir + i; -+ if((pgd_val(*pgdp) & PMD_SECT_VALID) && (pgd_val(*pgdp) & PMD_TABLE_BIT)) -+ { -+ for(j = 0; j < PAGE_SIZE/sizeof(pud_t); j++) -+ { -+ pud_t *pudp = (pud_t *)__va(__pgd_to_phys(*pgdp)) + i; -+ if((pud_val(*pudp) & PMD_SECT_VALID) && (pud_val(*pudp) & PMD_TABLE_BIT)) -+ { -+ pmd_t *current_pmdp = __va(__pud_to_phys(*pudp)); -+ if((unsigned long)current_pmdp == ((unsigned long)pmdp & PAGE_MASK)) -+ return true; -+ } -+ } -+ } -+ } -+ return false; -+} -+ -+// Return true if it is mapped to a physical range containing IEE page. -+// TODO : Optimize to get lower overhead. -+static inline bool check_addr_range_in_iee_valid(pmd_t pmd) -+{ -+ int i = 0; -+ unsigned long addr = __phys_to_iee(__pmd_to_phys(pmd)); -+ pgd_t *pgdir = swapper_pg_dir; -+ -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ -+ if(!(p4d_val(READ_ONCE(*p4dp)) & PTE_VALID)) -+ return false; -+ -+ pudp = pud_offset(p4dp, addr); -+ -+ if(!(pud_val(READ_ONCE(*pudp)) & PTE_VALID)) -+ return false; -+ -+ pmdp = pmd_offset(pudp, addr); -+ -+ if(!(pmd_val(READ_ONCE(*pmdp)) & PTE_VALID)) -+ return false; -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ -+ for(i = 0; i < PAGE_SIZE/sizeof(pte_t); i++) -+ { -+ if(pte_val(READ_ONCE(*ptep)) & PTE_VALID) -+ return true; -+ ptep++; -+ } -+ return false; -+} -+ -+void __iee_code _iee_set_pmd(unsigned long iee_offset, pmd_t *pmdp, pmd_t pmd) -+{ -+ char * addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); -+ -+ if(!(pmd_val(pmd) & PMD_SECT_VALID)) -+ { -+ WRITE_ONCE(*((pmd_t *)((unsigned long)pmdp + (unsigned long)iee_offset)), pmd); -+ return; -+ } -+ -+ // Check if the pte table is legally allocated. -+ if ((pmd_val(pmd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) -+ panic("You can't use non-iee-pgtable\n"); -+ -+ // Avoid mapping a huge pmd as U page. -+ // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && (pmd_val(pmd) & PMD_SECT_USER) && is_kernel_pmd_table(pmdp, pmd)) -+ // panic("Set a block descriptor in kernel space U page.\n"); -+ -+ // Avoid mapping a huge pmd to IEE physical page. -+ // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) -+ // panic("Mapping IEE physical page to a huge pmd.\n"); -+ -+ if(!check_pmd_dep(addr, pmd)) -+ return; -+ -+ WRITE_ONCE(*((pmd_t *)((unsigned long)pmdp + (unsigned long)iee_offset)), pmd); -+} -+ -+// Return true if the pte table is a part of kernel page table. -+// TODO : Optimize to get lower overhead. -+static inline bool is_kernel_pte_table(pte_t *ptep, pte_t pte) -+{ -+ return false; -+} -+ -+// Return true if it does not change the privilage or add new U page in kernel. -+static inline bool check_privilage_safe(pte_t *ptep, pte_t pte) -+{ -+ if(!(pte_val(pte) & PTE_VALID)) -+ return true; -+ -+ if((pte_val(*ptep) & PTE_VALID)) -+ { -+ if((pte_val(*ptep) & PTE_USER) != (pte_val(pte) & PTE_USER)) -+ panic("Incorrectly change privilage.\n"); -+ } -+ else -+ { -+ if((pte_val(pte) & PTE_USER) && is_kernel_pte_table(ptep, pte)) -+ panic("Add new U page in kernel space.\n"); -+ } -+ return true; -+} -+ -+// TODO : When adding a new executable page, check it for DEP. -+static inline bool safely_adding_new_exec_page(pte_t *ptep, pte_t pte) -+{ -+ return true; -+} -+ -+// Return true if it is only changing prot of a pte. -+static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte) -+{ -+ if(((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0) -+ return true; -+ else -+ return false; -+} -+ -+// Return true if the modify does not break DEP. -+static inline bool check_pte_dep(char *addr, pte_t pte) -+{ -+ // DEP for kernel code and readonly data -+ // _text: .text start addr, __init_begin: .rodata end addr -+ if (addr >= _stext && addr < _etext) -+ { -+ if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable -+ || !(PTE_RDONLY & pte_val(pte))) // DBM == 0 && AP[2] = 0 --> writable -+ { -+ panic("Can't make kernel's text/readonly page as writable!\n" -+ "addr = 0x%16llx, pte_val = 0x%16llx", -+ (u64)addr, pte_val(pte)); -+ } -+ } -+ return true; -+} -+ -+void __iee_code _iee_set_pte(unsigned long iee_offset, pte_t *ptep, pte_t pte) -+{ -+ char * addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); -+ -+ if(!(pte_val(pte) & PTE_VALID)) -+ { -+ WRITE_ONCE(*((pte_t *)((unsigned long)ptep + (unsigned long)iee_offset)), pte); -+ return; -+ } -+ -+ // Avoid modify privilage unsafely. -+ if(!check_privilage_safe(ptep, pte)) -+ panic("You are modify privilage unsafely.\n"); -+ -+ // Avoid mapping a new executable page. -+ if(!safely_adding_new_exec_page(ptep, pte)) -+ panic("You are adding a new executable page unsafely.\n"); -+ -+ // Avoid mapping a new VA to IEE PA. -+ if(!is_changing_pte_prot(ptep, pte) && -+ check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) -+ panic("You are remmaping IEE page to other VA.\n"); -+ -+ // Avoid mapping a writable VA to kernel code PA. -+ if(!check_pte_dep(addr, pte)) -+ return; -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+#endif -+ WRITE_ONCE(*((pte_t *)((unsigned long)ptep + (unsigned long)iee_offset)), pte); -+} -+ -+static void inline _iee_set_pte_single(pte_t *ptep, pte_t pte, unsigned long iee_offset) -+{ -+ char * addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); -+ -+ if(!(pte_val(pte) & PTE_VALID)) -+ { -+ WRITE_ONCE(*((pte_t *)((unsigned long)ptep + (unsigned long)iee_offset)), pte); -+ return; -+ } -+ -+ // Avoid modify privilage unsafely. -+ if(!check_privilage_safe(ptep, pte)) -+ panic("You are modify privilage unsafely.\n"); -+ -+ // Avoid mapping a new executable page. -+ if(!safely_adding_new_exec_page(ptep, pte)) -+ panic("You are adding a new executable page unsafely.\n"); -+ -+ // Avoid mapping a new VA to IEE PA. -+ if(!is_changing_pte_prot(ptep, pte) && -+ check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) -+ panic("You are remmaping IEE page to other VA.\n"); -+ -+ // Avoid mapping a writable VA to kernel code PA. -+ if(!check_pte_dep(addr, pte)) -+ return; -+ -+ WRITE_ONCE(*((pte_t *)((unsigned long)ptep + (unsigned long)iee_offset)), pte); -+} -+ -+void __iee_code _iee_set_stack_pte(unsigned long iee_offset, pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr) -+{ -+ int i; -+ unsigned long iee_addr = lm_addr + iee_offset; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, iee_addr); -+ pud_t *pudp = pud_offset(p4dp, iee_addr); -+ pmd_t *pmdp = pmd_offset(pudp, iee_addr); -+ pte_t *iee_ptep = (pte_t *)(((unsigned long)pte_offset_kernel(pmdp, iee_addr)) + iee_offset); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ -+ ptep = (pte_t *)((unsigned long)ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) & ~PTE_VALID); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*ptep, pte); -+ ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_unset_stack_pte(unsigned long iee_offset, pte_t *ptep, int order, int use_block_pmd, unsigned long lm_addr) -+{ -+ int i; -+ unsigned long iee_addr = lm_addr + iee_offset; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, iee_addr); -+ pud_t *pudp = pud_offset(p4dp, iee_addr); -+ pmd_t *pmdp = pmd_offset(pudp, iee_addr); -+ pte_t *iee_ptep = (pte_t *)(((unsigned long)pte_offset_kernel(pmdp, iee_addr)) + iee_offset); -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*iee_ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ WRITE_ONCE(*iee_ptep, pte); -+ iee_ptep++; -+ } -+ -+ ptep = (pte_t *)((unsigned long)ptep + iee_offset); -+ if(use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | PTE_VALID); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ WRITE_ONCE(*ptep, pte); -+ ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_set_bm_pte(unsigned long iee_offset, pte_t *ptep, pte_t pte) -+{ -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) -+ pte = __pte(pte_val(pte) | PTE_NG); -+#endif -+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa_symbol(ptep)))), pte); -+} -+ -+/* Data in iee_si_base is visible to all pgd while iee_si_data is private. */ -+unsigned long iee_base_swapper_pg_dir __iee_si_data; -+unsigned long iee_base_idmap_pg_dir __iee_si_data; -+unsigned long iee_base_reserved_pg_dir __iee_si_data; -+unsigned long iee_base__bp_harden_el1_vectors __iee_si_data; -+bool iee_init_done __iee_si_data; -+unsigned long iee_si_tcr __iee_si_data; -+s64 iee_si_offset __iee_si_data; -+ -+static u64 __iee_si_code inline iee_si_mask(unsigned long mask, unsigned long new_val, unsigned long old_val) -+{ -+ return (new_val & mask) | (old_val & ~mask); -+} -+/* -+ * handler function for requests of executing sensitive instrutions. -+ */ -+u64 __iee_si_code iee_si_handler(int flag, ...) -+{ -+ va_list pArgs; -+ u64 old_val, new_val; -+ -+ // BUG_ON(flag > IEE_WRITE_MDSCR); -+ va_start(pArgs, flag); -+ switch (flag) { -+ case IEE_SI_TEST: -+ break; -+ case IEE_WRITE_SCTLR: { -+ old_val = read_sysreg(sctlr_el1); -+ new_val = va_arg(pArgs, u64); -+ new_val = iee_si_mask(IEE_SCTLR_MASK, new_val, old_val); -+ write_sysreg(new_val, sctlr_el1); -+ break; -+ } -+ case IEE_WRITE_TTBR0: -+ case IEE_CONTEXT_SWITCH: { -+ u64 new_asid, new_phys, old_phys, token_phys; -+ struct task_struct *tsk; -+ struct task_token *token; -+ new_val = va_arg(pArgs, u64); -+ new_phys = (new_val & PAGE_MASK) & ~TTBR_ASID_MASK; -+ new_asid = new_val >> 48; -+ -+ // Check ASID first -+ if (new_phys == iee_base_reserved_pg_dir){ -+ if (new_asid != 1) -+ panic("IEE SI warning: reserved_pg_dir ASID invalid: %llx:%llx", new_asid, new_val); -+ } -+ // Already reserved asid 1 for iee rwx gate. -+ else if (new_asid == 0){ -+ new_val |= FIELD_PREP(TTBR_ASID_MASK, 1); -+ printk("IEE SI: Modify ASID of %llx to 1.", new_val); -+ } -+ // TO DO: operations to protect idmap_pg_dir -+ else if (new_phys == iee_base_idmap_pg_dir) -+ { -+ // printk("IEE SI: switch to idmap_pg_dir.); -+ } -+ else if (new_asid % 2 ==0) -+ panic("IEE SI warning: TTBR0 ASID invalid: %llx:%llx", new_asid, new_val); -+ -+ /* Skip verification if iee hasn't been initialized. */ -+ if (iee_init_done){ -+ // Verify current sp_el0 with iee token info -+ asm volatile("mrs %x0, sp_el0":"=r"(tsk)); -+ token = (struct task_token *)((unsigned long)tsk + (unsigned long)iee_offset); -+ -+ /* -+ * token->pgd != NULL means it is a user task, then we need to check whether current ttbr0 is correct. -+ */ -+ if (token->pgd){ -+ old_val = read_sysreg(ttbr0_el1); -+ // When TTBR0 is reserved_pg_dir then no checking is available. -+ if (old_val != iee_base_reserved_pg_dir){ -+ old_phys = (old_val & PAGE_MASK) & ~TTBR_ASID_MASK; -+ token_phys = __pa(token->pgd); -+ if (old_phys != token_phys) -+ panic("IEE SI warning: Pgd set error. old ttbr0:%lx, token ttbr0:%lx, token pgd:%lx", -+ (unsigned long)old_phys, (unsigned long)token_phys, (unsigned long)(token->pgd)); -+ } -+ } -+ } -+ // all checks are done. -+ write_sysreg(new_val, ttbr0_el1); -+ -+ // SET ASID in TTBR1 when context switch -+ if (flag == IEE_CONTEXT_SWITCH){ -+ new_val = (read_sysreg(ttbr1_el1) & ~TTBR_ASID_MASK) | FIELD_PREP(TTBR_ASID_MASK, new_asid-1); -+ write_sysreg(new_val, ttbr1_el1); -+ } -+ break; -+ } -+ case IEE_WRITE_VBAR: { -+ u64 el1_vector; -+ new_val = va_arg(pArgs, u64); -+ el1_vector = iee_base__bp_harden_el1_vectors; -+ if(new_val == el1_vector || new_val == el1_vector+SZ_2K || -+ new_val == el1_vector+SZ_2K*2 || new_val == el1_vector+SZ_2K*3) -+ write_sysreg(new_val, vbar_el1); -+ break; -+ } -+ case IEE_WRITE_TCR: { -+ old_val = read_sysreg(tcr_el1); -+ new_val = va_arg(pArgs, u64); -+ new_val = iee_si_mask(IEE_TCR_MASK, new_val, old_val); -+ write_sysreg(new_val, tcr_el1); -+ break; -+ } -+ } -+ va_end(pArgs); -+ return 0; -+} -+/* -+ * TODO: scan a page to check whether it contains sensitive instructions -+ * return 1 when finding sensitive inst, 0 on safe page. -+ */ -+int iee_si_scan_page(unsigned long addr); -+#endif -\ No newline at end of file -diff --git a/arch/arm64/kernel/iee/pgtable_slab.c b/arch/arm64/kernel/iee/pgtable_slab.c -new file mode 100644 -index 000000000000..abd3b3988250 ---- /dev/null -+++ b/arch/arm64/kernel/iee/pgtable_slab.c -@@ -0,0 +1,107 @@ -+#include -+#include -+#include -+ -+#define PGTABLE_INIT_ORDER 7 -+struct kmem_cache *pgtable_jar; -+struct kmem_cache *ptdesc_jar; -+struct kmem_cache *pgd_jar; -+unsigned long pgtable_jar_offset; -+unsigned long pgd_jar_offset; -+ -+#ifdef CONFIG_PTP -+static inline void iee_ptdesc_init(struct page *page) -+{ -+ /* Alloc struct ptdesc from iee memory pool. */ -+ struct ptdesc *tmp = kmem_cache_alloc(ptdesc_jar, GFP_KERNEL | __GFP_ZERO); -+ spin_lock_init(&tmp->ptl); -+ /* Fill the iee ptdesc pointer array. */ -+ page_to_iee_ptdesc(page) = tmp; -+ /* We need a ptr point back to struct page for list operations. */ -+ ((struct ptdesc_t *)page_to_iee_ptdesc(page))->page = page; -+} -+ -+/* Free ptdesc and clear the iee ptdesc ptr. */ -+static inline void iee_ptdesc_free(struct page *page) -+{ -+ kmem_cache_free(ptdesc_jar, page_to_iee_ptdesc(page)); -+ page_to_iee_ptdesc(page) = NULL; -+} -+ -+extern void early_pgtable_jar_alloc(struct kmem_cache *pgtable_jar); -+void __init iee_pmd_pgtable_init(pud_t *pud) { -+ struct page *page; -+ pmd_t *orig_pmd = pud_pgtable(*pud); -+ pmd_t *pmd; -+ int i; -+ -+ for (i = 0; i < PTRS_PER_PMD; i++) { -+ pmd = orig_pmd + i; -+ if (pmd_none(*pmd) || pmd_bad(*pmd)) -+ continue; -+ page = pmd_page(*pmd); -+ iee_ptdesc_init(page); -+ } -+} -+ -+void __init iee_pud_pgtable_init(p4d_t *p4d) { -+ struct page *page; -+ pud_t *orig_pud = p4d_pgtable(*p4d); -+ pud_t *pud; -+ int i; -+ -+ for (i = 0; i < PTRS_PER_PUD; i++) { -+ pud = orig_pud + i; -+ if (pud_none(*pud) || pud_bad(*pud)) -+ continue; -+ iee_pmd_pgtable_init(pud); -+ page = pud_page(*pud); -+ iee_ptdesc_init(page); -+ } -+} -+ -+void __init iee_pgtable_init(void) -+{ -+ int i; -+ pgd_t *pgd; -+ struct page* page; -+ -+ ptdesc_jar = kmem_cache_create("ptdesc_jar", sizeof(struct ptdesc_t), 0, SLAB_PANIC|SLAB_RED_ZONE, NULL); -+ pgtable_jar = kmem_cache_create("pgtable_jar", PAGE_SIZE, PAGE_SIZE, -+ SLAB_PANIC, NULL); -+ pgd_jar = NULL; -+ for(i = 0; i < ((1 << (PGTABLE_INIT_ORDER))/nr_cpu_ids); i++) -+ { -+ early_pgtable_jar_alloc(pgtable_jar); -+ } -+ for (i = 0; i < PTRS_PER_PGD; i++) { -+ pgd = swapper_pg_dir + i; -+ if (p4d_none_or_clear_bad((p4d_t *)pgd)) -+ continue; -+ iee_pud_pgtable_init((p4d_t *)pgd); -+ page = pgd_page(*pgd); -+ iee_ptdesc_init(page); -+ } -+} -+#endif -+ -+void *get_iee_pgtable_page(gfp_t gfpflags) -+{ -+ struct page *page; -+ void *res = kmem_cache_alloc(pgtable_jar, gfpflags); -+ if (!res) { -+ panic("IEE PTP: alloc pgtable failed."); -+ return res; -+ } -+ iee_rw_gate(IEE_OP_SET_FREEPTR, (void **)((unsigned long)res + pgtable_jar_offset), NULL); -+ page = virt_to_page(res); -+ // page_ref_inc(page); -+ iee_ptdesc_init(page); -+ return res; -+} -+ -+void free_iee_pgtable_page(void *obj) -+{ -+ iee_ptdesc_free(virt_to_page(obj)); -+ kmem_cache_free(pgtable_jar, obj); -+} -\ No newline at end of file -diff --git a/arch/arm64/kernel/iee/stack_slab.c b/arch/arm64/kernel/iee/stack_slab.c -new file mode 100644 -index 000000000000..c52a11d67415 ---- /dev/null -+++ b/arch/arm64/kernel/iee/stack_slab.c -@@ -0,0 +1,19 @@ -+#include -+#include -+ -+struct kmem_cache *iee_stack_jar; -+ -+void __init iee_stack_init(void) -+{ -+ iee_stack_jar = kmem_cache_create("iee_stack_jar", (PAGE_SIZE << 3), (PAGE_SIZE << 3), SLAB_PANIC, NULL); -+} -+ -+void *get_iee_stack(void) -+{ -+ return kmem_cache_alloc(iee_stack_jar, GFP_KERNEL) + iee_offset; -+} -+ -+void free_iee_stack(void *obj) -+{ -+ kmem_cache_free(iee_stack_jar, obj - iee_offset); -+} -diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c -index b1f2a9b49039..7b86a1047278 100644 ---- a/arch/arm64/kernel/irq.c -+++ b/arch/arm64/kernel/irq.c -@@ -31,7 +31,9 @@ - DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); - - DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); -- -+#ifdef CONFIG_KOI -+EXPORT_SYMBOL(irq_stack_ptr); -+#endif - - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); - -diff --git a/arch/arm64/kernel/koi/Makefile b/arch/arm64/kernel/koi/Makefile -new file mode 100644 -index 000000000000..9be8710b714a ---- /dev/null -+++ b/arch/arm64/kernel/koi/Makefile -@@ -0,0 +1 @@ -+obj-y += koi.o -\ No newline at end of file -diff --git a/arch/arm64/kernel/koi/koi.c b/arch/arm64/kernel/koi/koi.c -new file mode 100644 -index 000000000000..20daf4889ca6 ---- /dev/null -+++ b/arch/arm64/kernel/koi/koi.c -@@ -0,0 +1,1688 @@ -+#include "asm/koi.h" -+#include "linux/compiler_attributes.h" -+#include "linux/compiler_types.h" -+#include "asm/barrier.h" -+#include "asm-generic/bug.h" -+#include "asm-generic/errno-base.h" -+#include "asm-generic/memory_model.h" -+#include "asm-generic/pgtable-nop4d.h" -+#include "asm-generic/rwonce.h" -+#include "asm/pgalloc.h" -+#include "asm/memory.h" -+#include "linux/bitfield.h" -+#include "linux/compiler.h" -+#include "linux/types.h" -+#include "linux/module.h" -+#include "linux/spinlock.h" -+#include "linux/spinlock_types.h" -+#include "linux/kernel.h" -+#include "linux/rculist.h" -+#include "linux/rcupdate.h" -+#include "linux/list.h" -+#include "asm/current.h" -+#include "linux/compiler_types.h" -+#include "asm-generic/barrier.h" -+#include "asm-generic/rwonce.h" -+#include "asm-generic/pgalloc.h" -+#include "asm/cpufeature.h" -+#include "asm/kvm_hyp.h" -+#include "asm/mmu.h" -+#include "asm/mmu_context.h" -+#include "asm/page-def.h" -+#include "asm/pgalloc.h" -+#include "asm/pgtable-hwdef.h" -+#include "asm/pgtable-types.h" -+#include "asm/pgtable.h" -+#include "asm/string.h" -+#include "asm/sysreg.h" -+#include "linux/bitfield.h" -+#include "linux/compiler.h" -+#include "linux/export.h" -+#include "linux/gfp.h" -+#include "linux/huge_mm.h" -+#include "linux/kallsyms.h" -+#include "linux/kconfig.h" -+#include "linux/kern_levels.h" -+#include "linux/kernel.h" -+#include "linux/list.h" -+#include "linux/lockdep.h" -+#include "linux/mm.h" -+#include "linux/mm_types.h" -+#include "linux/pgtable.h" -+#include "linux/printk.h" -+#include "linux/rculist.h" -+#include "linux/rcupdate.h" -+#include "linux/rmap.h" -+#include "linux/sched.h" -+#include "linux/stddef.h" -+#include "linux/string.h" -+#include "linux/swap.h" -+#include "linux/swapops.h" -+#include "linux/types.h" -+#include "linux/slab.h" -+#include "linux/string.h" -+#include "linux/hashtable.h" -+#include -+// #define DEBUG -+ -+#ifdef DEBUG -+#define debug_printk(...) printk(KERN_ERR __VA_ARGS__) -+#else -+#define debug_printk(...) -+#endif -+ -+#define __koi_code __section(".koi.text") -+#define __koi_data __section(".data..koi") -+ -+#define KOI_FLAG_MASK 0xffff000000000fff -+ -+extern unsigned long __koi_code_start[]; -+extern unsigned long __koi_code_end[]; -+extern unsigned long __koi_data_start[]; -+extern unsigned long __koi_data_end[]; -+#ifdef CONFIG_IEE -+extern unsigned long __iee_si_data_start[]; -+extern unsigned long __iee_si_text_start[]; -+extern unsigned long _iee_read_token_ttbr1(unsigned long iee_offset, -+ struct task_struct *tsk); -+#endif -+ -+__koi_data unsigned long koi_swapper_ttbr1 = 0; -+EXPORT_SYMBOL(koi_swapper_ttbr1); -+#define KOI_SWAPPER_MASK 0x0000fffffffffff0 -+ -+__attribute__((aligned(PAGE_SIZE))) -+DEFINE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)], -+ koi_irq_current_ttbr1); -+EXPORT_SYMBOL(koi_irq_current_ttbr1); -+ -+extern void koi_switch_to_ko_stack(unsigned long stack_top); -+extern void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp); -+extern void koi_check_and_switch_context(struct mm_struct *mm); -+extern void koi_add_page_mapping(void *token, void *new); -+/** -+*struct koi_mem_list - maintain a linked list of free memory in the kernel -+*@addr: stating address of this memory -+*@size: the size of the memory -+*@list: the head of the koi_mem_list -+*@rcu: for rcu -+*/ -+struct koi_mem_list { -+ unsigned long addr; -+ unsigned long size; -+ struct list_head list; -+ struct rcu_head rcu; -+}; -+//mapping parameter pointer to copy -+struct koi_addr_map { -+ unsigned long buffer_addr; -+ unsigned long orig_addr; -+ int offset; -+ struct hlist_node node; -+ struct rcu_head rcu; -+}; -+ -+DEFINE_HASHTABLE(koi_mem_htbl, HASH_TABLE_BIT); -+EXPORT_SYMBOL(koi_mem_htbl); -+DEFINE_SPINLOCK(koi_mem_htbl_spin_lock); -+EXPORT_SYMBOL(koi_mem_htbl_spin_lock); -+ -+// EXPORT_SYMBOL(koi_do_switch_to_kernel_stack); -+// EXPORT_SYMBOL(koi_do_switch_to_ko_stack); -+ -+/** -+* koi_ttbr_ctor - return ttbr1 for the given driver module -+*/ -+unsigned long koi_ttbr_ctor(struct module *mod) -+{ -+ struct koi_mem_hash_node *ko; -+ struct mm_struct *ko_mm; -+ unsigned long ttbr1; -+ unsigned long asid; -+ int bkt; -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ if (ko->mod == mod) { -+ ko_mm = ko->ko_mm; -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (!ko_mm) { -+ printk(KERN_ERR "cannot found module %s in koi_mem_htbl", -+ mod->name); -+ return 0; -+ } -+ koi_check_and_switch_context(ko_mm); -+ asid = ASID(ko_mm) | USER_ASID_FLAG; -+ ttbr1 = ko->ko_ttbr1 | FIELD_PREP(TTBR_ASID_MASK, asid); -+ return ttbr1; -+} -+EXPORT_SYMBOL(koi_ttbr_ctor); -+//release the hash node -+static __maybe_unused void koi_mem_hash_node_free(struct rcu_head *rcu) -+{ -+ struct koi_mem_hash_node *node = -+ container_of(rcu, struct koi_mem_hash_node, rcu); -+ kfree(node); -+} -+//release free memory linked list nodes -+static void koi_mem_node_free(struct rcu_head *rcu) -+{ -+ struct koi_mem_list *mem_node = -+ container_of(rcu, struct koi_mem_list, rcu); -+ kfree(mem_node); -+} -+//release the node in koi_addr_map -+static void koi_addr_map_node_free(struct rcu_head *rcu) -+{ -+ struct koi_addr_map *addr_map_node = -+ container_of(rcu, struct koi_addr_map, rcu); -+ kfree(addr_map_node); -+} -+ -+static __koi_code noinline unsigned int koi_ldtrh_wrapper(unsigned long src_addr) -+{ -+ unsigned int ret; -+ asm volatile("ldtrh w0, [%1]\n" : "=r"(ret) : "r"(src_addr)); -+ return ret; -+} -+ -+static __koi_code noinline unsigned long koi_ldtr_wrapper(unsigned long src_addr) -+{ -+ unsigned long ret; -+ asm volatile("ldtr x0, [%1]\n" : "=r"(ret) : "r"(src_addr)); -+ return ret; -+} -+ -+static __koi_code noinline void koi_sttr_wrapper(unsigned long src, -+ unsigned long dst_addr) -+{ -+ asm volatile("sttr x0, [x1]\n" : :); -+} -+ -+static __koi_code noinline void koi_sttrh_wrapper(unsigned int src, -+ unsigned long dst_addr) -+{ -+ asm volatile("sttrh w0, [x1]\n" : :); -+} -+ -+#ifndef CONFIG_IEE -+/* -+ * This function is used to switch to ko's pgtable. -+ */ -+__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void) -+{ -+ struct koi_mem_hash_node *ko; -+ // struct mm_struct *ko_mm; -+ unsigned long addr; -+ unsigned long ttbr1, asid; -+ unsigned long *ptr; -+ struct task_token *token_addr = -+ (struct task_token *)((unsigned long)current + -+ (unsigned long)koi_offset); -+ int bkt; -+ unsigned long flags; -+ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr)); -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset()); -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ spin_lock_irqsave(&ko->mod_lock, flags); -+ if (!ko->is_valid) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ goto out; -+ } -+ if (ko->mod != NULL && ko->mod->mem[MOD_INIT_TEXT].base != NULL) { -+ if (addr >= (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base + ko->mod->mem[MOD_INIT_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (token_addr->current_ttbr1 == ko->ko_ttbr1 || -+ *ptr == ko->ko_ttbr1) { -+ // ko_mm = ko->ko_mm; -+ // koi_check_and_switch_context(ko_mm); -+ // asid = ASID(ko_mm); -+ // ttbr1 = ko->ko_ttbr1; -+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); -+ asm volatile("mrs %0, ttbr0_el1\n" -+ : "=r"(asid) -+ :); -+ asid &= TTBR_ASID_MASK; -+ ttbr1 = ko->ko_ttbr1 | asid; -+ rcu_read_unlock(); -+ return ttbr1; -+ } -+ goto out; -+ } -+ } -+ if (addr >= (unsigned long)ko->mod->mem[MOD_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_TEXT].base + ko->mod->mem[MOD_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (token_addr->current_ttbr1 == ko->ko_ttbr1 || -+ *ptr == ko->ko_ttbr1) { -+ // ko_mm = ko->ko_mm; -+ // koi_check_and_switch_context(ko_mm); -+ // asid = ASID(ko_mm); -+ // ttbr1 = ko->ko_ttbr1; -+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); -+ asm volatile("mrs %0, ttbr0_el1\n" -+ : "=r"(asid) -+ :); -+ asid &= TTBR_ASID_MASK; -+ ttbr1 = ko->ko_ttbr1 | asid; -+ rcu_read_unlock(); -+ return ttbr1; -+ } -+ goto out; -+ } -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ } -+out: -+ rcu_read_unlock(); -+ return 0; -+} -+/** -+* koi_do_switch_to_kernel_pgtbl - switch to kernel pagetable -+*/ -+__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void) -+{ -+ unsigned long curr_ttbr1, asid; -+ // if (!cpu_online(smp_processor_id())) -+ // return 0; -+ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1)); -+ if ((curr_ttbr1 & KOI_SWAPPER_MASK) == -+ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) { -+ return 0; -+ } -+ // if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) { -+ // return 0; -+ // } -+ asm volatile("mrs %0, ttbr0_el1\n" : "=r"(asid) :); -+ asid &= ~USER_ASID_FLAG; -+ asid &= TTBR_ASID_MASK; -+ write_sysreg(koi_swapper_ttbr1 | asid, ttbr1_el1); -+ isb(); -+ asm volatile(ALTERNATIVE("nop; nop; nop", "ic iallu; dsb nsh; isb", -+ ARM64_WORKAROUND_CAVIUM_27456)); -+ return 1; -+} -+#else -+__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void) -+{ -+ struct koi_mem_hash_node *ko; -+ unsigned long addr, pan_flag, current_ttbr1, asid, ttbr1, flags; -+ unsigned long *ptr; -+ struct mm_struct *ko_mm; -+ int bkt; -+ asm volatile("mrs %0, pan\n" -+ "msr pan, 0x0\n" -+ : "=r"(pan_flag) -+ :); -+ current_ttbr1 = _iee_read_token_ttbr1(iee_offset, current); -+ asm volatile("msr pan, %0\n" : : "r"(pan_flag)); -+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset()); -+ if (current_ttbr1 == 0 && *ptr == 0) -+ return 0; -+ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr)); -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ spin_lock_irqsave(&ko->mod_lock, flags); -+ if (!ko->is_valid) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ goto out; -+ } -+ if (ko->mod != NULL && ko->mod->mem[MOD_INIT_TEXT].base != NULL) { -+ if (addr >= (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base + ko->mod->mem[MOD_INIT_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) { -+ asm volatile("mrs %0, ttbr0_el1\n" -+ : "=r"(asid) -+ :); -+ asid &= TTBR_ASID_MASK; -+ ttbr1 = ko->ko_ttbr1 | asid; -+ rcu_read_unlock(); -+ return ttbr1; -+ } -+ goto out; -+ } -+ } -+ if (addr >= (unsigned long)ko->mod->mem[MOD_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_TEXT].base + ko->mod->mem[MOD_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) { -+ asm volatile("mrs %0, ttbr0_el1\n" -+ : "=r"(asid) -+ :); -+ asid &= TTBR_ASID_MASK; -+ ttbr1 = ko->ko_ttbr1 | asid; -+ rcu_read_unlock(); -+ return ttbr1; -+ } -+ goto out; -+ } -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ } -+out: -+ rcu_read_unlock(); -+ return 0; -+} -+ -+__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void) -+{ -+ unsigned long curr_ttbr1, asid, error_addr; -+ // if (!cpu_online(smp_processor_id())) -+ // return 0; -+ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1)); -+ if ((curr_ttbr1 & KOI_SWAPPER_MASK) == -+ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) { -+ return 0; -+ } -+ // if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) { -+ // return 0; -+ // } -+ error_addr = read_sysreg(elr_el1); -+ if ((error_addr >= (unsigned long)__iee_si_data_start) && -+ (error_addr <= (unsigned long)__iee_si_text_start)) { -+ unsigned long esr = read_sysreg(esr_el1); -+ asm volatile("mrs %0, ttbr1_el1\n" : "=r"(asid) :); -+ asid &= ~USER_ASID_FLAG; -+ asid &= TTBR_ASID_MASK; -+ write_sysreg(koi_swapper_ttbr1 | asid, ttbr1_el1); -+ isb(); -+ printk(KERN_ERR -+ "IEE SI: Error on switch to kernel. ELR_EL1:0x%llx, ESR_EL1:0x%llx, TTBR1:0x%llx", -+ error_addr, esr, curr_ttbr1); -+ } else { -+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); -+ } -+ return 1; -+} -+#endif -+/** -+* koi_save_ttbr - save ttbr of each driver module -+* @mod: driver module -+* @pgdp:pointer to driver module top page table,pgd -+*/ -+static void koi_save_ttbr(struct module *mod, pgd_t *pgdp, -+ struct koi_mem_hash_node *node) -+{ -+ phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); -+ if (system_supports_cnp()) -+ ttbr1 |= TTBR_CNP_BIT; -+ node->ko_ttbr1 = ttbr1; -+} -+ -+/** -+ * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page -+ * is required to copy this pte. -+*/ -+static inline int koi_copy_present_pte(struct mm_struct *ko_mm, pte_t *dst_pte, -+ pte_t *src_pte, unsigned long addr) -+{ -+ pte_t pte = *src_pte; -+ struct page *page; -+ -+ debug_printk("addr=0x%16llx, dst_pte=0x%16llx\n", addr, dst_pte); -+#ifdef CONFIG_IEE -+ if ((pte_val(pte) & PTE_USER) && (pte_val(pte) & PTE_DBM) && -+ !(pte_val(pte) & PTE_PXN)) { -+ set_pte(dst_pte, __pte(pte_val(pte) & (~PTE_DBM) | PTE_RDONLY)); -+ } else { -+#endif -+ set_pte(dst_pte, pte); -+#ifdef CONFIG_IEE -+ } -+#endif -+ return 0; -+} -+/** -+* copy huge pmd from kernel space to driver space. -+*/ -+static int koi_copy_huge_pmd(struct mm_struct *ko_mm, pmd_t *dst_pmd, -+ pmd_t *src_pmd, unsigned long addr, pteval_t prot) -+{ -+ spinlock_t *src_ptl, *dst_ptl; -+ pmd_t pmd; -+ int ret = -ENOMEM; -+ debug_printk( -+ "hugepmd: src_pmd=0x%16llx, dst_pmd=0x%16llx, src_pmd_val=0x%16llx, dst_pmd_val=0x%16llx, addr=0x%16llx\n", -+ src_pmd, dst_pmd, pmd_val(*src_pmd), pmd_val(*dst_pmd), addr); -+ dst_ptl = pmd_lockptr(ko_mm, dst_pmd); -+ src_ptl = pmd_lockptr(&init_mm, src_pmd); -+ spin_lock_bh(dst_ptl); -+ spin_lock_bh(src_ptl); -+ -+ set_pmd(dst_pmd, __pmd(pmd_val(*src_pmd) | prot)); -+ ret = 0; -+ spin_unlock_bh(src_ptl); -+ spin_unlock_bh(dst_ptl); -+ return ret; -+} -+ -+int __koi_pte_alloc(struct mm_struct *mm, pmd_t *pmd) -+{ -+ spinlock_t *ptl; -+ // pgtable_t new = alloc_page(GFP_PGTABLE_KERNEL); -+ pte_t *new = pte_alloc_one_kernel(mm); -+ printk(KERN_ERR "alloc new=0x%16llx\n", new); -+ if (!new) -+ return -ENOMEM; -+ -+ /* -+ * Ensure all pte setup (eg. pte page lock and page clearing) are -+ * visible before the pte is made visible to other CPUs by being -+ * put into page tables. -+ * -+ * The other side of the story is the pointer chasing in the page -+ * table walking code (when walking the page table without locking; -+ * ie. most of the time). Fortunately, these data accesses consist -+ * of a chain of data-dependent loads, meaning most CPUs (alpha -+ * being the notable exception) will already guarantee loads are -+ * seen in-order. See the alpha page table accessors for the -+ * smp_rmb() barriers in page table walking code. -+ */ -+ smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ -+ -+ ptl = pmd_lockptr(mm, pmd); -+ spin_lock_bh(ptl); -+ if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ -+ mm_inc_nr_ptes(mm); -+ pmd_populate_kernel(mm, pmd, new); -+ new = NULL; -+ } -+ spin_unlock_bh(ptl); -+ if (new) { -+ pte_free_kernel(mm, new); -+ } -+ return 0; -+} -+ -+#define koi_pte_alloc(mm, pmd) \ -+ (unlikely(pmd_none(*(pmd))) && __koi_pte_alloc(mm, pmd)) -+ -+#define koi_pte_offset_map_lock(mm, pmd, address, ptlp) \ -+ ({ \ -+ spinlock_t *__ptl = pte_lockptr(mm, pmd); \ -+ pte_t *__pte = pte_offset_kernel(pmd, address); \ -+ *(ptlp) = __ptl; \ -+ spin_lock_bh(__ptl); \ -+ __pte; \ -+ }) -+ -+#define koi_pte_alloc_map_lock(mm, pmd, address, ptlp) \ -+ (koi_pte_alloc(mm, pmd) ? \ -+ NULL : \ -+ koi_pte_offset_map_lock(mm, pmd, address, ptlp)) -+ -+/** -+*koi_copy_pte_range - copy pte from kernel space to driver space -+*/ -+static int koi_copy_pte_range(struct mm_struct *ko_mm, pmd_t *dst_pmd, -+ pmd_t *src_pmd, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pte_t *src_pte, *dst_pte; -+ spinlock_t *src_ptl, *dst_ptl; -+ -+ int ret = 0; -+again: -+ dst_pte = koi_pte_alloc_map_lock(ko_mm, dst_pmd, addr, &dst_ptl); -+ if (!dst_pte) { -+ ret = -ENOMEM; -+ goto unlock; -+ } -+ src_pte = pte_offset_kernel(src_pmd, addr); -+ src_ptl = pte_lockptr(&init_mm, src_pmd); -+ spin_lock_bh(src_ptl); -+ do { -+ if (pte_none(*src_pte)) -+ continue; -+ if (unlikely(!pte_present(*src_pte))) { -+ printk(KERN_ERR -+ "present pte found: addr=0x%16llx, end=0x%16llx\n", -+ addr, end); -+ continue; -+ } -+ if (pte_valid(*dst_pte)) { -+ continue; -+ } -+ /* koi_copy_present_pte() will clear `*prealloc` if consumed */ -+ ret = koi_copy_present_pte(ko_mm, dst_pte, src_pte, addr); -+ debug_printk( -+ "dst_pte=0x%16llx, dst_pte_val=0x%16llx, src_pte=0x%16llx, src_pte_val=0x%16llx, addr=0x%16llx\n", -+ dst_pte, pte_val(*dst_pte), src_pte, pte_val(*src_pte), -+ addr); -+ if (unlikely(ret == -EAGAIN)) -+ break; -+ } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); -+ spin_unlock_bh(src_ptl); -+ spin_unlock_bh(dst_ptl); -+ -+ if (ret) { -+ WARN_ON_ONCE(ret != -EAGAIN); -+ ret = 0; -+ } -+ if (addr != end) -+ goto again; -+ -+ return ret; -+unlock: -+ spin_unlock_bh(dst_ptl); -+ return ret; -+} -+ -+int __koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) -+{ -+ spinlock_t *ptl; -+ // struct page* page = alloc_page(GFP_PGTABLE_KERNEL); -+ // pmd_t *new = (pmd_t *)page_address(page); -+ pmd_t *new = pmd_alloc_one(mm, address); -+ if (!new) -+ return -ENOMEM; -+ -+ smp_wmb(); /* See comment in __pte_alloc */ -+ -+ ptl = pud_lockptr(mm, pud); -+ spin_lock_bh(ptl); -+ if (!pud_present(*pud)) { -+ mm_inc_nr_pmds(mm); -+ pud_populate(mm, pud, new); -+ } else { /* Another has populated it */ -+ pmd_free(mm, new); -+ // free_page((unsigned long)new); -+ } -+ spin_unlock_bh(ptl); -+ return 0; -+} -+ -+static inline pmd_t *koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, -+ unsigned long address) -+{ -+ return (unlikely(pud_none(*pud)) && __koi_pmd_alloc(mm, pud, address)) ? -+ NULL : -+ pmd_offset(pud, address); -+} -+ -+/** -+*kio_copy_pmd_range - copy pmd from kernel to driver space -+*/ -+static inline int koi_copy_pmd_range(struct mm_struct *ko_mm, pud_t *dst_pud, -+ pud_t *src_pud, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pmd_t *src_pmd, *dst_pmd; -+ unsigned long next, flag; -+ int err; -+ debug_printk( -+ "copy_pud_range src_pud=0x%16llx, dst_pud=0x%16llx, addr=0x%16llx, end=0x%16llx\n", -+ src_pud, dst_pud, addr, end); -+ dst_pmd = koi_pmd_alloc(ko_mm, dst_pud, addr); -+ if (!dst_pmd) { -+ return -ENOMEM; -+ } -+ src_pmd = pmd_offset(src_pud, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ flag = pmd_val(*src_pmd) & KOI_FLAG_MASK; -+ // debug_printk("src_pmd=0x%16llx, dst_pmd=0x%16llx, addr=0x%16llx\n", src_pmd, dst_pmd, next); -+ // CONFIG_TRANSPARENT_HUGEPAGE is enabled, so we must add copy_huge_pmd -+ if (pmd_none(*src_pmd)) -+ continue; -+ if (!(pmd_val(*src_pmd) & PMD_TABLE_BIT)) { -+ // if src_pmd is huge page -+ debug_printk( -+ "src_pmd=0x%16llx, dst_pmd=0x%16llx, addr=0x%16llx\n", -+ src_pmd, dst_pmd, addr); -+ err = koi_copy_huge_pmd(ko_mm, dst_pmd, src_pmd, addr, prot); -+ if (err == -ENOMEM) -+ return -ENOMEM; -+ continue; -+ } -+ if (koi_copy_pte_range(ko_mm, dst_pmd, src_pmd, addr, next, prot)) -+ return -ENOMEM; -+ set_pmd(dst_pmd, __pmd((pmd_val(*dst_pmd) & (~KOI_FLAG_MASK)) | flag)); -+ debug_printk( -+ "src_pmd_val=0x%16llx, dst_pmd_val=0x%16llx, addr=0x%16llx\n", -+ pmd_val(*src_pmd), pmd_val(*dst_pmd), addr); -+ } while (dst_pmd++, src_pmd++, addr = next, addr != end); -+ return 0; -+} -+ -+int __koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) -+{ -+ // pud_t *new = pud_alloc_one(mm, address); -+ -+ // struct page *page = alloc_page(GFP_PGTABLE_KERNEL); -+ // pud_t *new = (pud_t *)page_address(page); -+ pud_t *new = pud_alloc_one(mm, address); -+ printk(KERN_ERR "pud alloc pud=0x%16llx\n", new); -+ if (!new) -+ return -ENOMEM; -+ -+ smp_wmb(); /* See comment in __pte_alloc */ -+ -+ spin_lock_bh(&mm->page_table_lock); -+ if (!p4d_present(*p4d)) { -+ mm_inc_nr_puds(mm); -+ p4d_populate(mm, p4d, new); -+ } else {/* Another has populated it */ -+ pud_free(mm, new); -+ // free_page((unsigned long)new); -+ } -+ spin_unlock_bh(&mm->page_table_lock); -+ return 0; -+} -+ -+static inline pud_t *koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, -+ unsigned long address) -+{ -+ return (unlikely(p4d_none(*p4d)) && __koi_pud_alloc(mm, p4d, address)) ? -+ NULL : -+ pud_offset(p4d, address); -+} -+ -+static int koi_copy_huge_pud(struct mm_struct *ko_mm, pud_t *dst_pud, -+ pud_t *src_pud, unsigned long addr, pteval_t prot) -+{ -+ spinlock_t *src_ptl, *dst_ptl; -+ // pmd_t pmd; -+ int ret = -ENOMEM; -+ debug_printk("src_pud=0x%16llx, dst_pud=0x%16llx, addr=0x%16llx\n", -+ src_pud, dst_pud, addr); -+ dst_ptl = pud_lockptr(ko_mm, dst_pud); -+ src_ptl = pud_lockptr(&init_mm, src_pud); -+ spin_lock_bh(dst_ptl); -+ spin_lock_bh(src_ptl); -+ set_pte((pte_t *)dst_pud, __pte(pud_val(*src_pud) | prot)); -+ spin_unlock_bh(src_ptl); -+ spin_unlock_bh(dst_ptl); -+ ret = 0; -+ return ret; -+} -+ -+/** -+*koi_copy_pud_range - copy pud from kernel to driver -+*/ -+static inline int koi_copy_pud_range(struct mm_struct *ko_mm, p4d_t *dst_p4d, -+ p4d_t *src_p4d, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pud_t *src_pud, *dst_pud; -+ unsigned long next, flag; -+ dst_pud = koi_pud_alloc(ko_mm, dst_p4d, addr); -+ if (!dst_pud) -+ return -ENOMEM; -+ src_pud = pud_offset(src_p4d, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ flag = pud_val(*src_pud) & KOI_FLAG_MASK; -+ debug_printk( -+ "src_pud=0x%16llx, dst_pud=0x%16llx, addr=0x%16llx\n", -+ src_pud, dst_pud, next); -+ if (pud_none(*src_pud)) { -+ continue; -+ } -+ if (!(pud_val(*src_pud) & PMD_TABLE_BIT)) { -+ // indicates that the src_pud maps to a huge page -+ koi_copy_huge_pud(ko_mm, dst_pud, src_pud, addr, prot); -+ continue; -+ } -+ // if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) { -+ // debug_printk("pud_trans_huge=%d, pud_devmap=%d, src_pud=0x%16llx\n", pud_trans_huge(*src_pud) , pud_devmap(*src_pud), src_pud); -+ // continue; -+ // /* fall through */ -+ // } -+ if (koi_copy_pmd_range(ko_mm, dst_pud, src_pud, addr, next, prot)) -+ return -ENOMEM; -+ set_pud(dst_pud, __pud((pud_val(*dst_pud) & (~KOI_FLAG_MASK)) | flag)); -+ } while (dst_pud++, src_pud++, addr = next, addr != end); -+ return 0; -+} -+ -+/** -+* koi_copy_p4d_range - map the kernel pagetable to the driver space level by level -+* @ko_mm: the mm_struct of driver module -+* @dst_pgd: destination pgd -+* @src_pgd: source pgd -+* @addr: the start of address -+* @end: the end of address -+*/ -+static inline int koi_copy_p4d_range(struct mm_struct *ko_mm, pgd_t *dst_pgd, -+ pgd_t *src_pgd, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ p4d_t *src_p4d, *dst_p4d; -+ unsigned long next; -+ dst_p4d = p4d_alloc(ko_mm, dst_pgd, addr); -+ if (!dst_p4d) -+ return -ENOMEM; -+ src_p4d = p4d_offset(src_pgd, addr); -+ do { -+ next = p4d_addr_end(addr, end); -+ if (p4d_none(*src_p4d) || p4d_bad(*src_p4d)) -+ continue; -+ debug_printk( -+ "dst_p4d=0x%16llx, dst_p4d_val=0x%16llx\n", -+ dst_p4d, p4d_val(*dst_p4d)); -+ if (koi_copy_pud_range(ko_mm, dst_p4d, src_p4d, addr, next, prot)) { -+ return -ENOMEM; -+ } -+ } while (dst_p4d++, src_p4d++, addr = next, addr != end); -+ return 0; -+} -+int koi_share_kstack(struct module *mod) -+{ -+ unsigned long kstack_start; -+ struct koi_mem_hash_node *target = NULL; -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ kstack_start = (unsigned long)current->stack; -+ -+ return koi_copy_pagetable(target->ko_mm, target->pgdp, kstack_start, -+ kstack_start + THREAD_SIZE, (0)); -+} -+EXPORT_SYMBOL(koi_share_kstack); -+/** -+*int koi_copy_pagetable - map the address range from "addr" to "end" to the driver pagetable -+*@ko_mm: the mm_struct of the driver module -+*@koi_pg_dir: koi_pg_dir, related to the driver module, the entry for driver pagetable -+*@addr: the starting address of mapping zone -+*@end: the end address of mapping zone -+*/ -+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, -+ unsigned long addr, unsigned long end, pteval_t prot) -+{ -+ int ret = 0; -+ unsigned long next; -+ -+ pgd_t *src_pgd, *dst_pgd; -+ unsigned long flag; -+ src_pgd = pgd_offset_pgd(swapper_pg_dir, addr); -+ dst_pgd = pgd_offset_pgd(koi_pg_dir, addr); -+ do { -+ flag = pgd_val(*src_pgd) & KOI_FLAG_MASK; -+ next = pgd_addr_end(addr, end); -+ if (pgd_none(*src_pgd) || pgd_bad(*src_pgd)) -+ continue; -+ if (unlikely(koi_copy_p4d_range(ko_mm, dst_pgd, src_pgd, addr, -+ next, prot))) { -+ ret = -ENOMEM; -+ break; -+ } -+ set_pgd(dst_pgd, __pgd((pgd_val(*dst_pgd) & (~KOI_FLAG_MASK)) | flag)); -+ } while (dst_pgd++, src_pgd++, addr = next, addr != end); -+ -+ return ret; -+} -+EXPORT_SYMBOL(koi_copy_pagetable); -+void koi_set_rdonly(unsigned long addr, pgd_t *pgdir) -+{ -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { -+ return; -+ } -+ -+ p4dp = p4d_offset(pgdp, addr); -+ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) { -+ return; -+ } -+ -+ pudp = pud_offset(p4dp, addr); -+ if (pud_none(*pudp) || pud_bad(*pudp)) { -+ return; -+ } -+ pmdp = pmd_offset(pudp, addr); -+ if (pmd_none(*pmdp) || pmd_bad(*pmdp)) { -+ return; -+ } -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ if (pte_none(*ptep)) { -+ // printk(KERN_ERR "ptep 0x%16llx not available\n", ptep); -+ return; -+ } -+ set_pte(ptep, __pte(pte_val(*ptep) | PTE_RDONLY)); -+ // printk(KERN_ERR "set_readonly successfully\n"); -+ return; -+} -+ -+void koi_set_upage(struct module *mod, unsigned long addr, unsigned long size) { -+ struct koi_mem_hash_node *target = NULL; -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ -+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK, PTE_USER); -+} -+EXPORT_SYMBOL(koi_set_upage); -+ -+void koi_unmap_pte_table(struct mm_struct *ko_mm, pmd_t *pmd, -+ unsigned long addr, unsigned long end) -+{ -+ pte_t *pte; -+ if (!pmd) -+ return; -+ debug_printk("pmd=0x%16llx, addr=0x%16llx, end=0x%16llx\n", pmd, addr, -+ end); -+ -+ // struct page *page = pte_page(*pte); -+ // printk(KERN_ERR "pte=0x%16llx, pte_val=0x%16llx\n", pte, pte_val(*pte)); -+ // debug_printk("free pte table 0x%16llx, pmd=0x%16llx, page=0x%16llx, pmd points to page=0x%16llx\n", pte, pmd, page, pte_page(pmd_pte(*pmd))); -+ // printk(KERN_ERR "pmd_pfn=0x%16llx, pte_pfn=0x%16llx\n", pmd_pfn(*pmd), pte_pfn(*pte)); -+ // pte_free(ko_mm, pte_page(pmd_pte(*pmd))); -+ do { -+ pte = pte_offset_kernel(pmd, addr); -+ debug_printk("pte=0x%16llx, pte_val=0x%16llx\n", pte, -+ pte_val(*pte)); -+ set_pte(pte, __pte(0)); -+ } while (addr += PAGE_SIZE, addr != end); -+} -+ -+void koi_unmap_pmd_range(struct mm_struct *ko_mm, pud_t *pud, -+ unsigned long addr, unsigned long end) -+{ -+ pmd_t *pmd, *orig_pmd; -+ unsigned long next; -+ if (!pud) -+ return; -+ orig_pmd = pmd_offset(pud, addr); -+ pmd = orig_pmd; -+ debug_printk("pud=0x%16llx, addr=0x%16llx, end=0x%16llx\n", pud, addr, -+ end); -+ // printk(KERN_ERR "pud_pfn=0x%16llx, pmd_pfn=0x%16llx\n", pud_pfn(*pud), pmd_pfn(*pmd)); -+ do { -+ debug_printk(KERN_ERR "pmd=0x%16llx, pmd_val=0x%16llx\n", pmd, pmd_val(*pmd)); -+ next = pmd_addr_end(addr, end); -+ if (pmd_none(*pmd)) -+ continue; -+ if (pmd_bad(*pmd)) { -+ set_pmd(pmd, __pmd(0)); -+ continue; -+ } -+ koi_unmap_pte_table(ko_mm, pmd, addr, next); -+ } while (pmd++, addr = next, addr != end); -+} -+ -+void koi_unmap_pud_range(struct mm_struct *ko_mm, p4d_t *p4d, -+ unsigned long addr, unsigned long end) -+{ -+ pud_t *pud, *orig_pud; -+ unsigned long next; -+ if (!p4d) -+ return; -+ orig_pud = pud_offset(p4d, addr); -+ pud = orig_pud; -+ debug_printk("p4d=0x%16llx, addr=0x%16llx, end=0x%16llx\n", p4d, addr, -+ end); -+ do { -+ // printk(KERN_ERR "pud=0x%16llx, pud_val=0x%16llx\n", pud, pud_val(*pud)); -+ next = pud_addr_end(addr, end); -+ if (pud_none(*pud)) -+ continue; -+ if (pud_bad(*pud)) { -+ set_pud(pud, __pud(0)); -+ continue; -+ } -+ koi_unmap_pmd_range(ko_mm, pud, addr, next); -+ } while (pud++, addr = next, addr != end); -+ debug_printk("free pud 0x%16llx, p4d=0x%16llx, orig_pud=0x%16llx\n", -+ pud, p4d, orig_pud); -+ // pud_free(ko_mm, (unsigned long)orig_pud & PAGE_MASK); -+} -+ -+void koi_unmap_p4d_range(struct mm_struct *ko_mm, pgd_t *pgd, -+ unsigned long addr, unsigned long end) -+{ -+ p4d_t *p4d, *orig_p4d; -+ unsigned long next; -+ if (!pgd) -+ return; -+ debug_printk("pgd=0x%16llx, addr=0x%16llx, end=0x%16llx\n", pgd, addr, -+ end); -+ orig_p4d = p4d_offset(pgd, addr); -+ p4d = orig_p4d; -+ do { -+ next = p4d_addr_end(addr, end); -+ debug_printk(KERN_ERR "p4d=0x%16llx, p4d_val=0x%16llx, p4d_none=%d\n", p4d, p4d_val(*p4d), p4d_none(*p4d)); -+ if (p4d_none_or_clear_bad(p4d)) { -+ // printk(KERN_ERR "p4d none=%d, p4d_bad=%d\n", p4d_none(*p4d), p4d_bad(*p4d)); -+ continue; -+ } -+ koi_unmap_pud_range(ko_mm, p4d, addr, next); -+ } while (p4d++, addr = next, addr != end); -+} -+ -+void koi_unmap_pagetable(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, -+ unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ pgd_t *pgd = pgd_offset_pgd(ko_pg_dir, addr); -+ debug_printk("freepagetable addr=0x%16llx, end=0x%16llx\n", addr, end); -+ do { -+ next = pgd_addr_end(addr, end); -+ // printk(KERN_ERR "pgd=0x%16llx, pgd_val=0x%16llx\n", pgd, pgd_val(*pgd)); -+ if (pgd_none_or_clear_bad(pgd)) { -+ // printk(KERN_ERR "pgd none\n"); -+ continue; -+ } -+ koi_unmap_p4d_range(ko_mm, pgd, addr, next); -+ } while (pgd++, addr = next, addr != end); -+} -+ -+void koi_remove_pte_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, pmd_t *pmd) -+{ -+ pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); -+ debug_printk( -+ "pte=0x%16llx, page=0x%16llx, pmd=0x%16llx, pmd_val=0x%16llx\n", -+ pte, pte_page(pmd_pte(*pmd)), pmd, pmd_val(*pmd)); -+ // printk(KERN_ERR "pte=0x%16llx, pte_val=0x%16llx, pte_page=0x%16llx\n", pte, pte_val(*pte), pte_page(*pte)); -+ printk(KERN_ERR "free orig_pte=0x%16llx\n", pte); -+ pte_free_kernel(ko_mm, pte); -+ // __free_page((unsigned long)pte_page(pmd_pte(*pmd))); -+ // free_page((unsigned long)pte); -+} -+ -+void koi_remove_pmd_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, pud_t *pud) -+{ -+ pmd_t *orig_pmd = pud_pgtable(*pud); -+ pmd_t *pmd; -+ int i; -+ for (i = 0; i < PTRS_PER_PMD; i++) { -+ pmd = orig_pmd + i; -+ if (pmd_none(*pmd) || pmd_bad(*pmd)) -+ continue; -+ debug_printk("pmd=0x%16llx, pmd_val=0x%16llx\n", pmd, -+ pmd_val(*pmd)); -+ koi_remove_pte_range(ko_mm, ko_pg_dir, pmd); -+ } -+ debug_printk("free pmd=0x%16llx, page=0x%16llx\n", orig_pmd, -+ pte_page(pud_pte(*pud))); -+ printk(KERN_ERR "free orig_pmd=0x%16llx\n", orig_pmd); -+ pmd_free(ko_mm, orig_pmd); -+ // free_page((unsigned long)orig_pmd); -+} -+ -+void koi_remove_pud_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, p4d_t *p4d) -+{ -+ pud_t *orig_pud = p4d_pgtable(*p4d); -+ pud_t *pud; -+ int i; -+ for (i = 0; i < PTRS_PER_PUD; i++) { -+ pud = orig_pud + i; -+ if (pud_none(*pud) || pud_bad(*pud)) -+ continue; -+ debug_printk("pud=0x%16llx, pud_val=0x%16llx\n", pud, -+ pud_val(*pud)); -+ koi_remove_pmd_range(ko_mm, ko_pg_dir, pud); -+ } -+ debug_printk("free pud=0x%16llx, page=0x%16llx\n", orig_pud, -+ pte_page(p4d_pte(*p4d))); -+ printk(KERN_ERR "free orig_pud=0x%16llx\n", orig_pud); -+ pud_free(ko_mm, orig_pud); -+ // free_page((unsigned long)orig_pud); -+} -+ -+void koi_remove_pagetable(struct mm_struct *ko_mm, pgd_t *ko_pg_dir) -+{ -+ pgd_t *pgd; -+ int i; -+ for (i = 0; i < PTRS_PER_PGD; i++) { -+ pgd = ko_pg_dir + i; -+ if (p4d_none(*(p4d_t *)pgd) || p4d_bad(*(p4d_t *)pgd)) -+ continue; -+ debug_printk("pgd=0x%16llx, pgd_val=0x%16llx\n", pgd, -+ pgd_val(*pgd)); -+ koi_remove_pud_range(ko_mm, ko_pg_dir, (p4d_t *)pgd); -+ } -+ debug_printk("free pgd=0x%16llx\n", ko_pg_dir); -+ pgd_free(ko_mm, ko_pg_dir); -+ flush_tlb_all(); -+} -+ -+void koi_destroy_pagetable(struct module *mod) -+{ -+ // int cpu; -+ // unsigned long *ptr; -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_mem_list *mem_node; -+ struct koi_addr_map *addr_map_node; -+ unsigned long bkt; -+ unsigned long vbar; -+ unsigned long flags; -+ asm volatile("mrs %0, VBAR_EL1\n" : "=r"(vbar) :); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ // printk(KERN_ERR "mem node for module: %s not found, maybe destroyed before?\n", -+ // mod->name); -+ return; -+ } -+ spin_lock_irqsave(&target->mod_lock, flags); -+ target->is_valid = false; -+ spin_unlock_irqrestore(&target->mod_lock, flags); -+ -+ spin_lock_irqsave(&koi_mem_htbl_spin_lock, flags); -+ hash_del_rcu(&target->node); -+ call_rcu(&target->rcu, koi_mem_hash_node_free); -+ spin_unlock_irqrestore(&koi_mem_htbl_spin_lock, flags); -+ -+ // free addr_htbl -+ spin_lock(&target->addr_htbl_spin_lock); -+ hash_for_each_rcu (target->addr_htbl, bkt, addr_map_node, node) { -+ hash_del_rcu(&addr_map_node->node); -+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); -+ } -+ spin_unlock(&target->addr_htbl_spin_lock); -+ // free free mem list -+ spin_lock(&target->spin_lock); -+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } -+ spin_unlock(&target->spin_lock); -+ -+ koi_remove_pagetable(target->ko_mm, target->ko_mm->pgd); -+ kfree(target->ko_mm); -+} -+ -+/** -+* koi_create_pagetable - create pagetable for driver -+* @mod: driver module -+* 1.create a new koi_mem_hash_node new_node -+* 2.create page table return the pgd address, init the new_node->pgdp -+* 3.create and init the new_node->ko_mm -+* 4.map swapper_ttbr1 to the newly created pagetable -+* 5.map the interrupt vector table to the newly created pagetable -+* 6.map the init_layout of the module -+* 7.map the core_layout of the module -+* 8.map switch_to_kernel_pgtable into driver view -+* 9.map share memory -+*/ -+void koi_create_pagetable(struct module *mod) -+{ -+ int ret = 0, cpu; -+ unsigned long vbar, addr, ttbr1; -+ pgd_t *pgdp; -+ unsigned long *ptr; -+ struct koi_mem_list *new_mem_node; -+ struct koi_mem_hash_node *new_node = -+ kzalloc(sizeof(struct koi_mem_hash_node), GFP_KERNEL); -+ if (!new_node) { -+ // printk(KERN_ERR "NULL new_node\n"); -+ return; -+ }; -+ if (koi_swapper_ttbr1 == 0) { -+ pgdp = lm_alias(swapper_pg_dir); -+ ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); -+ if (system_supports_cnp() && -+ !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) -+ ttbr1 |= TTBR_CNP_BIT; -+ -+ koi_swapper_ttbr1 = ttbr1; -+ // __WRITE_ONCE(koi_swapper_ttbr1, ttbr1); -+ // koi_set_rdonly(&koi_swapper_ttbr1, swapper_pg_dir); -+ } -+ new_node->pgdp = koi_pgd_alloc(); -+ new_node->ko_mm = -+ kzalloc(sizeof(struct mm_struct) + -+ sizeof(unsigned long) * BITS_TO_LONGS(NR_CPUS), -+ GFP_KERNEL); -+ init_ko_mm(new_node->ko_mm, new_node->pgdp); -+ new_node->mod = mod; -+ koi_save_ttbr(mod, new_node->pgdp, new_node); -+ debug_printk("copying koi_data, start=0x%16llx, end=0x%16llx\n", -+ (unsigned long)__koi_data_start, -+ (unsigned long)__koi_data_end); -+ // copy koi_swapper_ttbr1, which records page dir base for kernel view -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)__koi_data_start, -+ (unsigned long)__koi_data_end, (0)); -+ asm volatile("mrs %0, VBAR_EL1\n" : "=r"(vbar) :); -+ -+ // copy interrupt vectors -+ printk(KERN_ERR -+ "\033[33mcopying interrupt vectors, start=0x%16llx, end=0x%16llx, vbar=0x%16llx\033[0m\n", -+ vbar & PAGE_MASK, (vbar + PAGE_SIZE) & PAGE_MASK, vbar); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, vbar & PAGE_MASK, -+ (vbar + PAGE_SIZE) & PAGE_MASK, (0)); -+ -+ for_each_mod_mem_type(type) { -+ printk(KERN_ERR "\033[33mcopying mem range, start=0x%16llx, end=0x%16llx\033[0m\n", -+ (unsigned long)mod->mem[type].base, -+ (unsigned long)mod->mem[type].base + mod->mem[type].size); -+ if (!mod->mem[type].base || !mod->mem[type].size) { -+ continue; -+ } -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)mod->mem[type].base, (unsigned long)mod->mem[type].base + mod->mem[type].size, (0)); -+ if (ret != 0) -+ printk(KERN_ERR -+ "\033[33mError occured when copying range from 0x%llx to 0x%llx, Eno:%d\033[0m\n", -+ (unsigned long)mod->mem[type].base, -+ (unsigned long)mod->mem[type].base + mod->mem[type].size, -+ ret); -+ } -+ -+ // mapping switch_to_kernel_pgtable into driver view, which is used to switch to kernel view when entering INT -+ printk(KERN_ERR "\033[33mcopying koi_code_range, start=0x%16llx, end=0x%16llx\033[0m\n", -+ (unsigned long)__koi_code_start, -+ (unsigned long)__koi_code_end); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)__koi_code_start, -+ (unsigned long)__koi_code_end, (0)); -+ -+ for_each_possible_cpu (cpu) { -+ ptr = per_cpu(irq_stack_ptr, cpu); -+ debug_printk( -+ "\033[33mirq_stack_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n", -+ cpu, (unsigned long)ptr, -+ (unsigned long)ptr + IRQ_STACK_SIZE); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)ptr, -+ (unsigned long)ptr + IRQ_STACK_SIZE, (0)); -+ } -+ -+ for_each_possible_cpu (cpu) { -+ ptr = per_cpu(koi_irq_current_ttbr1, cpu); -+ debug_printk( -+ "\033[33mirq_current_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n", -+ cpu, (unsigned long)ptr, -+ (unsigned long)ptr + PAGE_SIZE); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)ptr, -+ (unsigned long)ptr + PAGE_SIZE, (0)); -+ } -+ -+#ifdef CONFIG_IEE -+ debug_printk("\033[33miee_si addr=0x%16llx, end=0x%16llx\033[0m\n", -+ (unsigned long)__iee_si_data_start, -+ (unsigned long)__iee_si_text_start); -+ // mapping iee_rwx_gate_entry and iee_si_base to ko's pagetable -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)__iee_si_data_start, -+ (unsigned long)__iee_si_text_start, (0)); -+ debug_printk("\033[33miee_si mapping finished\n"); -+#endif -+ -+ // alloc 16KB memory for new ko, and add it into hashtable -+ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL); -+ if ((void *)addr == NULL) { -+ printk(KERN_ERR "alloc buffer error\n"); -+ } -+ debug_printk( -+ "\033[33mcopying buffer, start=0x%16llx, end=0x%16llx\033[0m\n", -+ addr, addr + THREAD_SIZE); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, -+ addr + THREAD_SIZE, (0)); -+#ifdef CONFIG_IEE -+ iee_rw_gate(IEE_SET_KOI_PGD, new_node->ko_mm->pgd); -+#endif -+ new_mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL); -+ if (new_mem_node == NULL) { -+ printk(KERN_ERR "alloc new_mem_node error\n"); -+ } -+ new_mem_node->addr = addr; -+ new_mem_node->size = THREAD_SIZE; -+ -+ new_node->mem_list_head = -+ (struct list_head)LIST_HEAD_INIT(new_node->mem_list_head); -+ hash_init(new_node->addr_htbl); -+ spin_lock_init(&new_node->addr_htbl_spin_lock); -+ spin_lock_init(&new_node->spin_lock); -+ spin_lock_init(&new_node->mod_lock); -+ new_node->is_valid = true; -+ -+ spin_lock(&new_node->spin_lock); -+ list_add_rcu(&new_mem_node->list, &new_node->mem_list_head); -+ spin_unlock(&new_node->spin_lock); -+ -+ spin_lock(&koi_mem_htbl_spin_lock); -+ hash_add_rcu(koi_mem_htbl, &new_node->node, -+ (unsigned long)new_node->mod); -+ spin_unlock(&koi_mem_htbl_spin_lock); -+ -+ // printk(KERN_DEBUG "mod=0x%16llx, end=0x16llx\n", mod, (unsigned long)mod + sizeof(struct module)); -+ printk(KERN_ERR "[KOI] create pagetable pgd=0x%16llx for module %s\n", new_node->pgdp, mod->name); -+ // printk(KERN_ERR "koi mm=0x%16llx\n", (unsigned long)(new_node->ko_mm)); -+} -+/** -+* koi_mem_alloc -+*@mod: driver module -+*@orig_addr: the starting address of the parameter in kernel -+*@size: the size of the parameter -+*/ -+unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr, -+ unsigned long size) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_mem_list *mem_node; -+ struct koi_addr_map *new_addr_node; -+ unsigned long addr = 0, flags; -+ struct koi_mem_list *new_mem_node; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return 0; -+ } -+ spin_lock_irqsave(&target->spin_lock, flags); -+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { -+ if (mem_node->size >= size) { -+ addr = mem_node->addr; -+ mem_node->size -= size; -+ if (mem_node->size == 0) { -+ list_del_rcu(&mem_node->list); -+ } else { -+ new_mem_node = -+ kmalloc(sizeof(struct koi_mem_list), -+ GFP_ATOMIC); -+ new_mem_node->addr = addr + size; -+ new_mem_node->size = mem_node->size; -+ list_replace_rcu(&mem_node->list, -+ &new_mem_node->list); -+ } -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } -+ } -+ spin_unlock_irqrestore(&target->spin_lock, flags); -+ if (!addr) { -+ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL); -+ if ((void *)addr == NULL) { -+ return 0; -+ } -+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr, -+ addr + THREAD_SIZE, (0)); -+ mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL); -+ if (!mem_node) { -+ printk(KERN_ERR "NULL mem_node\n"); -+ } -+ if (size > THREAD_SIZE) { -+ return 0; -+ } -+ mem_node->addr = addr + size; -+ mem_node->size = THREAD_SIZE - size; -+ spin_lock_irqsave(&target->spin_lock, flags); -+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); -+ spin_unlock_irqrestore(&target->spin_lock, flags); -+ } -+ -+ new_addr_node = kzalloc(sizeof(struct koi_addr_map), GFP_KERNEL); -+ new_addr_node->buffer_addr = addr; -+ new_addr_node->orig_addr = orig_addr; -+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); -+ hash_add_rcu(target->addr_htbl, &new_addr_node->node, -+ new_addr_node->buffer_addr); -+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); -+ return addr; -+} -+EXPORT_SYMBOL(koi_mem_alloc); -+// find the parameter pointer corresponding to the copy -+noinline void *koi_mem_lookup(struct module *mod, unsigned long addr) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_addr_map *addr_map_node; -+ unsigned long orig_addr = addr; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return NULL; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, -+ orig_addr) { -+ if (addr_map_node->buffer_addr == addr) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (addr_map_node) { -+ return (void *)(addr_map_node->orig_addr); -+ } else { -+ return NULL; -+ } -+} -+EXPORT_SYMBOL(koi_mem_lookup); -+/** -+* kio_mem_free - recycle a copy of the copied parameters and synchronize the parameters -+* @mod: driver module -+* @addr: the starting addr of parameter -+* @size: the size of the parameter -+* @is_const: const pointers or not -+* @count: contry the number of parameters -+*/ -+noinline void koi_mem_free(struct module *mod, unsigned long addr, -+ unsigned long size, bool is_const, int count, ...) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_mem_list *mem_node; -+ struct list_head *pos = NULL; -+ struct koi_addr_map *addr_map_node; -+ unsigned long orig_size = size; -+ unsigned long orig_addr = addr; -+ va_list valist; -+ int i; -+ unsigned int offset; -+ unsigned long flags; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, -+ orig_addr) { -+ if (addr_map_node->buffer_addr == orig_addr) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ va_start(valist, count); -+ for (i = 0; i < count; i++) { -+ offset = va_arg(valist, int); -+ *(unsigned long *)(addr_map_node->buffer_addr + offset) = -+ *(unsigned long *)(addr_map_node->orig_addr + offset); -+ } -+ va_end(valist); -+ memcpy((void *)addr_map_node->orig_addr, -+ (void *)addr_map_node->buffer_addr, orig_size); -+ -+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); -+ hlist_del_init_rcu(&addr_map_node->node); -+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); -+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); -+ -+ spin_lock_irqsave(&target->spin_lock, flags); -+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { -+ if (mem_node->addr + mem_node->size == addr) { -+ pos = mem_node->list.prev; -+ addr = mem_node->addr; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size == mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size < mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ break; -+ } -+ } -+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); -+ mem_node->addr = addr; -+ mem_node->size = size; -+ if (pos) -+ list_add_rcu(&mem_node->list, pos); -+ else -+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); -+ spin_unlock_irqrestore(&target->spin_lock, flags); -+} -+EXPORT_SYMBOL(koi_mem_free); -+/** -+* koi_mem_free_callback - used to recycle the copy of parameter. -+*@addr: the address of the parameter -+*@(*func)(void*): callback func, used to release the copy of the parameter pointer -+*/ -+noinline void koi_mem_free_callback(struct module *mod, unsigned long addr, -+ unsigned long size, void (*func)(void *)) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_mem_list *mem_node; -+ struct list_head *pos = NULL; -+ struct koi_addr_map *addr_map_node; -+ unsigned long flags; -+ unsigned long orig_size = size; -+ unsigned long orig_addr = addr; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ // printk("mem node for module: %s not found\n", mod->name); -+ return; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, -+ orig_addr) { -+ if (addr_map_node->buffer_addr == orig_addr) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (addr_map_node != NULL) { -+ memcpy((void *)addr_map_node->orig_addr, -+ (void *)addr_map_node->buffer_addr, orig_size); -+ func((void *)addr_map_node->orig_addr); -+ } else { -+ printk("Cannot find addr_map_node in addr_htbl, maybe addr is in kernel space!!\n"); -+ func((void *)orig_addr); -+ } -+ -+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); -+ if (addr_map_node != NULL) { -+ hlist_del_init_rcu(&addr_map_node->node); -+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); -+ } -+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); -+ spin_lock_irqsave(&target->spin_lock, flags); -+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { -+ if (mem_node->addr + mem_node->size == addr) { -+ pos = mem_node->list.prev; -+ addr = mem_node->addr; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size == mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size < mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ break; -+ } -+ } -+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); -+ mem_node->addr = addr; -+ mem_node->size = size; -+ if (pos) -+ list_add_rcu(&mem_node->list, pos); -+ else -+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); -+ spin_unlock_irqrestore(&target->spin_lock, flags); -+} -+EXPORT_SYMBOL(koi_mem_free_callback); -+ -+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK, (0)); -+ flush_tlb_kernel_range(addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK); -+} -+EXPORT_SYMBOL(koi_map_mem); -+ -+void koi_unmap_mem(struct module *mod, unsigned long addr, unsigned long size) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ if (!addr || ! size) { -+ return; -+ } -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (target == NULL) { -+ printk(KERN_ERR "[KOI UNMAP] mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ koi_unmap_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK); -+ flush_tlb_kernel_range(addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK); -+} -+EXPORT_SYMBOL(koi_unmap_mem); -+/** -+* koi_mem_free_to_user - function 'copy_to_user' in driver space -+*/ -+void koi_mem_free_to_user(struct module *mod, unsigned long addr, -+ unsigned long size) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ struct koi_mem_list *mem_node; -+ struct list_head *pos = NULL; -+ struct koi_addr_map *addr_map_node; -+ unsigned long flags; -+ unsigned long orig_size = size; -+ unsigned long orig_addr = addr; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, -+ orig_addr) { -+ if (addr_map_node->buffer_addr == orig_addr) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (copy_to_user((void *)addr_map_node->orig_addr, -+ (void *)addr_map_node->buffer_addr, orig_size)) { -+ return; -+ } -+ -+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); -+ hlist_del_init_rcu(&addr_map_node->node); -+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); -+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); -+ spin_lock_irqsave(&target->spin_lock, flags); -+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { -+ if (mem_node->addr + mem_node->size == addr) { -+ pos = mem_node->list.prev; -+ addr = mem_node->addr; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size == mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ size += mem_node->size; -+ list_del_rcu(&mem_node->list); -+ call_rcu(&mem_node->rcu, koi_mem_node_free); -+ } else if (addr + size < mem_node->addr) { -+ if (!pos) -+ pos = mem_node->list.prev; -+ break; -+ } -+ } -+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); -+ mem_node->addr = addr; -+ mem_node->size = size; -+ if (pos) -+ list_add_rcu(&mem_node->list, pos); -+ else -+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); -+ spin_unlock_irqrestore(&target->spin_lock, flags); -+} -+EXPORT_SYMBOL(koi_mem_free_to_user); -+// map the driver stack to kernel -+void koi_map_kostack(struct module *mod) -+{ -+} -+EXPORT_SYMBOL(koi_map_kostack); -+ -+#ifndef CONFIG_IEE -+void koi_init_token(struct task_struct *tsk) -+{ -+ struct task_token *token_addr = -+ (struct task_token *)(__phys_to_virt(__pa(tsk)) + KOI_OFFSET); -+ -+ token_addr->koi_kernel_stack = NULL; -+ // token_addr->koi_stack = NULL; -+ // token_addr->koi_stack_base = NULL; -+ token_addr->current_ttbr1 = 0; -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c -index c4999d2d7ce8..601b5bc6a2ae 100644 ---- a/arch/arm64/kernel/mte.c -+++ b/arch/arm64/kernel/mte.c -@@ -79,8 +79,13 @@ int memcmp_pages(struct page *page1, struct page *page2) - static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) - { - /* Enable MTE Sync Mode for EL1. */ -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK, -+ SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); -+#else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, - SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); -+#endif - isb(); - - pr_info_once("MTE: enabled in %s mode at EL1\n", mode); -diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c -index 068e5bb2661b..e545a2df805f 100644 ---- a/arch/arm64/kernel/process.c -+++ b/arch/arm64/kernel/process.c -@@ -471,7 +471,14 @@ DEFINE_PER_CPU(struct task_struct *, __entry_task); - - static void entry_task_switch(struct task_struct *next) - { -+ #if defined(CONFIG_IEE) || defined (CONFIG_KOI) -+ if(next == &init_task) -+ __this_cpu_write(__entry_task, (struct task_struct *)__va(__pa_symbol(next))); -+ else -+ __this_cpu_write(__entry_task, next); -+ #else - __this_cpu_write(__entry_task, next); -+ #endif - } - - /* -@@ -506,11 +513,15 @@ static void erratum_1418040_new_exec(void) - */ - void update_sctlr_el1(u64 sctlr) - { -- /* -+ /* - * EnIA must not be cleared while in the kernel as this is necessary for - * in-kernel PAC. It will be cleared on kernel exit if needed. - */ -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); -+ #else - sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); -+ #endif - - /* ISB required for the kernel uaccess routines when setting TCF0. */ - isb(); -diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c -index 4bca7ad7b5e3..a584557625f7 100644 ---- a/arch/arm64/kernel/proton-pack.c -+++ b/arch/arm64/kernel/proton-pack.c -@@ -551,7 +551,11 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) - return state; - - if (spectre_v4_mitigations_off()) { -+#ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_DSSBS); -+#else - sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); -+#endif - set_pstate_ssbs(1); - return SPECTRE_VULNERABLE; - } -@@ -987,7 +991,11 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) - if (arm64_kernel_unmapped_at_el0()) - return; - -+#ifdef CONFIG_IEE -+ iee_rwx_gate_entry(IEE_WRITE_vbar_el1, v); -+#else - write_sysreg(v, vbar_el1); -+#endif - isb(); - } - -diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c -index d82fd6902ea8..84f4fce0b8bc 100644 ---- a/arch/arm64/kernel/setup.c -+++ b/arch/arm64/kernel/setup.c -@@ -33,6 +33,11 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#include -+#endif -+ - #include - #include - #include -@@ -336,10 +341,31 @@ u64 cpu_logical_map(unsigned int cpu) - return __cpu_logical_map[cpu]; - } - -+#ifdef CONFIG_IEE -+/* used for secure modification of vbar*/ -+extern char __bp_harden_el1_vectors[]; -+/* prepare iee rwx gate for senario of ttbr1=init_pg_dir */ -+static void __init iee_si_init_early(void) -+{ -+ /* prepare data used for iee rwx gate. */ -+ iee_base_swapper_pg_dir = phys_to_ttbr(__pa_symbol(swapper_pg_dir)); -+ iee_base_idmap_pg_dir = phys_to_ttbr(__pa_symbol(idmap_pg_dir)); -+ iee_base_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir)) -+ | FIELD_PREP(TTBR_ASID_MASK, 1); -+ iee_base__bp_harden_el1_vectors = (unsigned long)__bp_harden_el1_vectors; -+ iee_si_tcr = 0; -+} -+#endif -+ - void __init __no_sanitize_address setup_arch(char **cmdline_p) - { - setup_initial_init_mm(_stext, _etext, _edata, _end); - -+ #ifdef CONFIG_IEE -+ init_new_context(&init_task, &init_mm); -+ atomic64_set(&init_mm.context.id, (1UL << get_cpu_asid_bits()) | INIT_ASID); -+ #endif -+ - *cmdline_p = boot_command_line; - - kaslr_init(); -@@ -375,6 +401,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) - */ - local_daif_restore(DAIF_PROCCTX_NOIRQ); - -+#ifdef CONFIG_IEE -+ /* -+ * Map iee si codes to init_pg_dir to run the following -+ * cpu_uninstall_idmap() which writes ttbr0. -+ */ -+ iee_si_init_early(); -+#endif -+ - /* - * TTBR0 is only used for the identity mapping at this stage. Make it - * point to zero page to avoid speculatively fetching new entries. -diff --git a/arch/arm64/kernel/sfi_bpf_arch.c b/arch/arm64/kernel/sfi_bpf_arch.c -new file mode 100644 -index 000000000000..1201cd11a5de ---- /dev/null -+++ b/arch/arm64/kernel/sfi_bpf_arch.c -@@ -0,0 +1,85 @@ -+#include -+#include -+#include -+ -+pte_t *bpf_sfi_get_ptep(u64 addr) -+{ -+ pgd_t *pgdp; -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ -+ /* DEBUG check pgd */ -+ // u64 ttbr1_phy, ttbr1_vir; -+ // __asm__ volatile( -+ // "mrs %0, ttbr1_el1\n\t" -+ // : "=r" (ttbr1_phy) -+ // :: -+ // ); -+ // pr_debug("phy = 0x%llx, after mask = 0x%llx\n", ttbr1_phy, (u64)(ttbr1_phy << 16) >> 16); -+ // ttbr1_vir = (u64)__phys_to_kimg((u64)(ttbr1_phy << 16) >> 16); -+ // pr_info("1, ttbr1_vir = 0x%llx, \n", ttbr1_vir); -+ // pr_info("2, init_mm.pgd = 0x%llx\n", (u64)init_mm.pgd); -+ // pr_info("3, swapper_pg_dir = 0x%llx\n", (u64)swapper_pg_dir); -+ -+ pgdp = pgd_offset(&init_mm, addr); -+ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { -+ pr_err("get pgdp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ p4dp = p4d_offset(pgdp, addr); -+ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) { -+ pr_err("get p4dp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ /* IMPORTANT judge huge page first, then judge table */ -+ pudp = pud_offset(p4dp, addr); -+ if (pud_huge(*pudp)) { -+ // pud is huge page -+ pr_warn("pud of 0x%llx is huge page", addr); -+ // return (pte_t *)pudp; -+ return ERR_PTR(-ENOTSUPP); -+ } -+ if (pud_none(*pudp) || pud_bad(*pudp)) { -+ pr_err("get pudp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ pmdp = pmd_offset(pudp, addr); -+ if (pmd_huge(*pmdp)) { -+ // pmd is huge page -+ pr_warn("pmd of 0x%llx is huge page", addr); -+ // return (pte_t *)pmdp; -+ return ERR_PTR(-ENOTSUPP); -+ } -+ if (pmd_none(*pmdp) || pmd_bad(*pmdp)) { -+ pr_err("get pmdp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ if (!ptep) { -+ pr_err("get ptep of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ return ptep; -+} -+ -+int bpf_sfi_hook_kernel_fault(u64 addr) -+{ -+ pte_t *ptep; -+ -+ ptep = bpf_sfi_get_ptep(addr); -+ if (IS_ERR(ptep)) -+ return PTR_ERR(ptep); -+ -+ if (pte_val(*ptep) & PTE_BPF_SFI_GP) { -+ return true; -+ } -+ else -+ return false; -+} -\ No newline at end of file -diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c -index b7b7afb4a8c7..168a9390d6e9 100644 ---- a/arch/arm64/kernel/traps.c -+++ b/arch/arm64/kernel/traps.c -@@ -902,6 +902,32 @@ const char *esr_get_class_string(unsigned long esr) - return esr_class_str[ESR_ELx_EC(esr)]; - } - -+#ifdef CONFIG_IEE -+extern void arm64_enter_nmi(struct pt_regs *regs); -+static const char *handler[]= { -+ "SP_EL0", -+ "ELR_EL1", -+ "TCR_EL1", -+ "TTBR0 ASID" -+ "IEE_SI" -+}; -+ -+asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, unsigned int esr) -+{ -+ arm64_enter_nmi(regs); -+ -+ console_verbose(); -+ -+ pr_crit("IEE : Bad mode in %s check detected on CPU%d, code 0x%08x -- %s\n", -+ handler[reason], smp_processor_id(), esr, -+ esr_get_class_string(esr)); -+ -+ __show_regs(regs); -+ local_daif_mask(); -+ panic("bad mode"); -+} -+#endif -+ - /* - * bad_el0_sync handles unexpected, but potentially recoverable synchronous - * exceptions taken from EL0. -diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S -index 3cd7e76cc562..008b69a83b91 100644 ---- a/arch/arm64/kernel/vmlinux.lds.S -+++ b/arch/arm64/kernel/vmlinux.lds.S -@@ -134,6 +134,51 @@ jiffies = jiffies_64; - #define UNWIND_DATA_SECTIONS - #endif - -+#ifdef CONFIG_IEE -+#define IEE_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_code_start = .; \ -+ *(.iee.text.header) \ -+ *(.iee.text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_code_end = .; -+#else -+#define IEE_TEXT -+#endif -+ -+#ifdef CONFIG_IEE -+#define IEE_SI_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_data_start = .; \ -+ *(.iee.si_data) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_exec_entry_start = .; \ -+ __iee_si_no_irq = . + (16); \ -+ *(.iee.exec_entry) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_text_start = .; \ -+ *(.iee.si_text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ . += PAGE_SIZE - (24); \ -+ __iee_si_text_end = . + (24); \ -+ *(.iee.exec_exit) \ -+ . = ALIGN(PAGE_SIZE); -+ -+#else -+#define IEE_SI_TEXT -+#endif -+ -+#ifdef CONFIG_KOI -+#define KOI_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_code_start = .; \ -+ *(.koi.text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_code_end = .; -+#else -+#define KOI_TEXT -+#endif -+ - /* - * The size of the PE/COFF section that covers the kernel image, which - * runs from _stext to _edata, must be a round multiple of the PE/COFF -@@ -176,10 +221,13 @@ SECTIONS - SOFTIRQENTRY_TEXT - ENTRY_TEXT - TEXT_TEXT -+ IEE_TEXT - SCHED_TEXT - LOCK_TEXT - KPROBES_TEXT - HYPERVISOR_TEXT -+ IEE_SI_TEXT -+ KOI_TEXT - *(.gnu.warning) - } - -@@ -318,6 +366,13 @@ SECTIONS - . += INIT_DIR_SIZE; - init_pg_end = .; - -+ #ifdef CONFIG_IEE -+ . = ALIGN(PAGE_SIZE*8); -+ init_iee_stack_begin = .; -+ . += PAGE_SIZE*4; -+ init_iee_stack_end = .; -+ #endif -+ - . = ALIGN(SEGMENT_ALIGN); - __pecoff_data_size = ABSOLUTE(. - __initdata_begin); - _end = .; -diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c -index 188197590fc9..b6d8a7d0aeca 100644 ---- a/arch/arm64/mm/context.c -+++ b/arch/arm64/mm/context.c -@@ -17,6 +17,10 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#endif -+ - static u32 asid_bits; - static DEFINE_RAW_SPINLOCK(cpu_asid_lock); - -@@ -39,7 +43,11 @@ static unsigned long *pinned_asid_map; - #define asid2ctxid(asid, genid) ((asid) | (genid)) - - /* Get the ASIDBits supported by the current CPU */ -+#ifdef CONFIG_IEE -+u32 get_cpu_asid_bits(void) -+#else - static u32 get_cpu_asid_bits(void) -+#endif - { - u32 asid; - int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1), -@@ -96,6 +104,16 @@ static void set_reserved_asid_bits(void) - set_kpti_asid_bits(asid_map); - else - bitmap_clear(asid_map, 0, NUM_USER_ASIDS); -+ #ifdef CONFIG_IEE -+ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(asid_map, 0xaa, len); -+ __set_bit(INIT_ASID, asid_map); -+ #else -+ #ifdef CONFIG_KOI -+ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(asid_map, 0xaa, len); -+ #endif -+ #endif - } - - #define asid_gen_match(asid) \ -@@ -212,6 +230,38 @@ static u64 new_context(struct mm_struct *mm) - return asid2ctxid(asid, generation); - } - -+#ifdef CONFIG_KOI -+/* -+ * This function is used to check and allocate ASID for ko's pgd -+ * The mm MUST point to the isolated kos' mm_struct, other behaviours are undefined. -+ */ -+void koi_check_and_switch_context(struct mm_struct *mm) { -+ u64 asid = atomic64_read(&mm->context.id); -+ u64 old_active_asid; -+ unsigned long flags; -+ unsigned int cpu; -+ -+ old_active_asid = atomic64_read(this_cpu_ptr(&active_asids)); -+ if (old_active_asid && asid_gen_match(asid) && atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_asids), old_active_asid, asid)) { -+ return; -+ } -+ -+ raw_spin_lock_irqsave(&cpu_asid_lock, flags); -+ asid = atomic64_read(&mm->context.id); -+ if (!asid_gen_match(asid)) { -+ asid = new_context(mm); -+ atomic64_set(&mm->context.id, asid); -+ } -+ -+ cpu = smp_processor_id(); -+ if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) -+ local_flush_tlb_all(); -+ -+ atomic64_set(this_cpu_ptr(&active_asids), asid); -+ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); -+} -+#endif -+ - void check_and_switch_context(struct mm_struct *mm) - { - unsigned long flags; -@@ -348,7 +398,9 @@ asmlinkage void post_ttbr_update_workaround(void) - - void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) - { -+ #ifndef CONFIG_IEE - unsigned long ttbr1 = read_sysreg(ttbr1_el1); -+ #endif - unsigned long asid = ASID(mm); - unsigned long ttbr0 = phys_to_ttbr(pgd_phys); - -@@ -360,14 +412,28 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) - if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN)) - ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); - -- /* Set ASID in TTBR1 since TCR.A1 is set */ -+ #ifdef CONFIG_IEE -+ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1); -+ iee_rwx_gate_entry(IEE_CONTEXT_SWITCH, ttbr0); -+ // TODO : if defined CONFIG_IEE and defined CONFIG_KOI -+ #else -+ /* Set ASID in TTBR0 since TCR.A1 is set 0*/ -+ -+ #ifdef CONFIG_KOI -+ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1); -+ ttbr1 &= ~TTBR_ASID_MASK; -+ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); -+ #else - ttbr1 &= ~TTBR_ASID_MASK; - ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); -- -+ -+ #endif - cpu_set_reserved_ttbr0_nosync(); - write_sysreg(ttbr1, ttbr1_el1); - write_sysreg(ttbr0, ttbr0_el1); - isb(); -+ #endif -+ - post_ttbr_update_workaround(); - } - -@@ -375,11 +441,28 @@ static int asids_update_limit(void) - { - unsigned long num_available_asids = NUM_USER_ASIDS; - -- if (arm64_kernel_unmapped_at_el0()) { -- num_available_asids /= 2; -- if (pinned_asid_map) -- set_kpti_asid_bits(pinned_asid_map); -+ if (arm64_kernel_unmapped_at_el0()) { -+ num_available_asids /= 2; -+ if (pinned_asid_map) -+ set_kpti_asid_bits(pinned_asid_map); - } -+ #if defined(CONFIG_IEE) -+ num_available_asids /= 2; -+ if (pinned_asid_map) { -+ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(pinned_asid_map, 0xaa, len); -+ __set_bit(INIT_ASID, pinned_asid_map); -+ } -+ #else -+ #ifdef CONFIG_KOI -+ num_available_asids /= 2; -+ if (pinned_asid_map) { -+ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(pinned_asid_map, 0xaa, len); -+ } -+ #endif -+ #endif -+ - /* - * Expect allocation after rollover to fail if we don't have at least - * one more ASID than CPUs. ASID #0 is reserved for init_mm. -@@ -400,6 +483,10 @@ arch_initcall(asids_update_limit); - - static int asids_init(void) - { -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ unsigned int len; -+ #endif -+ - asid_bits = get_cpu_asid_bits(); - atomic64_set(&asid_generation, ASID_FIRST_VERSION); - asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); -@@ -410,13 +497,25 @@ static int asids_init(void) - pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); - nr_pinned_asids = 0; - -- /* -- * We cannot call set_reserved_asid_bits() here because CPU -- * caps are not finalized yet, so it is safer to assume KPTI -- * and reserve kernel ASID's from beginning. -- */ -- if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) -- set_kpti_asid_bits(asid_map); -+ #ifdef CONFIG_IEE -+ len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(asid_map, 0xaa, len); -+ __set_bit(INIT_ASID, asid_map); -+ #else -+ #ifdef CONFIG_KOI -+ len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); -+ memset(asid_map, 0xaa, len); -+ #else -+ /* -+ * We cannot call set_reserved_asid_bits() here because CPU -+ * caps are not finalized yet, so it is safer to assume KPTI -+ * and reserve kernel ASID's from beginning. -+ */ -+ if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) -+ set_kpti_asid_bits(asid_map); -+ #endif -+ #endif -+ - return 0; - } - early_initcall(asids_init); -diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c -index 4ea07caba71c..00f85acfc671 100644 ---- a/arch/arm64/mm/fault.c -+++ b/arch/arm64/mm/fault.c -@@ -26,6 +26,9 @@ - #include - #include - #include -+#ifdef CONFIG_HIVE -+#include -+#endif - - #include - #include -@@ -44,6 +47,10 @@ - #include - #include - -+#ifdef CONFIG_HIVE -+extern int bpf_sfi_hook_kernel_fault(u64 addr); -+#endif -+ - static int sysctl_machine_check_safe = IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC); - - #ifdef CONFIG_ARCH_HAS_COPY_MC -@@ -261,7 +268,15 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, - pteval ^= PTE_RDONLY; - pteval |= pte_val(entry); - pteval ^= PTE_RDONLY; -+ #ifdef CONFIG_KOI -+ if (pteval & PTE_VALID) -+ pteval |= PTE_NG; -+ #endif -+ #ifdef CONFIG_PTP -+ pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval); -+ #else - pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); -+ #endif - } while (pteval != old_pteval); - - /* Invalidate a stale read-only entry */ -@@ -335,6 +350,14 @@ static void die_kernel_fault(const char *msg, unsigned long addr, - { - bust_spinlocks(1); - -+// #ifdef CONFIG_HIVE -+// if (bpf_sfi_hook_kernel_fault(addr)) { -+// pr_err("detected bpf sfi guard page %lx access\n", addr); -+// regs->pc += 4; -+// return; -+// } -+// #endif -+ - pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg, - addr); - -@@ -376,8 +399,13 @@ static void do_tag_recovery(unsigned long addr, unsigned long esr, - * It will be done lazily on the other CPUs when they will hit a - * tag fault. - */ -+ #ifdef CONFIG_IEE -+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK, -+ SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); -+ #else - sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, - SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); -+ #endif - isb(); - } - -diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c -index a55b36d04590..fb80a7251a8e 100644 ---- a/arch/arm64/mm/fixmap.c -+++ b/arch/arm64/mm/fixmap.c -@@ -32,6 +32,22 @@ static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; - static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; - static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; - -+#ifdef CONFIG_IEE -+void *bm_pte_addr = (void *)bm_pte; -+void *bm_pmd_addr = (void *)bm_pmd; -+void *bm_pud_addr = (void *)bm_pud; -+#endif -+ -+#ifdef CONFIG_PTP -+extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot); -+extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot); -+extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, -+ pmdval_t prot); -+ -+extern void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d); -+#define set_pgd_init(pgdptr, pgdval) iee_set_p4d_pre_init((p4d_t *)(pgdptr), (p4d_t) { pgdval }) -+#endif -+ - static inline pte_t *fixmap_pte(unsigned long addr) - { - return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; -@@ -44,8 +60,12 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) - - if (pmd_none(pmd)) { - ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; -+ #ifdef CONFIG_PTP -+ __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE | PMD_TABLE_AF); -+ #else - __pmd_populate(pmdp, __pa_symbol(ptep), - PMD_TYPE_TABLE | PMD_TABLE_AF); -+ #endif - } - } - -@@ -56,10 +76,14 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, - pud_t pud = READ_ONCE(*pudp); - pmd_t *pmdp; - -- if (pud_none(pud)) -+ if (pud_none(pud)) { -+ #ifdef CONFIG_PTP -+ __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE | PUD_TABLE_AF); -+ #else - __pud_populate(pudp, __pa_symbol(bm_pmd), - PUD_TYPE_TABLE | PUD_TABLE_AF); -- -+ #endif -+ } - pmdp = pmd_offset_kimg(pudp, addr); - do { - next = pmd_addr_end(addr, end); -@@ -84,9 +108,14 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, - BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); - } - -- if (p4d_none(p4d)) -+ if (p4d_none(p4d)) { -+ #ifdef CONFIG_PTP -+ __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE | P4D_TABLE_AF); -+ #else - __p4d_populate(p4dp, __pa_symbol(bm_pud), - P4D_TYPE_TABLE | P4D_TABLE_AF); -+ #endif -+ } - - pudp = pud_offset_kimg(p4dp, addr); - early_fixmap_init_pmd(pudp, addr, end); -@@ -109,6 +138,27 @@ void __init early_fixmap_init(void) - early_fixmap_init_pud(p4dp, addr, end); - } - -+#ifdef CONFIG_PTP -+extern void iee_set_pte_pre_init(pte_t *ptep, pte_t pte); -+void __iee_set_fixmap_pre_init(enum fixed_addresses idx, -+ phys_addr_t phys, pgprot_t flags) -+{ -+ unsigned long addr = __fix_to_virt(idx); -+ pte_t *ptep; -+ -+ BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); -+ -+ ptep = fixmap_pte(addr); -+ -+ if (pgprot_val(flags)) { -+ iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ } else { -+ iee_set_pte_pre_init(ptep, __pte(0)); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); -+ } -+} -+#endif -+ - /* - * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we - * ever need to use IPIs for TLB broadcasting, then we're in trouble here. -@@ -124,9 +174,17 @@ void __set_fixmap(enum fixed_addresses idx, - ptep = fixmap_pte(addr); - - if (pgprot_val(flags)) { -+ #ifdef CONFIG_PTP -+ iee_set_bm_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ #else - __set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ #endif - } else { -+ #ifdef CONFIG_PTP -+ iee_set_bm_pte(ptep, __pte(0)); -+ #else - __pte_clear(&init_mm, addr, ptep); -+ #endif - flush_tlb_kernel_range(addr, addr+PAGE_SIZE); - } - } -@@ -182,8 +240,13 @@ void __init fixmap_copy(pgd_t *pgdir) - * live in the carveout for the swapper_pg_dir. We can simply - * re-use the existing dir for the fixmap. - */ -+ #ifdef CONFIG_PTP -+ set_pgd_init(pgd_offset_pgd(pgdir, FIXADDR_TOT_START), -+ READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START))); -+ #else - set_pgd(pgd_offset_pgd(pgdir, FIXADDR_TOT_START), - READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START))); -+ #endif - } else if (CONFIG_PGTABLE_LEVELS > 3) { - pgd_t *bm_pgdp; - p4d_t *bm_p4dp; -@@ -197,9 +260,15 @@ void __init fixmap_copy(pgd_t *pgdir) - BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); - bm_pgdp = pgd_offset_pgd(pgdir, FIXADDR_TOT_START); - bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_TOT_START); -+ #ifdef CONFIG_PTP -+ bm_pudp = pud_set_fixmap_offset_init(bm_p4dp, FIXADDR_TOT_START); -+ __iee_pud_populate_pre_init(bm_pudp, __pa(lm_alias(bm_pmd)), PMD_TYPE_TABLE); -+ pud_clear_fixmap_init(); -+ #else - bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_TOT_START); - pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd)); - pud_clear_fixmap(); -+ #endif - } else { - BUG(); - } -diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c -index 66a7fff9f373..fdd5ad51307d 100644 ---- a/arch/arm64/mm/init.c -+++ b/arch/arm64/mm/init.c -@@ -56,8 +56,20 @@ - * that cannot be mistaken for a real physical address. - */ - s64 memstart_addr __ro_after_init = -1; -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+s64 memstart_addr_init __ro_after_init = -1; -+#endif -+#ifdef CONFIG_KOI -+s64 koi_offset __ro_after_init = -1; -+EXPORT_SYMBOL(koi_offset); -+#endif -+#ifdef CONFIG_IEE -+s64 iee_offset __ro_after_init = -1; -+extern s64 iee_si_offset; -+#endif - EXPORT_SYMBOL(memstart_addr); - -+ - /* - * If the corresponding config options are enabled, we create both ZONE_DMA - * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory -@@ -422,7 +434,11 @@ early_param("memmap", parse_memmap_opt); - - void __init arm64_memblock_init(void) - { -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ s64 linear_region_size = BIT(vabits_actual - 2); -+ #else - s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); -+ #endif - - /* - * Corner case: 52-bit VA capable systems running KVM in nVHE mode may -@@ -439,13 +455,24 @@ void __init arm64_memblock_init(void) - } - - /* Remove memory above our supported physical address size */ -+ #ifdef CONFIG_IEE -+ // If config iee, phys size can not be above 0x400000000000 -+ if(__pa_symbol(_end) > BIT_ULL(vabits_actual - 2)) -+ panic("Image on too high phys mem.\n"); -+ else -+ memblock_remove(BIT_ULL(vabits_actual - 2), ULLONG_MAX); -+ #else - memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); -+ #endif - - /* - * Select a suitable value for the base of physical memory. - */ - memstart_addr = round_down(memblock_start_of_DRAM(), - ARM64_MEMSTART_ALIGN); -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ memstart_addr_init = memstart_addr; -+ #endif - - if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size) - pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n"); -@@ -532,6 +559,15 @@ void __init arm64_memblock_init(void) - ((range * memstart_offset_seed) >> 16); - } - } -+ -+ #ifdef CONFIG_KOI -+ koi_offset = memstart_addr - memstart_addr_init + ((unsigned long)BIT(vabits_actual - 2)); -+ #endif -+ #ifdef CONFIG_IEE -+ iee_offset = memstart_addr - memstart_addr_init + ((unsigned long)BIT(vabits_actual - 2)); -+ iee_si_offset = iee_offset; -+ #endif -+ //printk(KERN_ERR "koi_offset: 0x%16llx\n", koi_offset); - - /* - * Register the kernel text, kernel data, initrd, and initial -@@ -579,6 +615,10 @@ void __init bootmem_init(void) - * done after the fixed reservations - */ - sparse_init(); -+#ifdef CONFIG_PTP -+ extern void iee_ptdesc_sparse_init(void); -+ iee_ptdesc_sparse_init(); -+#endif - zone_sizes_init(); - - /* -diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c -index c846cc54e9ce..1148d95e4888 100644 ---- a/arch/arm64/mm/mmu.c -+++ b/arch/arm64/mm/mmu.c -@@ -6,6 +6,7 @@ - * Copyright (C) 2012 ARM Ltd. - */ - -+#include "asm/pgtable.h" - #include - #include - #include -@@ -41,6 +42,15 @@ - #include - #include - #include -+#ifdef CONFIG_IEE -+#include -+#include -+#include -+#include -+#endif -+#ifdef CONFIG_PTP -+#include -+#endif - - #define NO_BLOCK_MAPPINGS BIT(0) - #define NO_CONT_MAPPINGS BIT(1) -@@ -77,8 +87,236 @@ EXPORT_SYMBOL(empty_zero_page); - static DEFINE_SPINLOCK(swapper_pgdir_lock); - static DEFINE_MUTEX(fixmap_lock); - -+#ifdef CONFIG_IEE -+extern struct cred init_cred; -+ -+extern unsigned long __iee_si_data_start[]; -+extern unsigned long __iee_exec_entry_start[]; -+extern unsigned long __iee_si_text_start[]; -+extern unsigned long __iee_si_text_end[]; -+ -+extern void *bm_pte_addr; -+extern void *bm_pmd_addr; -+extern void *bm_pud_addr; -+ -+void *init_token_page_vaddr; -+ -+#ifdef CONFIG_PTP -+ -+/* Funcs to set pgtable before iee initialized. */ -+static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) -+{ -+ pgd_t *fixmap_pgdp; -+ -+ spin_lock(&swapper_pgdir_lock); -+ fixmap_pgdp = pgd_set_fixmap_init(__pa_symbol(pgdp)); -+ WRITE_ONCE(*fixmap_pgdp, pgd); -+ /* -+ * We need dsb(ishst) here to ensure the page-table-walker sees -+ * our new entry before set_p?d() returns. The fixmap's -+ * flush_tlb_kernel_range() via clear_fixmap() does this for us. -+ */ -+ pgd_clear_fixmap_init(); -+ spin_unlock(&swapper_pgdir_lock); -+} -+ -+void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) -+{ -+ if (in_swapper_pgdir(p4dp)) { -+ iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); -+ return; -+ } -+ -+ WRITE_ONCE(*p4dp, p4d); -+ dsb(ishst); -+ isb(); -+} -+ -+static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud) -+{ -+#ifdef __PAGETABLE_PUD_FOLDED -+ if (in_swapper_pgdir(pudp)) { -+ iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud))); -+ return; -+ } -+#endif /* __PAGETABLE_PUD_FOLDED */ -+#ifdef CONFIG_KOI -+ pudval_t val = pud_val(pud); -+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { -+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. -+ pud = __pud(val | PMD_SECT_NG); -+ } -+#endif -+ WRITE_ONCE(*pudp, pud); -+ -+ if (pud_valid(pud)) { -+ dsb(ishst); -+ isb(); -+ } -+} -+ -+static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) -+{ -+#ifdef __PAGETABLE_PMD_FOLDED -+ if (in_swapper_pgdir(pmdp)) { -+ iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd))); -+ return; -+ } -+#endif /* __PAGETABLE_PMD_FOLDED */ -+#ifdef CONFIG_KOI -+ pmdval_t val = pmd_val(pmd); -+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { -+ pmd = __pmd(val | PMD_SECT_NG); -+ } -+#endif -+ WRITE_ONCE(*pmdp, pmd); -+ -+ if (pmd_valid(pmd)) { -+ dsb(ishst); -+ isb(); -+ } -+} -+ -+ -+void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) -+{ -+ iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); -+} -+ -+void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) -+{ -+ iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); -+} -+ -+void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, -+ pmdval_t prot) -+{ -+ iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); -+} -+ -+/* Funcs to set fixmap before iee initialized. */ -+bool pgattr_change_is_safe(u64 old, u64 new); -+ -+static int iee_pmd_set_huge_fixmap(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) -+{ -+ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); -+ -+ /* Only allow permission changes for now */ -+ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), -+ pmd_val(new_pmd))) -+ return 0; -+ -+ VM_BUG_ON(phys & ~PMD_MASK); -+ iee_set_fixmap_pmd_pre_init(pmdp, new_pmd); -+ return 1; -+} -+ -+static inline void __iee_pmd_populate_fixmap(pmd_t *pmdp, phys_addr_t ptep, -+ pmdval_t prot) -+{ -+ iee_set_fixmap_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); -+} -+ -+static inline void __iee_pud_populate_fixmap(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) -+{ -+ iee_set_fixmap_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); -+} -+#endif /* END CONFIG_PTP*/ -+ -+void iee_set_pte_pre_init(pte_t *ptep, pte_t pte) -+{ -+#ifdef CONFIG_KOI -+ if (pte_valid(pte)) { -+ pte = __pte(pte_val(pte) | PTE_NG); -+ } -+#endif -+ WRITE_ONCE(*ptep, pte); -+ -+ /* -+ * Only if the new pte is valid and kernel, otherwise TLB maintenance -+ * or update_mmu_cache() have the necessary barriers. -+ */ -+ if (pte_valid_not_user(pte)) { -+ dsb(ishst); -+ isb(); -+ } -+} -+ -+static void __init iee_set_token_page_valid_pre_init(void *token, void *new) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(ptep, pte); -+ #else -+ set_pte(ptep, pte); -+ #endif -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); -+ isb(); -+} -+#endif /* END CONFIG_IEE*/ -+ -+#if defined(CONFIG_KOI) && !defined(CONFIG_IEE) -+ -+void koi_add_page_mapping(void *token, void *new) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); -+ set_pte(ptep, pte); -+ dsb(ishst); -+ isb(); -+ -+ flush_tlb_kernel_range((unsigned long)new, (unsigned long)new+PAGE_SIZE); -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)token+PAGE_SIZE); -+ isb(); -+} -+ -+void koi_remove_page_mapping(unsigned long token, void *__unused, unsigned long order) { -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, token); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, token); -+ -+ pud_t *pudp = pud_offset(p4dp, token); -+ -+ pmd_t *pmdp = pmd_offset(pudp, token); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, token); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~((unsigned long)0x1)) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token - KOI_OFFSET))); -+ set_pte(ptep, pte); -+ flush_tlb_kernel_range(token, token+PAGE_SIZE); -+ isb(); -+} -+#endif -+ - void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) - { -+ #ifdef CONFIG_PTP -+ spin_lock(&swapper_pgdir_lock); -+ iee_rw_gate(IEE_OP_SET_SWAPPER_PGD, pgdp, pgd); -+ spin_unlock(&swapper_pgdir_lock); -+ #else - pgd_t *fixmap_pgdp; - - spin_lock(&swapper_pgdir_lock); -@@ -91,6 +329,7 @@ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) - */ - pgd_clear_fixmap(); - spin_unlock(&swapper_pgdir_lock); -+ #endif - } - - pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, -@@ -104,6 +343,34 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, - } - EXPORT_SYMBOL(phys_mem_access_prot); - -+#ifdef CONFIG_PTP -+phys_addr_t __init early_pgtable_alloc(int shift) -+{ -+ phys_addr_t phys; -+ void *ptr; -+ -+ phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); -+ if (!phys) -+ panic("Failed to allocate page table page\n"); -+ -+ /* -+ * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE -+ * slot will be free, so we can (ab)use the FIX_PTE slot to initialise -+ * any level of table. -+ */ -+ ptr = pte_set_fixmap_init(phys); -+ -+ memset(ptr, 0, PAGE_SIZE); -+ -+ /* -+ * Implicit barriers also ensure the zeroed page is visible to the page -+ * table walker -+ */ -+ pte_clear_fixmap_init(); -+ -+ return phys; -+} -+#else - static phys_addr_t __init early_pgtable_alloc(int shift) - { - phys_addr_t phys; -@@ -119,7 +386,11 @@ static phys_addr_t __init early_pgtable_alloc(int shift) - * slot will be free, so we can (ab)use the FIX_PTE slot to initialise - * any level of table. - */ -+ #ifdef CONFIG_PTP -+ ptr = pte_set_fixmap_init(phys); -+ #else - ptr = pte_set_fixmap(phys); -+ #endif - - memset(ptr, 0, PAGE_SIZE); - -@@ -127,10 +398,15 @@ static phys_addr_t __init early_pgtable_alloc(int shift) - * Implicit barriers also ensure the zeroed page is visible to the page - * table walker - */ -+ #ifdef CONFIG_PTP -+ pte_clear_fixmap_init(); -+ #else - pte_clear_fixmap(); -+ #endif - - return phys; - } -+#endif - - bool pgattr_change_is_safe(u64 old, u64 new) - { -@@ -179,7 +455,11 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, - do { - pte_t old_pte = __ptep_get(ptep); - -+ #ifdef CONFIG_PTP -+ iee_set_fixmap_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot)); -+ #else - __set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); -+ #endif - - /* - * After the PTE entry has been populated once, we -@@ -212,7 +492,11 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, - pmdval |= PMD_TABLE_PXN; - BUG_ON(!pgtable_alloc); - pte_phys = pgtable_alloc(PAGE_SHIFT); -+ #ifdef CONFIG_PTP -+ __iee_pmd_populate_fixmap(pmdp, pte_phys, pmdval); -+ #else - __pmd_populate(pmdp, pte_phys, pmdval); -+ #endif - pmd = READ_ONCE(*pmdp); - } - BUG_ON(pmd_bad(pmd)); -@@ -247,9 +531,17 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, - next = pmd_addr_end(addr, end); - - /* try section mapping first */ -+ #ifdef CONFIG_IEE -+ if (!((pmd_val(old_pmd) & PTE_VALID) && (pmd_val(old_pmd) & PTE_TABLE_BIT)) && (((addr | next | phys) & ~PMD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0)) { -+ #else - if (((addr | next | phys) & ~PMD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { -+ #endif -+ #ifdef CONFIG_PTP -+ iee_pmd_set_huge_fixmap(pmdp, phys, prot); -+ #else - pmd_set_huge(pmdp, phys, prot); -+ #endif - - /* - * After the PMD entry has been populated once, we -@@ -290,7 +582,11 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, - pudval |= PUD_TABLE_PXN; - BUG_ON(!pgtable_alloc); - pmd_phys = pgtable_alloc(PMD_SHIFT); -- __pud_populate(pudp, pmd_phys, pudval); -+ #ifdef CONFIG_PTP -+ __iee_pud_populate_fixmap(pudp, pmd_phys, PUD_TYPE_TABLE); -+ #else -+ __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); -+ #endif - pud = READ_ONCE(*pudp); - } - BUG_ON(pud_bad(pud)); -@@ -343,6 +639,13 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, - /* - * For 4K granule only, attempt to put down a 1GB block - */ -+ #ifdef CONFIG_IEE -+ alloc_init_cont_pmd(pudp, addr, next, phys, prot, -+ pgtable_alloc, flags); -+ -+ BUG_ON(pud_val(old_pud) != 0 && -+ pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); -+ #else - if (pud_sect_supported() && - ((addr | next | phys) & ~PUD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { -@@ -361,6 +664,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, - BUG_ON(pud_val(old_pud) != 0 && - pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); - } -+ #endif - phys += next - addr; - } while (pudp++, addr = next, addr != end); - -@@ -375,6 +679,10 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, - { - unsigned long addr, end, next; - pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); -+ #ifdef CONFIG_IEE -+ p4d_t *p4dp; -+ p4d_t p4d; -+ #endif - - /* - * If the virtual and physical address don't have the same offset -@@ -391,10 +699,14 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, - next = pgd_addr_end(addr, end); - alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, - flags); -+ #ifdef CONFIG_IEE -+ p4dp = p4d_offset(pgdp, addr); -+ p4d = READ_ONCE(*p4dp); -+ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE)); -+ #endif - phys += next - addr; - } while (pgdp++, addr = next, addr != end); - } -- - static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, -@@ -414,103 +726,363 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, - phys_addr_t (*pgtable_alloc)(int), int flags); - #endif - --static phys_addr_t __pgd_pgtable_alloc(int shift) -+#ifdef CONFIG_PTP -+static int __init iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) - { -- void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); -- BUG_ON(!ptr); -+ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); - -- /* Ensure the zeroed page is visible to the page table walker */ -- dsb(ishst); -- return __pa(ptr); -+ /* Only allow permission changes for now */ -+ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), -+ pmd_val(new_pmd))) -+ return 0; -+ -+ VM_BUG_ON(phys & ~PMD_MASK); -+ iee_set_pmd_pre_init(pmdp, new_pmd); -+ return 1; - } - --static phys_addr_t pgd_pgtable_alloc(int shift) -+static __init void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end, -+ phys_addr_t phys, pgprot_t prot) - { -- phys_addr_t pa = __pgd_pgtable_alloc(shift); -- struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); -+ pte_t *ptep; - -- /* -- * Call proper page table ctor in case later we need to -- * call core mm functions like apply_to_page_range() on -- * this pre-allocated page table. -- * -- * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is -- * folded, and if so pagetable_pte_ctor() becomes nop. -- */ -- if (shift == PAGE_SHIFT) -- BUG_ON(!pagetable_pte_ctor(ptdesc)); -- else if (shift == PMD_SHIFT) -- BUG_ON(!pagetable_pmd_ctor(ptdesc)); -+ ptep = pte_set_fixmap_offset_init(pmdp, addr); -+ do { -+ pte_t old_pte = READ_ONCE(*ptep); - -- return pa; --} -+ iee_set_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot)); - --/* -- * This function can only be used to modify existing table entries, -- * without allocating new levels of table. Note that this permits the -- * creation of new section or page entries. -- */ --void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, -- phys_addr_t size, pgprot_t prot) --{ -- if (virt < PAGE_OFFSET) { -- pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", -- &phys, virt); -- return; -- } -- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, -- NO_CONT_MAPPINGS); -+ /* -+ * After the PTE entry has been populated once, we -+ * only allow updates to the permission attributes. -+ */ -+ BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), -+ READ_ONCE(pte_val(*ptep)))); -+ -+ phys += PAGE_SIZE; -+ } while (ptep++, addr += PAGE_SIZE, addr != end); -+ -+ pte_clear_fixmap_init(); - } - --void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, -- unsigned long virt, phys_addr_t size, -- pgprot_t prot, bool page_mappings_only) -+static __init void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr, -+ unsigned long end, phys_addr_t phys, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) - { -- int flags = 0; -+ unsigned long next; -+ pmd_t pmd = READ_ONCE(*pmdp); - -- BUG_ON(mm == &init_mm); -+ BUG_ON(pmd_sect(pmd)); -+ if (pmd_none(pmd)) { -+ pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN; -+ phys_addr_t pte_phys; - -- if (page_mappings_only) -- flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; -+ if (flags & NO_EXEC_MAPPINGS) -+ pmdval |= PMD_TABLE_PXN; -+ BUG_ON(!pgtable_alloc); -+ pte_phys = pgtable_alloc(PAGE_SHIFT); -+ __iee_pmd_populate_pre_init(pmdp, pte_phys, pmdval); -+ pmd = READ_ONCE(*pmdp); -+ } -+ BUG_ON(pmd_bad(pmd)); - -- __create_pgd_mapping(mm->pgd, phys, virt, size, prot, -- pgd_pgtable_alloc, flags); --} -+ do { -+ pgprot_t __prot = prot; - --static void update_mapping_prot(phys_addr_t phys, unsigned long virt, -- phys_addr_t size, pgprot_t prot) --{ -- if (virt < PAGE_OFFSET) { -- pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", -- &phys, virt); -- return; -- } -+ next = pte_cont_addr_end(addr, end); - -- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, -- NO_CONT_MAPPINGS); -+ /* use a contiguous mapping if the range is suitably aligned */ -+ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && -+ (flags & NO_CONT_MAPPINGS) == 0) -+ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); - -- /* flush the TLBs after updating live kernel mappings */ -- flush_tlb_kernel_range(virt, virt + size); --} -+ iee_init_pte_pre_init(pmdp, addr, next, phys, __prot); - --static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, -- phys_addr_t end, pgprot_t prot, int flags) --{ -- __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, -- prot, early_pgtable_alloc, flags); -+ phys += next - addr; -+ } while (addr = next, addr != end); - } - --void __init mark_linear_text_alias_ro(void) -+static __init void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end, -+ phys_addr_t phys, pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), int flags) - { -- /* -- * Remove the write permissions from the linear alias of .text/.rodata -- */ -- update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), -- (unsigned long)__init_begin - (unsigned long)_stext, -- PAGE_KERNEL_RO); --} -+ unsigned long next; -+ pmd_t *pmdp; - --#ifdef CONFIG_KFENCE -+ pmdp = pmd_set_fixmap_offset_init(pudp, addr); -+ do { -+ pmd_t old_pmd = READ_ONCE(*pmdp); -+ -+ next = pmd_addr_end(addr, end); -+ -+ /* try section mapping first */ -+ if (((addr | next | phys) & ~PMD_MASK) == 0 && -+ (flags & NO_BLOCK_MAPPINGS) == 0) { -+ iee_pmd_set_huge_pre_init(pmdp, phys, prot); -+ -+ /* -+ * After the PMD entry has been populated once, we -+ * only allow updates to the permission attributes. -+ */ -+ BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), -+ READ_ONCE(pmd_val(*pmdp)))); -+ } else { -+ iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot, -+ pgtable_alloc, flags); -+ -+ BUG_ON(pmd_val(old_pmd) != 0 && -+ pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); -+ } -+ phys += next - addr; -+ } while (pmdp++, addr = next, addr != end); -+ -+ pmd_clear_fixmap_init(); -+} -+ -+static __init void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr, -+ unsigned long end, phys_addr_t phys, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), int flags) -+{ -+ unsigned long next; -+ pud_t pud = READ_ONCE(*pudp); -+ -+ /* -+ * Check for initial section mappings in the pgd/pud. -+ */ -+ BUG_ON(pud_sect(pud)); -+ if (pud_none(pud)) { -+ pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN; -+ phys_addr_t pmd_phys; -+ -+ if (flags & NO_EXEC_MAPPINGS) -+ pudval |= PUD_TABLE_PXN; -+ BUG_ON(!pgtable_alloc); -+ pmd_phys = pgtable_alloc(PMD_SHIFT); -+ __iee_pud_populate_pre_init(pudp, pmd_phys, pudval); -+ pud = READ_ONCE(*pudp); -+ } -+ BUG_ON(pud_bad(pud)); -+ -+ do { -+ pgprot_t __prot = prot; -+ -+ next = pmd_cont_addr_end(addr, end); -+ -+ /* use a contiguous mapping if the range is suitably aligned */ -+ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && -+ (flags & NO_CONT_MAPPINGS) == 0) -+ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); -+ -+ iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags); -+ -+ phys += next - addr; -+ } while (addr = next, addr != end); -+} -+ -+static __init void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, -+ phys_addr_t phys, pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ unsigned long next; -+ pud_t *pudp; -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ -+ if (p4d_none(p4d)) { -+ p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN; -+ phys_addr_t pud_phys; -+ -+ if (flags & NO_EXEC_MAPPINGS) -+ p4dval |= P4D_TABLE_PXN; -+ BUG_ON(!pgtable_alloc); -+ pud_phys = pgtable_alloc(PUD_SHIFT); -+ __iee_p4d_populate_pre_init(p4dp, pud_phys, p4dval); -+ p4d = READ_ONCE(*p4dp); -+ } -+ BUG_ON(p4d_bad(p4d)); -+ -+ pudp = pud_set_fixmap_offset_init(p4dp, addr); -+ do { -+ pud_t old_pud = READ_ONCE(*pudp); -+ -+ next = pud_addr_end(addr, end); -+ -+ /* -+ * For 4K granule only, attempt to put down a 1GB block -+ */ -+ iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot, -+ pgtable_alloc, flags); -+ -+ BUG_ON(pud_val(old_pud) != 0 && -+ pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); -+ phys += next - addr; -+ } while (pudp++, addr = next, addr != end); -+ -+ pud_clear_fixmap_init(); -+} -+ -+static __init void __iee_create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); -+ p4d_t *p4dp; -+ p4d_t p4d; -+ -+ /* -+ * If the virtual and physical address don't have the same offset -+ * within a page, we cannot map the region as the caller expects. -+ */ -+ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) -+ return; -+ -+ phys &= PAGE_MASK; -+ addr = virt & PAGE_MASK; -+ end = PAGE_ALIGN(virt + size); -+ -+ do { -+ next = pgd_addr_end(addr, end); -+ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, -+ flags); -+ p4dp = p4d_offset(pgdp, addr); -+ p4d = READ_ONCE(*p4dp); -+ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE)); -+ phys += next - addr; -+ } while (pgdp++, addr = next, addr != end); -+} -+ -+static __init void __iee_create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ mutex_lock(&fixmap_lock); -+ __iee_create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot, -+ pgtable_alloc, flags); -+ mutex_unlock(&fixmap_lock); -+} -+#endif -+ -+static phys_addr_t __pgd_pgtable_alloc(int shift) -+{ -+ #ifdef CONFIG_PTP -+ void *ptr = get_iee_pgtable_page(GFP_PGTABLE_KERNEL); -+ #else -+ void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); -+ #endif -+ BUG_ON(!ptr); -+ -+ /* Ensure the zeroed page is visible to the page table walker */ -+ dsb(ishst); -+ return __pa(ptr); -+} -+ -+static phys_addr_t pgd_pgtable_alloc(int shift) -+{ -+ phys_addr_t pa = __pgd_pgtable_alloc(shift); -+ struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); -+ -+ /* -+ * Call proper page table ctor in case later we need to -+ * call core mm functions like apply_to_page_range() on -+ * this pre-allocated page table. -+ * -+ * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is -+ * folded, and if so pagetable_pte_ctor() becomes nop. -+ */ -+ if (shift == PAGE_SHIFT) -+ BUG_ON(!pagetable_pte_ctor(ptdesc)); -+ else if (shift == PMD_SHIFT) -+ BUG_ON(!pagetable_pmd_ctor(ptdesc)); -+ -+ return pa; -+} -+ -+/* -+ * This function can only be used to modify existing table entries, -+ * without allocating new levels of table. Note that this permits the -+ * creation of new section or page entries. -+ */ -+void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, -+ phys_addr_t size, pgprot_t prot) -+{ -+ if (virt < PAGE_OFFSET) { -+ pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", -+ &phys, virt); -+ return; -+ } -+ -+ #ifdef CONFIG_PTP -+ __iee_create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL, -+ NO_CONT_MAPPINGS); -+ #else -+ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, -+ NO_CONT_MAPPINGS); -+ #endif -+} -+ -+void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, bool page_mappings_only) -+{ -+ int flags = 0; -+ -+ BUG_ON(mm == &init_mm); -+ -+ if (page_mappings_only) -+ flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; -+ -+ __create_pgd_mapping(mm->pgd, phys, virt, size, prot, -+ pgd_pgtable_alloc, flags); -+} -+ -+static void update_mapping_prot(phys_addr_t phys, unsigned long virt, -+ phys_addr_t size, pgprot_t prot) -+{ -+ if (virt < PAGE_OFFSET) { -+ pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", -+ &phys, virt); -+ return; -+ } -+ -+ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, -+ NO_CONT_MAPPINGS); -+ -+ /* flush the TLBs after updating live kernel mappings */ -+ flush_tlb_kernel_range(virt, virt + size); -+} -+ -+static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, -+ phys_addr_t end, pgprot_t prot, int flags) -+{ -+ #ifdef CONFIG_PTP -+ __iee_create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), end - start, -+ prot, early_pgtable_alloc, flags); -+ #else -+ __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, -+ prot, early_pgtable_alloc, flags); -+ #endif -+} -+ -+void __init mark_linear_text_alias_ro(void) -+{ -+ /* -+ * Remove the write permissions from the linear alias of .text/.rodata -+ */ -+ update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), -+ (unsigned long)__init_begin - (unsigned long)_stext, -+ PAGE_KERNEL_RO); -+} -+ -+#ifdef CONFIG_KFENCE - - bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL; - -@@ -655,147 +1227,808 @@ static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, - BUG_ON(!PAGE_ALIGNED(pa_start)); - BUG_ON(!PAGE_ALIGNED(size)); - -- __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, -- early_pgtable_alloc, flags); -+ #ifdef CONFIG_PTP -+ __iee_create_pgd_mapping_pre_init(pgdp, pa_start, (unsigned long)va_start, size, prot, -+ early_pgtable_alloc, flags); -+ #else -+ __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, -+ early_pgtable_alloc, flags); -+ #endif -+ -+ if (!(vm_flags & VM_NO_GUARD)) -+ size += PAGE_SIZE; -+ -+ vma->addr = va_start; -+ vma->phys_addr = pa_start; -+ vma->size = size; -+ vma->flags = VM_MAP | vm_flags; -+ vma->caller = __builtin_return_address(0); -+ -+ vm_area_add_early(vma); -+} -+ -+static pgprot_t kernel_exec_prot(void) -+{ -+ return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; -+} -+ -+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -+static int __init map_entry_trampoline(void) -+{ -+ int i; -+ -+ pgprot_t prot = kernel_exec_prot(); -+ phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); -+ -+ /* The trampoline is always mapped and can therefore be global */ -+ pgprot_val(prot) &= ~PTE_NG; -+ -+ /* Map only the text into the trampoline page table */ -+ memset(tramp_pg_dir, 0, PGD_SIZE); -+ // #ifdef CONFIG_PTP -+ // iee_set_logical_mem_ro((unsigned long)tramp_pg_dir); -+ // #endif -+ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, -+ entry_tramp_text_size(), prot, -+ __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); -+ -+ /* Map both the text and data into the kernel page table */ -+ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) -+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, -+ pa_start + i * PAGE_SIZE, prot); -+ -+ if (IS_ENABLED(CONFIG_RELOCATABLE)) -+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, -+ pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO); -+ -+ return 0; -+} -+core_initcall(map_entry_trampoline); -+#endif -+ -+/* -+ * Open coded check for BTI, only for use to determine configuration -+ * for early mappings for before the cpufeature code has run. -+ */ -+static bool arm64_early_this_cpu_has_bti(void) -+{ -+ u64 pfr1; -+ -+ if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) -+ return false; -+ -+ pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); -+ return cpuid_feature_extract_unsigned_field(pfr1, -+ ID_AA64PFR1_EL1_BT_SHIFT); -+} -+ -+#ifdef CONFIG_IEE -+/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */ -+static void __init iee_si_set_pmd_APtable(unsigned long addr, pgd_t *pgdir) -+{ -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ -+ #ifdef CONFIG_PTP -+ pud_t *pudp = pud_set_fixmap_offset_init(p4dp, addr); -+ -+ pmd_t *pmdp = pmd_set_fixmap_offset_init(pudp, addr); -+ -+ pmd_t pmd = READ_ONCE(*pmdp); -+ -+ __iee_pmd_populate_pre_init(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE); -+ -+ pud_clear_fixmap_init(); -+ pmd_clear_fixmap_init(); -+ #else -+ pud_t *pudp = pud_set_fixmap_offset(p4dp, addr); -+ -+ pmd_t *pmdp = pmd_set_fixmap_offset(pudp, addr); -+ -+ pmd_t pmd = READ_ONCE(*pmdp); -+ -+ __pmd_populate(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE); -+ -+ pud_clear_fixmap(); -+ pmd_clear_fixmap(); -+ #endif -+} -+/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */ -+static void __init mark_iee_si_pmd_APtable(pgd_t *pgdir) -+{ -+ unsigned long addr = (unsigned long)__iee_si_text_start; -+ iee_si_set_pmd_APtable(addr, pgdir); -+ // iee rwx gate exit may be mapped by another pmd. -+ iee_si_set_pmd_APtable(addr + PAGE_SIZE, pgdir); -+} -+#endif -+ -+/* -+ * Create fine-grained mappings for the kernel. -+ */ -+static void __init map_kernel(pgd_t *pgdp) -+{ -+ static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, -+ vmlinux_initdata, vmlinux_data; -+ -+ #ifdef CONFIG_IEE -+ static struct vm_struct vmlinux_iee_code, vmlinux_iee_data, vmlinux_iee_gate, vmlinux_text_end; -+ #endif -+ -+ /* -+ * External debuggers may need to write directly to the text -+ * mapping to install SW breakpoints. Allow this (only) when -+ * explicitly requested with rodata=off. -+ */ -+ pgprot_t text_prot = kernel_exec_prot(); -+ -+ /* -+ * If we have a CPU that supports BTI and a kernel built for -+ * BTI then mark the kernel executable text as guarded pages -+ * now so we don't have to rewrite the page tables later. -+ */ -+ if (arm64_early_this_cpu_has_bti()) -+ text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); -+ -+ /* -+ * Only rodata will be remapped with different permissions later on, -+ * all other segments are allowed to use contiguous mappings. -+ */ -+ #ifdef CONFIG_IEE -+ map_kernel_segment(pgdp, _stext, __iee_si_data_start, text_prot, &vmlinux_text, -+ 0, VM_NO_GUARD); -+ /* Set iee si data RW. */ -+ map_kernel_segment(pgdp, __iee_si_data_start, __iee_exec_entry_start, SET_NG(PAGE_KERNEL), -+ &vmlinux_iee_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); -+ /* Set iee entry codes NG. */ -+ map_kernel_segment(pgdp, __iee_exec_entry_start, __iee_si_text_start, SET_NG(text_prot), &vmlinux_iee_gate, -+ NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); -+ /* Map __iee_si_text_start - __iee_si_text_end as U RWX pages and set PMD APTABLE = (1,1). */ -+ map_kernel_segment(pgdp, __iee_si_text_start, __iee_si_text_end, SET_NG((PAGE_KERNEL_EXEC)), -+ &vmlinux_iee_code, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); -+ mark_iee_si_pmd_APtable(pgdp); -+ -+ map_kernel_segment(pgdp, __iee_si_text_end, _etext, text_prot, &vmlinux_text_end, 0, -+ VM_NO_GUARD); -+ -+ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, -+ &vmlinux_rodata, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); -+ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, -+ &vmlinux_inittext, 0, VM_NO_GUARD); -+ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, -+ &vmlinux_initdata, 0, VM_NO_GUARD); -+ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, 0); -+ #else -+ map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, -+ VM_NO_GUARD); -+ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, -+ &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); -+ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, -+ &vmlinux_inittext, 0, VM_NO_GUARD); -+ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, -+ &vmlinux_initdata, 0, VM_NO_GUARD); -+ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); -+ #endif -+ -+ -+ fixmap_copy(pgdp); -+ kasan_copy_shadow(pgdp); -+} -+ -+static void __init create_idmap(void) -+{ -+ u64 start = __pa_symbol(__idmap_text_start); -+ u64 size = __pa_symbol(__idmap_text_end) - start; -+ pgd_t *pgd = idmap_pg_dir; -+ u64 pgd_phys; -+ -+ /* check if we need an additional level of translation */ -+ if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) { -+ pgd_phys = early_pgtable_alloc(PAGE_SHIFT); -+ set_pgd(&idmap_pg_dir[start >> VA_BITS], -+ __pgd(pgd_phys | P4D_TYPE_TABLE)); -+ pgd = __va(pgd_phys); -+ } -+ #ifdef CONFIG_PTP -+ __iee_create_pgd_mapping_pre_init(pgd, start, start, size, PAGE_KERNEL_ROX, -+ early_pgtable_alloc, 0); -+ #else -+ __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX, -+ early_pgtable_alloc, 0); -+ #endif -+ -+ if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { -+ extern u32 __idmap_kpti_flag; -+ u64 pa = __pa_symbol(&__idmap_kpti_flag); -+ -+ /* -+ * The KPTI G-to-nG conversion code needs a read-write mapping -+ * of its synchronization flag in the ID map. -+ */ -+ #ifdef CONFIG_PTP -+ __iee_create_pgd_mapping_pre_init(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, -+ early_pgtable_alloc, 0); -+ #else -+ __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, -+ early_pgtable_alloc, 0); -+ #endif -+ } -+} -+ -+#ifdef CONFIG_IEE -+#ifndef CONFIG_PTP -+static void set_init_iee_stack_page(unsigned long addr) -+{ -+ unsigned long iee_addr = __phys_to_iee(__pa_symbol(addr)); -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ -+ pud_t *pudp = pud_offset(p4dp, addr); -+ -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ -+ int i; -+ for(i = 0; i < 4; i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ set_pte(ptep, pte); -+ ptep++; -+ } -+ -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); -+ -+ p4dp = p4d_offset(pgdp, iee_addr); -+ p4d = READ_ONCE(*p4dp); -+ -+ pudp = pud_offset(p4dp, iee_addr); -+ -+ pmdp = pmd_offset(pudp, iee_addr); -+ -+ ptep = pte_offset_kernel(pmdp, iee_addr); -+ -+ for(i = 0; i < 4; i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ set_pte(ptep, pte); -+ ptep++; -+ } -+ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE); -+ isb(); -+} -+#endif -+ -+static void __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); -+ p4d_t *p4dp; -+ p4d_t p4d; -+ -+ /* -+ * If the virtual and physical address don't have the same offset -+ * within a page, we cannot map the region as the caller expects. -+ */ -+ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) -+ return; -+ -+ phys &= PAGE_MASK; -+ addr = virt & PAGE_MASK; -+ end = PAGE_ALIGN(virt + size); -+ -+ do { -+ next = pgd_addr_end(addr, end); -+ #ifdef CONFIG_PTP -+ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, -+ flags); -+ #else -+ alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, -+ flags); -+ #endif -+ p4dp = p4d_offset(pgdp, addr); -+ p4d = READ_ONCE(*p4dp); -+ #ifdef CONFIG_PTP -+ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE)); -+ #else -+ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE)); -+ #endif -+ phys += next - addr; -+ } while (pgdp++, addr = next, addr != end); -+} -+ -+static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ mutex_lock(&fixmap_lock); -+ __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot, -+ pgtable_alloc, flags); -+ mutex_unlock(&fixmap_lock); -+} -+ -+static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start, -+ phys_addr_t end, pgprot_t prot, int flags) -+{ -+ #ifdef CONFIG_PTP -+ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, -+ prot, early_pgtable_alloc, flags); -+ #else -+ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, -+ prot, early_pgtable_alloc, flags); -+ #endif -+} -+ -+static void __init map_iee(pgd_t *pgdp) -+{ -+ static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); -+ phys_addr_t kernel_start = __pa_symbol(_stext); -+ phys_addr_t kernel_end = __pa_symbol(__init_begin); -+ phys_addr_t start, end; -+ phys_addr_t early_kfence_pool; -+ int flags = NO_EXEC_MAPPINGS; -+ u64 i; -+ -+ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; -+ -+ /* -+ * Setting hierarchical PXNTable attributes on table entries covering -+ * the linear region is only possible if it is guaranteed that no table -+ * entries at any level are being shared between the linear region and -+ * the vmalloc region. Check whether this is true for the PGD level, in -+ * which case it is guaranteed to be true for all other levels as well. -+ */ -+ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); -+ -+ early_kfence_pool = arm64_kfence_alloc_pool(); -+ -+ /* -+ * Take care not to create a writable alias for the -+ * read-only text and rodata sections of the kernel image. -+ * So temporarily mark them as NOMAP to skip mappings in -+ * the following for-loop -+ */ -+ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); -+ -+ /* map all the memory banks */ -+ for_each_mem_range(i, &start, &end) { -+ if (start >= end) -+ break; -+ /* -+ * The linear map must allow allocation tags reading/writing -+ * if MTE is present. Otherwise, it has the same attributes as -+ * PAGE_KERNEL. -+ */ -+ __map_memblock_for_iee(pgdp, start, end, SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags); -+ } -+ -+ /* -+ * Map the linear alias of the [_text, __init_begin) interval -+ * as non-executable now, and remove the write permission in -+ * mark_linear_text_alias_ro() below (which will be called after -+ * alternative patching has completed). This makes the contents -+ * of the region accessible to subsystems such as hibernate, -+ * but protects it from inadvertent modification or execution. -+ * Note that contiguous mappings cannot be remapped in this way, -+ * so we should avoid them here. -+ */ -+ __map_memblock_for_iee(pgdp, kernel_start, kernel_end, -+ SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags); -+ memblock_clear_nomap(kernel_start, kernel_end - kernel_start); -+ arm64_kfence_map_pool(early_kfence_pool, pgdp); -+} -+ -+/* -+ * Change page access permission, whereas not handling huge pages. -+ * Only used on IEE init functions. -+ */ -+static void __init iee_si_set_page_attr(unsigned long addr, pteval_t attr) -+{ -+ unsigned long flag; -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ -+ pud_t *pudp = pud_offset(p4dp, addr); -+ -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ pte_t pte = READ_ONCE(*ptep); -+ -+ if(attr & PTE_RDONLY) -+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); -+ pte = __pte(pte_val(pte) | attr); -+ #ifdef CONFIG_PTP -+ // Write pgtable in IEE directly. -+ flag = local_daif_save(); -+ asm volatile ("msr pan, #0"); -+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); -+ asm volatile ("msr pan, #1"); -+ local_daif_restore(flag); -+ #else -+ WRITE_ONCE(*ptep, pte); -+ #endif -+} -+ -+/* Prepare data used for iee rwx gates. These data are setted only once. */ -+void __init iee_si_prepare_data(void) -+{ -+ unsigned long va; -+ // Record current TCR val after system init. -+ // iee_si_tcr = read_sysreg(tcr_el1) & ~(SYS_TCR_IEE_SI); -+ // CNP maybe enable. -+ if (system_supports_cnp()) { -+ iee_base_swapper_pg_dir |= TTBR_CNP_BIT; -+ } -+ // Mark iee data as RO and move it to iee after setting up. -+ va = (unsigned long)__iee_si_data_start; -+ iee_si_set_page_attr(va, PTE_RDONLY); -+ // iee_si_set_page_attr(lm_alias(va)+iee_offset, 0x1 | PTE_RDONLY); -+ // Set iee sensitive inst code page U RWX here to hide it from kernel. -+ va = (unsigned long)__iee_si_text_start; -+ iee_si_set_page_attr(va, PTE_USER); -+ va = (unsigned long)__iee_si_text_start + PAGE_SIZE; -+ iee_si_set_page_attr(va, PTE_USER); -+ flush_tlb_all(); -+} -+ -+#endif -+ -+#ifdef CONFIG_PTP -+// Attention : Using set_xxx without adding offset. -+static void __init set_iee_valid_pre_init(unsigned long addr) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ -+ pud_t *pudp = pud_offset(p4dp, addr); -+ -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ pte_t pte = READ_ONCE(*ptep); -+ -+ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) -+ return; -+ -+ pte = __pte(pte_val(pte) | 0x1); -+ iee_set_pte_pre_init(ptep, pte); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); -+ isb(); -+} -+ -+static void __init move_pte_table_into_iee(pmd_t *pmdp, unsigned long addr, unsigned long end) -+{ -+ pmd_t pmd = READ_ONCE(*pmdp); -+ unsigned long iee_addr = __phys_to_iee(__pmd_to_phys(pmd)); -+ set_iee_valid_pre_init(iee_addr); -+} -+ -+static void __init move_pmd_table_into_iee(pud_t *pudp, unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ pud_t pud = READ_ONCE(*pudp); -+ pmd_t *pmdp; -+ pmd_t pmd; -+ unsigned long iee_addr = __phys_to_iee(__pud_to_phys(pud)); -+ set_iee_valid_pre_init(iee_addr); -+ -+ pmdp = pmd_offset(pudp, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ pmd = READ_ONCE(*pmdp); -+ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0) -+ { -+ continue; -+ } -+ else -+ { -+ move_pte_table_into_iee(pmdp, addr, next); -+ } -+ } while (pmdp++, addr = next, addr != end); -+} -+ -+static void __init move_pud_table_into_iee(pgd_t *pgdp, unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ pud_t *pudp; -+ pud_t pud; -+ unsigned long iee_addr = __phys_to_iee(__p4d_to_phys(p4d)); -+ set_iee_valid_pre_init(iee_addr); -+ -+ pudp = pud_offset(p4dp, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ pud = READ_ONCE(*pudp); -+ if ((pud_val(pud) & PUD_TABLE_BIT) == 0) -+ { -+ continue; -+ } -+ else -+ { -+ move_pmd_table_into_iee(pudp, addr, next); -+ } -+ } while (pudp++, addr = next, addr != end); -+} -+ -+static void __init init_iee_for_one_region(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) -+{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); -+ -+ addr = va_start & PAGE_MASK; -+ end = PAGE_ALIGN(va_end); -+ -+ do { -+ next = pgd_addr_end(addr, end); -+ move_pud_table_into_iee(pgdp, addr, next); -+ } while (pgdp++, addr = next, addr != end); -+} -+ -+static void __init init_iee(void) -+{ -+ unsigned long iee_addr; -+ phys_addr_t start, end; -+ u64 i; -+ pgd_t *pgdp; -+ -+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -+ // handling 1-level tramp page table tramp_pg_dir -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(tramp_pg_dir)); -+ set_iee_valid_pre_init(iee_addr); -+ #endif -+ // handling 1-level page table swapper_pg_dir -+ pgdp = swapper_pg_dir; -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(swapper_pg_dir)); -+ set_iee_valid_pre_init(iee_addr); -+ // handling 2/3/4-level page table for kernel -+ init_iee_for_one_region(pgdp, (unsigned long)_text, (unsigned long)_etext); -+ init_iee_for_one_region(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin); -+ init_iee_for_one_region(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end); -+ init_iee_for_one_region(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end); -+ init_iee_for_one_region(pgdp, (unsigned long)_data, (unsigned long)_end); -+ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pte_addr)); -+ set_iee_valid_pre_init(iee_addr); -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pmd_addr)); -+ set_iee_valid_pre_init(iee_addr); -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pud_addr)); -+ set_iee_valid_pre_init(iee_addr); -+ // handling 2/3/4-level page table for logical mem and iee -+ for_each_mem_range(i, &start, &end) { -+ if (start >= end) -+ break; -+ /* -+ * The linear map must allow allocation tags reading/writing -+ * if MTE is present. Otherwise, it has the same attributes as -+ * PAGE_KERNEL. -+ */ -+ init_iee_for_one_region(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); -+ init_iee_for_one_region(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); -+ } -+} -+ -+static void set_init_iee_stack_page_pre_init(unsigned long addr) -+{ -+ unsigned long iee_addr = __phys_to_iee(__pa_symbol(addr)); -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); - -- if (!(vm_flags & VM_NO_GUARD)) -- size += PAGE_SIZE; -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); - -- vma->addr = va_start; -- vma->phys_addr = pa_start; -- vma->size = size; -- vma->flags = VM_MAP | vm_flags; -- vma->caller = __builtin_return_address(0); -+ pud_t *pudp = pud_offset(p4dp, addr); - -- vm_area_add_early(vma); --} -+ pmd_t *pmdp = pmd_offset(pudp, addr); - --static pgprot_t kernel_exec_prot(void) --{ -- return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; --} -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); - --#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 --static int __init map_entry_trampoline(void) --{ - int i; -+ for(i = 0; i < 4; i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) & ~PTE_VALID); -+ iee_set_pte_pre_init(ptep, pte); -+ ptep++; -+ } - -- pgprot_t prot = kernel_exec_prot(); -- phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); -+ pgdp = pgd_offset_pgd(pgdir, iee_addr); - -- /* The trampoline is always mapped and can therefore be global */ -- pgprot_val(prot) &= ~PTE_NG; -+ p4dp = p4d_offset(pgdp, iee_addr); -+ p4d = READ_ONCE(*p4dp); - -- /* Map only the text into the trampoline page table */ -- memset(tramp_pg_dir, 0, PGD_SIZE); -- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, -- entry_tramp_text_size(), prot, -- __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); -+ pudp = pud_offset(p4dp, iee_addr); - -- /* Map both the text and data into the kernel page table */ -- for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) -- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, -- pa_start + i * PAGE_SIZE, prot); -+ pmdp = pmd_offset(pudp, iee_addr); - -- if (IS_ENABLED(CONFIG_RELOCATABLE)) -- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, -- pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO); -+ ptep = pte_offset_kernel(pmdp, iee_addr); - -- return 0; -+ for(i = 0; i < 4; i++) -+ { -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(pte_val(pte) | PTE_VALID); -+ iee_set_pte_pre_init(ptep, pte); -+ ptep++; -+ } -+ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE); -+ isb(); - } --core_initcall(map_entry_trampoline); --#endif - --/* -- * Open coded check for BTI, only for use to determine configuration -- * for early mappings for before the cpufeature code has run. -- */ --static bool arm64_early_this_cpu_has_bti(void) -+static void __init iee_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) - { -- u64 pfr1; -- -- if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) -- return false; -- -- pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); -- return cpuid_feature_extract_unsigned_field(pfr1, -- ID_AA64PFR1_EL1_BT_SHIFT); -+ pmd_t pmd = READ_ONCE(*pmdp); -+ unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); -+ iee_set_logical_mem_ro(logical_addr); - } - --/* -- * Create fine-grained mappings for the kernel. -- */ --static void __init map_kernel(pgd_t *pgdp) -+static void __init iee_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) - { -- static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, -- vmlinux_initdata, vmlinux_data; -- -- /* -- * External debuggers may need to write directly to the text -- * mapping to install SW breakpoints. Allow this (only) when -- * explicitly requested with rodata=off. -- */ -- pgprot_t text_prot = kernel_exec_prot(); -+ unsigned long next; -+ pud_t pud = READ_ONCE(*pudp); -+ pmd_t *pmdp; -+ pmd_t pmd; -+ unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); -+ iee_set_logical_mem_ro(logical_addr); - -- /* -- * If we have a CPU that supports BTI and a kernel built for -- * BTI then mark the kernel executable text as guarded pages -- * now so we don't have to rewrite the page tables later. -- */ -- if (arm64_early_this_cpu_has_bti()) -- text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); -+ pmdp = pmd_offset(pudp, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ pmd = READ_ONCE(*pmdp); -+ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0) -+ { -+ continue; -+ } -+ else -+ { -+ iee_set_pte_table_ro(pmdp, addr, next); -+ } -+ } while (pmdp++, addr = next, addr != end); -+} - -- /* -- * Only rodata will be remapped with different permissions later on, -- * all other segments are allowed to use contiguous mappings. -- */ -- map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, -- VM_NO_GUARD); -- map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, -- &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); -- map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, -- &vmlinux_inittext, 0, VM_NO_GUARD); -- map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, -- &vmlinux_initdata, 0, VM_NO_GUARD); -- map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); -+static void __init iee_set_pud_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ pud_t *pudp; -+ pud_t pud; -+ unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); -+ iee_set_logical_mem_ro(logical_addr); - -- fixmap_copy(pgdp); -- kasan_copy_shadow(pgdp); -+ pudp = pud_offset(p4dp, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ pud = READ_ONCE(*pudp); -+ if ((pud_val(pud) & PUD_TABLE_BIT) == 0) -+ { -+ continue; -+ } -+ else -+ { -+ iee_set_pmd_table_ro(pudp, addr, next); -+ } -+ } while (pudp++, addr = next, addr != end); - } - --static void __init create_idmap(void) -+static void __init iee_mark_pgtable_for_one_region_ro(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) - { -- u64 start = __pa_symbol(__idmap_text_start); -- u64 size = __pa_symbol(__idmap_text_end) - start; -- pgd_t *pgd = idmap_pg_dir; -- u64 pgd_phys; -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); - -- /* check if we need an additional level of translation */ -- if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) { -- pgd_phys = early_pgtable_alloc(PAGE_SHIFT); -- set_pgd(&idmap_pg_dir[start >> VA_BITS], -- __pgd(pgd_phys | P4D_TYPE_TABLE)); -- pgd = __va(pgd_phys); -- } -- __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX, -- early_pgtable_alloc, 0); -+ addr = va_start & PAGE_MASK; -+ end = PAGE_ALIGN(va_end); - -- if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { -- extern u32 __idmap_kpti_flag; -- u64 pa = __pa_symbol(&__idmap_kpti_flag); -+ do { -+ next = pgd_addr_end(addr, end); -+ iee_set_pud_table_ro(pgdp, addr, next); -+ } while (pgdp++, addr = next, addr != end); -+} - -+// Mark pgtable outside as RO. -+void __init iee_mark_all_lm_pgtable_ro(void) -+{ -+ extern void *bm_pte_addr; -+ iee_set_logical_mem_ro((unsigned long)bm_pte_addr); -+ unsigned long logical_addr; -+ phys_addr_t start, end; -+ u64 i; -+ pgd_t *pgdp; -+ -+ // handling static allocated page table -+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -+ // handling 1-level tramp page table tramp_pg_dir -+ // logical_addr = (unsigned long)__va(__pa_symbol(tramp_pg_dir)); -+ // iee_set_logical_mem_ro(logical_addr); -+ #endif -+ // handling 1-level page table swapper_pg_dir -+ pgdp = swapper_pg_dir; -+ // iee_set_logical_mem_ro((unsigned long)swapper_pg_dir); -+ // logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); -+ // iee_set_logical_mem_ro(logical_addr); -+ -+ // handling 2/3/4-level page table for kernel -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_text, (unsigned long)_etext); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_data, (unsigned long)_end); -+ -+ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx -+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pte_addr)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ iee_set_logical_mem_ro((unsigned long)bm_pmd_addr); -+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pmd_addr)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ iee_set_logical_mem_ro((unsigned long)bm_pud_addr); -+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pud_addr)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ // handling 2/3/4-level page table for logical mem and iee -+ for_each_mem_range(i, &start, &end) { -+ if (start >= end) -+ break; - /* -- * The KPTI G-to-nG conversion code needs a read-write mapping -- * of its synchronization flag in the ID map. -+ * The linear map must allow allocation tags reading/writing -+ * if MTE is present. Otherwise, it has the same attributes as -+ * PAGE_KERNEL. - */ -- __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, -- early_pgtable_alloc, 0); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); - } - } -+#endif -+ -+#ifdef CONFIG_KOI -+extern s64 koi_offset; -+#endif - - void __init paging_init(void) - { -+ #ifdef CONFIG_IEE -+ unsigned long SP_EL0; -+ void *new; -+ void *init_token; -+ struct task_token *token; -+ unsigned long tcr; -+ -+ // Check if cpu has PAN and HPDS. -+ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), -+ ID_AA64MMFR1_EL1_PAN_SHIFT)) -+ panic("Architecture doesn't support PAN, please disable CONFIG_IEE.\n"); -+ -+ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), -+ ID_AA64MMFR1_EL1_HPDS_SHIFT)) -+ panic("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n"); -+ #endif -+ -+ // Avoid using iee code to modify pgtable before iee initialized. -+ #ifdef CONFIG_PTP -+ pgd_t *pgdp = pgd_set_fixmap_init(__pa_symbol(swapper_pg_dir)); -+ #else - pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); -+ #endif -+ -+ - extern pgd_t init_idmap_pg_dir[]; - - idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0)); -@@ -803,7 +2036,21 @@ void __init paging_init(void) - map_kernel(pgdp); - map_mem(pgdp); - -+ // Map the whole physical mem into IEE, but set invalid. -+#ifdef CONFIG_IEE -+ map_iee(pgdp); -+#else -+#ifdef CONFIG_KOI -+ map_koi(pgdp); -+#endif -+#endif -+ -+ // Avoid using iee code to modify pgtable before iee initialized. -+ #ifdef CONFIG_PTP -+ pgd_clear_fixmap_init(); -+ #else - pgd_clear_fixmap(); -+ #endif - - cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir); - init_mm.pgd = swapper_pg_dir; -@@ -814,6 +2061,83 @@ void __init paging_init(void) - memblock_allow_resize(); - - create_idmap(); -+ -+ #ifdef CONFIG_IEE -+ // test iee_exec_entry -+ iee_rwx_gate_entry(IEE_SI_TEST); -+ // Initialize init iee stack. -+ #ifdef CONFIG_PTP -+ set_init_iee_stack_page_pre_init((unsigned long)init_iee_stack_begin); -+ #else -+ set_init_iee_stack_page((unsigned long)init_iee_stack_begin); -+ #endif -+ #endif -+ -+ // Init token for init_task. -+ #ifdef CONFIG_IEE -+ // Change SP_EL0 from Image VA to Logical VA. -+ SP_EL0 = (unsigned long)__va(__pa_symbol(&init_task)); -+ write_sysreg(SP_EL0, sp_el0); -+ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask); -+ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev))); -+ init_task.children.next = (__va(__pa_symbol(init_task.children.next))); -+ // Alloc a page for init_token. -+ new = __va(early_pgtable_alloc(0)); -+ init_token_page_vaddr = new; -+ init_token = (void *)__phys_to_iee(__pa_symbol(&init_task)); -+ // Use lm to write token before IEE initialized. -+ token = (struct task_token *)((unsigned long)new + (((unsigned long)&init_task) & ~PAGE_MASK)); -+ token->pgd = NULL; -+ token->iee_stack = (void *)__phys_to_iee(__pa_symbol((void *)init_iee_stack_end)); -+ token->valid = true; -+ iee_set_token_page_valid_pre_init(init_token, new); -+ #endif -+ -+ #ifdef CONFIG_PTP -+ // Map the existing pgtable into IEE, set valid. -+ init_iee(); -+ #endif -+ -+ #ifdef CONFIG_IEE -+ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SPAN); -+ #endif -+ -+ // IEE ready. -+ // Pgtable writing before uses logical memory and after uses IEE memory. -+ -+ #ifdef CONFIG_IEE -+ // Set HPD1 as 1. -+ tcr = read_sysreg(tcr_el1); -+ tcr |= ((unsigned long)0x1 << 42); -+ write_sysreg(tcr, tcr_el1); -+ isb(); -+ -+ // Flush tlb to enable IEE. -+ flush_tlb_all(); -+ -+ // mark that iee is prepared. -+ iee_init_done = true; -+#else -+#ifdef CONFIG_KOI -+ unsigned long SP_EL0 = __va(__pa_symbol(&init_task)); -+ write_sysreg(SP_EL0, sp_el0); -+ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask); -+ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev))); -+ init_task.children.next = (__va(__pa_symbol(init_task.children.next))); -+ // create a new page for token -+ void *alloc_token = __va(early_pgtable_alloc(0)); -+ // get the address of token -+ void *token_addr = __phys_to_virt(__pa_symbol(&init_task)) + KOI_OFFSET; -+ // add memory mapping for token -+ koi_add_page_mapping(token_addr, (void *)alloc_token); -+ // printk(KERN_ERR "token_addr=0x%16llx, alloc_token=0x%16llx, init_task=0x%16llx, virt=0x%16llx\n", token_addr, alloc_token, &init_task, __phys_to_virt(__pa_symbol(&init_task))); -+ struct task_token *token = (struct task_token *)((unsigned long)alloc_token + (((unsigned long)&init_task) & ~PAGE_MASK)); -+ token->koi_kernel_stack = NULL; -+ token->koi_stack = NULL; -+ token->koi_stack_base = NULL; -+ token->current_ttbr1 = 0; -+#endif -+#endif - } - - #ifdef CONFIG_MEMORY_HOTPLUG -@@ -1202,6 +2526,139 @@ void vmemmap_free(unsigned long start, unsigned long end, - } - #endif /* CONFIG_MEMORY_HOTPLUG */ - -+#ifdef CONFIG_PTP -+ -+static void * __init iee_ptdesc_alloc_block_zero(unsigned long size, int node, bool pt) -+{ -+ void *p = memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS), -+ MEMBLOCK_ALLOC_ACCESSIBLE, node); -+ -+ if (!p) -+ return NULL; -+ memset(p, 0, size); -+ -+ /* Vaild iee address to enable page table operations. */ -+ if (pt) -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(p))); -+ -+ return p; -+} -+ -+static pte_t * __init iee_ptdesc_pte_populate(pmd_t *pmd, unsigned long addr, int node) -+{ -+ pte_t *pte = pte_offset_kernel(pmd, addr); -+ if (pte_none(ptep_get(pte))) { -+ pte_t entry; -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, false); -+ if (!p) -+ return NULL; -+ -+ entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); -+ set_pte_at(&init_mm, addr, pte, entry); -+ } -+ return pte; -+} -+ -+static pmd_t * __init iee_ptdesc_pmd_populate(pud_t *pud, unsigned long addr, int node) -+{ -+ pmd_t *pmd = pmd_offset(pud, addr); -+ if (pmd_none(*pmd)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); -+ if (!p) -+ return NULL; -+ pmd_populate_kernel(&init_mm, pmd, p); -+ } -+ return pmd; -+} -+ -+static pud_t * __init iee_ptdesc_pud_populate(p4d_t *p4d, unsigned long addr, int node) -+{ -+ pud_t *pud = pud_offset(p4d, addr); -+ if (pud_none(*pud)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); -+ if (!p) -+ return NULL; -+ pmd_init(p); -+ pud_populate(&init_mm, pud, p); -+ } -+ return pud; -+} -+ -+static p4d_t * __init iee_ptdesc_p4d_populate(pgd_t *pgd, unsigned long addr, int node) -+{ -+ p4d_t *p4d = p4d_offset(pgd, addr); -+ if (p4d_none(*p4d)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); -+ if (!p) -+ return NULL; -+ pud_init(p); -+ p4d_populate(&init_mm, p4d, p); -+ } -+ return p4d; -+} -+ -+static pgd_t * __init iee_ptdesc_pgd_populate(unsigned long addr, int node) -+{ -+ pgd_t *pgd = pgd_offset_k(addr); -+ if (pgd_none(*pgd)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); -+ if (!p) -+ return NULL; -+ pgd_populate(&init_mm, pgd, p); -+ } -+ return pgd; -+} -+ -+/* Create mappings if that address is not mapped. */ -+static pte_t * __init iee_ptdesc_populate_address(unsigned long addr) -+{ -+ pgd_t *pgd; -+ p4d_t *p4d; -+ pud_t *pud; -+ pmd_t *pmd; -+ pte_t *pte; -+ int node = NUMA_NO_NODE; -+ -+ pgd = iee_ptdesc_pgd_populate(addr, node); -+ if (!pgd) -+ return NULL; -+ p4d = iee_ptdesc_p4d_populate(pgd, addr, node); -+ if (!p4d) -+ return NULL; -+ pud = iee_ptdesc_pud_populate(p4d, addr, node); -+ if (!pud) -+ return NULL; -+ pmd = iee_ptdesc_pmd_populate(pud, addr, node); -+ if (!pmd) -+ return NULL; -+ pte = iee_ptdesc_pte_populate(pmd, addr, node); -+ if (!pte) -+ return NULL; -+ -+ return pte; -+} -+ -+/* Init ptdesc array used by iee. */ -+int __init iee_ptdesc_sparse_init(void) -+{ -+ unsigned long start_pfn, end_pfn; -+ int i, nid; -+ /* */ -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { -+ struct ptdesc** ptdesc_start = ALIGN_DOWN((u64)(__pfn_to_ptdesc(start_pfn)), PAGE_SIZE); -+ struct ptdesc** end = ALIGN((u64)(__pfn_to_ptdesc(end_pfn)), PAGE_SIZE); -+ unsigned long addr = ptdesc_start; -+ pte_t *pte; -+ for (; addr < end; addr += PAGE_SIZE) { -+ pte = iee_ptdesc_populate_address(addr); -+ if (!pte) -+ return -ENOMEM; -+ } -+ } -+ return 0; -+} -+#endif /* CONFIG_PTP */ -+ - int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot) - { - pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); -@@ -1261,6 +2718,9 @@ int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) - table = pte_offset_kernel(pmdp, addr); - pmd_clear(pmdp); - __flush_tlb_kernel_pgtable(addr); -+ #ifdef CONFIG_PTP -+ iee_memset(table, 0, PAGE_SIZE); -+ #endif - pte_free_kernel(NULL, table); - return 1; - } -@@ -1521,3 +2981,93 @@ void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte - { - set_pte_at(vma->vm_mm, addr, ptep, pte); - } -+ -+#if !defined(CONFIG_IEE) && defined (CONFIG_KOI) -+static void __create_pgd_mapping_for_koi(pgd_t *pgdir, phys_addr_t phys, -+ unsigned long virt, phys_addr_t size, -+ pgprot_t prot, -+ phys_addr_t (*pgtable_alloc)(int), -+ int flags) -+{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); -+ -+ /* -+ * If the virtual and physical address don't have the same offset -+ * within a page, we cannot map the region as the caller expects. -+ */ -+ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) -+ return; -+ -+ phys &= PAGE_MASK; -+ addr = virt & PAGE_MASK; -+ end = PAGE_ALIGN(virt + size); -+ -+ do { -+ next = pgd_addr_end(addr, end); -+ alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, -+ flags); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ __p4d_populate(p4dp, __p4d_to_phys(p4d), PUD_TYPE_TABLE); -+ phys += next - addr; -+ } while (pgdp++, addr = next, addr != end); -+} -+ -+static void __init __map_memblock_for_koi(pgd_t *pgdp, phys_addr_t start, -+ phys_addr_t end, pgprot_t prot, int flags) -+{ -+ __create_pgd_mapping_for_koi(pgdp, start, __phys_to_virt(start) + KOI_OFFSET, end - start, -+ prot, early_pgtable_alloc, flags); -+} -+ -+static void __init map_koi(pgd_t *pgdp) -+{ -+ phys_addr_t kernel_start = __pa_symbol(_text); -+ phys_addr_t kernel_end = __pa_symbol(__init_begin); -+ phys_addr_t start, end; -+ int flags = 0; -+ u64 i; -+ -+ flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; -+ -+ /* -+ * Take care not to create a writable alias for the -+ * read-only text and rodata sections of the kernel image. -+ * So temporarily mark them as NOMAP to skip mappings in -+ * the following for-loop -+ */ -+ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); -+#ifdef CONFIG_KEXEC_CORE -+ if (crashk_res.end) -+ memblock_mark_nomap(crashk_res.start, -+ resource_size(&crashk_res)); -+#endif -+ -+ /* map all the memory banks */ -+ for_each_mem_range(i, &start, &end) { -+ if (start >= end) -+ break; -+ /* -+ * The linear map must allow allocation tags reading/writing -+ * if MTE is present. Otherwise, it has the same attributes as -+ * PAGE_KERNEL. -+ */ -+ __map_memblock_for_koi(pgdp, start, end, SET_NG(SET_INVALID(PAGE_KERNEL)), flags); -+ } -+#ifdef CONFIG_KEXEC_CORE -+ /* -+ * Use page-level mappings here so that we can shrink the region -+ * in page granularity and put back unused memory to buddy system -+ * through /sys/kernel/kexec_crash_size interface. -+ */ -+ if (crashk_res.end) { -+ __map_memblock_for_koi(pgdp, crashk_res.start, crashk_res.end + 1, -+ SET_NG(SET_INVALID(PAGE_KERNEL)), -+ NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); -+ memblock_clear_nomap(crashk_res.start, -+ resource_size(&crashk_res)); -+ } -+#endif -+} -+#endif -\ No newline at end of file -diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c -index 4a64089e5771..20008c35d150 100644 ---- a/arch/arm64/mm/pgd.c -+++ b/arch/arm64/mm/pgd.c -@@ -15,14 +15,38 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif -+ - static struct kmem_cache *pgd_cache __ro_after_init; - -+#ifdef CONFIG_KOI -+pgd_t *koi_pgd_alloc(void) -+{ -+ pgd_t *pgd; -+#ifdef CONFIG_PTP -+ pgd = (pgd_t *)get_iee_pgtable_page(GFP_PGTABLE_KERNEL); -+#else -+ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL); -+#endif -+ return pgd; -+} -+#endif -+ - pgd_t *pgd_alloc(struct mm_struct *mm) - { - gfp_t gfp = GFP_PGTABLE_USER; - - if (PGD_SIZE == PAGE_SIZE) -+#ifdef CONFIG_PTP -+ { -+ pgd_t* new = (pgd_t *)get_iee_pgtable_page(gfp); -+ return new; -+ } -+#else - return (pgd_t *)__get_free_page(gfp); -+#endif - else - return kmem_cache_alloc(pgd_cache, gfp); - } -@@ -30,7 +54,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm) - void pgd_free(struct mm_struct *mm, pgd_t *pgd) - { - if (PGD_SIZE == PAGE_SIZE) -+#ifdef CONFIG_PTP -+ { -+ struct page *page = virt_to_page((void *)pgd); -+ set_page_count(page, 1); -+ free_iee_pgtable_page((void *)pgd); -+ } -+#else - free_page((unsigned long)pgd); -+#endif - else - kmem_cache_free(pgd_cache, pgd); - } -diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S -index 9ef7b07349d7..b8561144a45a 100644 ---- a/arch/arm64/mm/proc.S -+++ b/arch/arm64/mm/proc.S -@@ -22,6 +22,8 @@ - #include - #include - -+ -+ - #ifdef CONFIG_ARM64_64K_PAGES - #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K - #elif defined(CONFIG_ARM64_16K_PAGES) -@@ -190,6 +192,7 @@ SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) - __idmap_cpu_set_reserved_ttbr1 x1, x3 - - offset_ttbr1 x0, x3 -+ - msr ttbr1_el1, x0 - isb - -@@ -459,6 +462,11 @@ SYM_FUNC_START(__cpu_setup) - #endif /* CONFIG_ARM64_HAFT */ - 1: - #endif /* CONFIG_ARM64_HW_AFDBM */ -+ -+#ifdef CONFIG_IEE -+ orr tcr, tcr, #TCR_HPD1 // Hierarchical permission disables -+#endif -+ - msr mair_el1, mair - msr tcr_el1, tcr - -diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c -index 5139a28130c0..23d8a4b91574 100644 ---- a/arch/arm64/mm/trans_pgd.c -+++ b/arch/arm64/mm/trans_pgd.c -@@ -25,6 +25,9 @@ - #include - #include - #include -+#ifdef CONFIG_PTP -+#include -+#endif - - static void *trans_alloc(struct trans_pgd_info *info) - { -@@ -65,10 +68,12 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, - pte_t *src_ptep; - pte_t *dst_ptep; - unsigned long addr = start; -- - dst_ptep = trans_alloc(info); - if (!dst_ptep) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)dst_ptep,0); -+ #endif - pmd_populate_kernel(NULL, dst_pmdp, dst_ptep); - dst_ptep = pte_offset_kernel(dst_pmdp, start); - -@@ -92,6 +97,9 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, - dst_pmdp = trans_alloc(info); - if (!dst_pmdp) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)dst_pmdp,0); -+ #endif - pud_populate(NULL, dst_pudp, dst_pmdp); - } - dst_pmdp = pmd_offset(dst_pudp, start); -@@ -128,6 +136,9 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, - dst_pudp = trans_alloc(info); - if (!dst_pudp) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)dst_pudp,0); -+ #endif - p4d_populate(NULL, dst_p4dp, dst_pudp); - } - dst_pudp = pud_offset(dst_p4dp, start); -@@ -212,6 +223,10 @@ int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp, - return -ENOMEM; - } - -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)trans_pgd,0); -+ #endif -+ - rc = copy_page_tables(info, trans_pgd, start, end); - if (!rc) - *dst_pgdp = trans_pgd; -@@ -247,12 +262,20 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, - if (!levels[this_level]) - return -ENOMEM; - -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)levels[this_level],0); -+ #endif -+ - level_lsb = ARM64_HW_PGTABLE_LEVEL_SHIFT(this_level); - level_msb = min(level_lsb + bits_mapped, max_msb); - level_mask = GENMASK_ULL(level_msb, level_lsb); - - index = (dst_addr & level_mask) >> level_lsb; -+ #ifdef CONFIG_PTP -+ set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry)); -+ #else - *(levels[this_level] + index) = prev_level_entry; -+ #endif - - pfn = virt_to_pfn(levels[this_level]); - prev_level_entry = pte_val(pfn_pte(pfn, -diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c -index 76ae4a3131ba..b3b962fcc3f8 100644 ---- a/arch/arm64/net/bpf_jit_comp.c -+++ b/arch/arm64/net/bpf_jit_comp.c -@@ -13,6 +13,9 @@ - #include - #include - #include -+#ifdef CONFIG_HIVE -+#include -+#endif - - #include - #include -@@ -67,6 +70,10 @@ static const int bpf2a64[] = { - /* temporary register for blinding constants */ - [BPF_REG_AX] = A64_R(9), - [FP_BOTTOM] = A64_R(27), -+#ifdef CONFIG_HIVE -+ /* fbpf base addr */ -+ [BPF_REG_BASE] = A64_R(28), -+#endif - }; - - struct jit_ctx { -@@ -283,7 +290,11 @@ static bool is_lsi_offset(int offset, int scale) - #define POKE_OFFSET (BTI_INSNS + 1) - - /* Tail call offset to jump into */ -+#ifdef CONFIG_HIVE -+#define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8 + 6 - 1) -+#else - #define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8) -+#endif - - static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) - { -@@ -296,6 +307,9 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) - const u8 fp = bpf2a64[BPF_REG_FP]; - const u8 tcc = bpf2a64[TCALL_CNT]; - const u8 fpb = bpf2a64[FP_BOTTOM]; -+#ifdef CONFIG_HIVE -+ const u8 base = bpf2a64[BPF_REG_BASE]; -+#endif - const int idx0 = ctx->idx; - int cur_offset; - -@@ -345,10 +359,35 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) - emit(A64_PUSH(r6, r7, A64_SP), ctx); - emit(A64_PUSH(r8, r9, A64_SP), ctx); - emit(A64_PUSH(fp, tcc, A64_SP), ctx); -+#ifdef CONFIG_HIVE -+ emit(A64_PUSH(fpb, base, A64_SP), ctx); -+#else - emit(A64_PUSH(fpb, A64_R(28), A64_SP), ctx); -+#endif - -+#ifdef CONFIG_HIVE -+ /* Set up BPF prog stack base and sfi base register */ -+ u64 sfi_base = (u64)prog->shadow_region_addr; -+ u64 sfi_stack = (u64)(prog->shadow_stack_addr); -+ int idx_before; -+ -+ pr_info("fp=%016llx, base=%016llx\n", sfi_stack, sfi_base); -+ -+ idx_before = ctx->idx; -+ emit_a64_mov_i64(fp, sfi_stack, ctx); // 3 insns -+ while (ctx->idx - idx_before < 3) { -+ emit(A64_NOP, ctx); -+ } -+ idx_before = ctx->idx; -+ emit_a64_mov_i64(base, sfi_base, ctx); // 3 insns -+ while (ctx->idx - idx_before < 3) { -+ emit(A64_NOP, ctx); -+ } -+#else - /* Set up BPF prog stack base register */ - emit(A64_MOV(1, fp, A64_SP), ctx); -+#endif -+ - - if (!ebpf_from_cbpf && is_main_prog) { - /* Initialize tail_call_cnt */ -@@ -356,8 +395,13 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) - - cur_offset = ctx->idx - idx0; - if (cur_offset != PROLOGUE_OFFSET) { -+ #ifdef CONFIG_HIVE -+ pr_err("PROLOGUE_OFFSET = %d, expected %d!\n", -+ cur_offset, PROLOGUE_OFFSET); -+ #else - pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", - cur_offset, PROLOGUE_OFFSET); -+ #endif - return -1; - } - -@@ -662,12 +706,20 @@ static void build_epilogue(struct jit_ctx *ctx) - const u8 r9 = bpf2a64[BPF_REG_9]; - const u8 fp = bpf2a64[BPF_REG_FP]; - const u8 fpb = bpf2a64[FP_BOTTOM]; -+#ifdef CONFIG_HIVE -+ const u8 base = bpf2a64[BPF_REG_BASE]; -+#endif - - /* We're done with BPF stack */ - emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); - -+#ifdef CONFIG_HIVE -+ /* Restore x27 and base (x28) */ -+ emit(A64_POP(fpb, base, A64_SP), ctx); -+#else - /* Restore x27 and x28 */ - emit(A64_POP(fpb, A64_R(28), A64_SP), ctx); -+#endif - /* Restore fs (x25) and x26 */ - emit(A64_POP(fp, A64_R(26), A64_SP), ctx); - -@@ -769,6 +821,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, - const u8 fp = bpf2a64[BPF_REG_FP]; - const u8 fpb = bpf2a64[FP_BOTTOM]; - const s16 off = insn->off; -+#ifdef CONFIG_HIVE -+ u8 off_reg; -+#endif - const s32 imm = insn->imm; - const int i = insn - ctx->prog->insnsi; - const bool is64 = BPF_CLASS(code) == BPF_ALU64 || -@@ -1237,6 +1292,33 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, - return ret; - break; - -+#ifdef CONFIG_HIVE -+ case BPF_LDX | BPF_REG | BPF_W: -+ case BPF_LDX | BPF_REG | BPF_H: -+ case BPF_LDX | BPF_REG | BPF_B: -+ case BPF_LDX | BPF_REG | BPF_DW: -+ off_reg = bpf2a64[off]; -+ switch (BPF_SIZE(code)) { -+ case BPF_W: -+ emit(A64_LDR32(dst, src, off_reg), ctx); -+ break; -+ case BPF_H: -+ emit(A64_LDRH(dst, src, off_reg), ctx); -+ break; -+ case BPF_B: -+ emit(A64_LDRB(dst, src, off_reg), ctx); -+ break; -+ case BPF_DW: -+ emit(A64_LDR64(dst, src, off_reg), ctx); -+ break; -+ } -+ -+ ret = add_exception_handler(insn, ctx, dst); -+ if (ret) -+ return ret; -+ break; -+#endif -+ - /* speculation barrier */ - case BPF_ST | BPF_NOSPEC: - /* -@@ -1347,6 +1429,29 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, - break; - } - break; -+ -+ #ifdef CONFIG_HIVE -+ case BPF_STX | BPF_REG | BPF_W: -+ case BPF_STX | BPF_REG | BPF_H: -+ case BPF_STX | BPF_REG | BPF_B: -+ case BPF_STX | BPF_REG | BPF_DW: -+ off_reg = bpf2a64[off]; -+ switch (BPF_SIZE(code)) { -+ case BPF_W: -+ emit(A64_STR32(src, dst, off_reg), ctx); -+ break; -+ case BPF_H: -+ emit(A64_STRH(src, dst, off_reg), ctx); -+ break; -+ case BPF_B: -+ emit(A64_STRB(src, dst, off_reg), ctx); -+ break; -+ case BPF_DW: -+ emit(A64_STR64(src, dst, off_reg), ctx); -+ break; -+ } -+ break; -+ #endif - - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index a6bbe6029121..d307e6ec992a 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -1549,6 +1549,25 @@ config AMD_MEM_ENCRYPT - This requires an AMD processor that supports Secure Memory - Encryption (SME). - -+config CREDP -+ bool "Config for credentials isolation" -+ depends on IEE -+ def_bool y -+ -+config IEE -+ bool "Config for Isolated Executed Environment" -+ depends on X86_64 -+ def_bool y -+ -+config KEYP -+ depends on IEE -+ def_bool y -+ -+config IEE_SELINUX_P -+ depends on IEE -+ depends on SECURITY_SELINUX -+ def_bool n -+ - # Common NUMA Features - config NUMA - bool "NUMA Memory Allocation and Scheduler Support" -@@ -2468,6 +2487,12 @@ source "kernel/livepatch/Kconfig" - - endmenu - -+# Config for kernel module isolation -+config KOI -+ bool "Config for Kernel Module Isolation" -+ depends on X86_64 -+ def_bool n -+ - config CC_HAS_SLS - def_bool $(cc-option,-mharden-sls=all) - -diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c -index a7b4148a943f..23e85a84e45b 100644 ---- a/arch/x86/boot/compressed/ident_map_64.c -+++ b/arch/x86/boot/compressed/ident_map_64.c -@@ -23,7 +23,11 @@ - /* Use the static base for this part of the boot process */ - #undef __PAGE_OFFSET - #define __PAGE_OFFSET __PAGE_OFFSET_BASE -+#ifdef CONFIG_PTP -+#include "../../mm/ident_map_for_iee.c" -+#else - #include "../../mm/ident_map.c" -+#endif - - #define _SETUP - #include /* For COMMAND_LINE_SIZE */ -@@ -101,9 +105,15 @@ void kernel_add_identity_map(unsigned long start, unsigned long end) - return; - - /* Build the mapping. */ -+ #ifdef CONFIG_PTP -+ ret = kernel_ident_mapping_init_for_iee(&mapping_info, (pgd_t *)top_level_pgt, start, end); -+ if (ret) -+ error("Error: kernel_ident_mapping_init_for_iee() failed\n"); -+ #else - ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); - if (ret) - error("Error: kernel_ident_mapping_init() failed\n"); -+ #endif - } - - /* Locates and clears a region for a new top level page table. */ -@@ -179,7 +189,11 @@ void initialize_identity_maps(void *rmode) - sev_prep_identity_maps(top_level_pgt); - - /* Load the new page-table. */ -+ #ifdef CONFIG_IEE -+ native_write_cr3_pre_init(top_level_pgt); -+ #else - write_cr3(top_level_pgt); -+ #endif - - /* - * Now that the required page table mappings are established and a -@@ -207,7 +221,11 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, - - /* Populate the PTEs */ - for (i = 0; i < PTRS_PER_PMD; i++) { -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(&pte[i], __pte(address | page_flags)); -+ #else - set_pte(&pte[i], __pte(address | page_flags)); -+ #endif - address += PAGE_SIZE; - } - -@@ -221,9 +239,17 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, - * of a TLB multihit. - */ - pmd = __pmd((unsigned long)pte | info->kernpg_flag); -+ #ifdef CONFIG_PTP -+ iee_set_pmd_pre_init(pmdp, pmd); -+ #else - set_pmd(pmdp, pmd); -+ #endif - /* Flush TLB to establish the new PMD */ -+ #ifdef CONFIG_IEE -+ native_write_cr3_pre_init(top_level_pgt); -+ #else - write_cr3(top_level_pgt); -+ #endif - - return pte + pte_index(__address); - } -@@ -313,7 +339,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, - pte = *ptep; - pte = pte_set_flags(pte, set); - pte = pte_clear_flags(pte, clr); -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(ptep, pte); -+ #else - set_pte(ptep, pte); -+ #endif - - /* - * If the encryption attribute is being set, then change the page state to -@@ -324,7 +354,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, - snp_set_page_private(__pa(address & PAGE_MASK)); - - /* Flush TLB after changing encryption attribute */ -+ #ifdef CONFIG_IEE -+ native_write_cr3_pre_init(top_level_pgt); -+ #else - write_cr3(top_level_pgt); -+ #endif - - return 0; - } -diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c -index 7939eb6e6ce9..c922aec0d88a 100644 ---- a/arch/x86/boot/compressed/pgtable_64.c -+++ b/arch/x86/boot/compressed/pgtable_64.c -@@ -196,7 +196,11 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) - * Move the top level page table out of trampoline memory. - */ - memcpy(pgtable, trampoline_32bit, PAGE_SIZE); -+ #ifdef CONFIG_IEE -+ native_write_cr3_pre_init((unsigned long)pgtable); -+ #else - native_write_cr3((unsigned long)pgtable); -+ #endif - - /* Restore trampoline memory */ - memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); -diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S -index 2192b6c33ea0..d7dd2d7b6138 100644 ---- a/arch/x86/entry/entry_64.S -+++ b/arch/x86/entry/entry_64.S -@@ -663,6 +663,17 @@ SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) - jz 1f - ud2 - 1: -+#endif -+#ifdef CONFIG_KOI -+ movq %cr4, %rcx -+ testq $(1 << 17), %rcx -+ jz 2f -+ # movq koi_kern_cr3(%rip), %rcx -+ # cmpq $0, %rcx -+ # je 2f -+ movq CS-8(%rsp), %rdi -+ call koi_error_return_to_ko -+2: - #endif - POP_REGS - addq $8, %rsp /* skip regs->orig_ax */ -@@ -1072,6 +1083,27 @@ SYM_CODE_START(error_entry) - * for these here too. - */ - .Lerror_kernelspace: -+#ifdef CONFIG_KOI -+ movq %cr3, %rcx -+ testq $(1 << 11), %rcx -+ jz 1f -+ pushq %rdx -+ pushq %rsi -+ pushq %rdi -+ pushq %r8 -+ pushq %r9 -+ pushq %r10 -+ pushq %r11 -+ call koi_error_entry_to_kernel -+ popq %r11 -+ popq %r10 -+ popq %r9 -+ popq %r8 -+ popq %rdi -+ popq %rsi -+ popq %rdx -+1: -+#endif - leaq native_irq_return_iret(%rip), %rcx - cmpq %rcx, RIP+8(%rsp) - je .Lerror_bad_iret -diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h -index ab97b22ac04a..cef8de1be0c0 100644 ---- a/arch/x86/include/asm/desc.h -+++ b/arch/x86/include/asm/desc.h -@@ -13,6 +13,10 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#endif -+ - static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info) - { - desc->limit0 = info->limit & 0x0ffff; -@@ -210,9 +214,23 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) - asm volatile("lgdt %0"::"m" (*dtr)); - } - -+#ifdef CONFIG_IEE -+static __always_inline void iee_load_idt_pre_init(const struct desc_ptr *dtr) -+{ -+ asm volatile("lidt %0"::"m" (*dtr)); -+} -+#endif -+ - static __always_inline void native_load_idt(const struct desc_ptr *dtr) - { -+ /* IEE note: load_idt in __restore_processor_state() is not triggered in qemu, -+ * but still hooked -+ */ -+ #ifdef CONFIG_IEE -+ iee_rwx_gate(IEE_LOAD_IDT, dtr); -+ #else - asm volatile("lidt %0"::"m" (*dtr)); -+ #endif - } - - static inline void native_store_gdt(struct desc_ptr *dtr) -diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h -index d0dcefb5cc59..c6dbea6ab469 100644 ---- a/arch/x86/include/asm/fixmap.h -+++ b/arch/x86/include/asm/fixmap.h -@@ -196,5 +196,10 @@ void __init *early_memremap_decrypted_wp(resource_size_t phys_addr, - void __early_set_fixmap(enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags); - -+#ifdef CONFIG_PTP -+void __iee_set_fixmap_pre_init(enum fixed_addresses idx, -+ phys_addr_t phys, pgprot_t flags); -+#endif -+ - #endif /* !__ASSEMBLY__ */ - #endif /* _ASM_X86_FIXMAP_H */ -diff --git a/arch/x86/include/asm/iee-access.h b/arch/x86/include/asm/iee-access.h -new file mode 100644 -index 000000000000..3eb2346afd6a ---- /dev/null -+++ b/arch/x86/include/asm/iee-access.h -@@ -0,0 +1,39 @@ -+#ifndef _LINUX_IEE_ACCESS_H -+#define _LINUX_IEE_ACCESS_H -+ -+#include -+#include -+ -+extern unsigned long long iee_rw_gate(int flag, ...); -+ -+static inline void iee_memcpy(void *dst, const void *src, size_t n) -+{ -+ iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); -+} -+ -+static inline void iee_memset(void *ptr, int data, size_t n) -+{ -+ iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); -+} -+ -+static inline void iee_set_track(struct track *ptr, struct track *data) -+{ -+ iee_rw_gate(IEE_OP_SET_TRACK, ptr, data); -+} -+ -+static inline void iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr) -+{ -+ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); -+} -+ -+static inline void iee_split_huge_pmd(pmd_t *pmdp, pte_t *pgtable) -+{ -+ iee_rw_gate(IEE_OP_SPLIT_HUGE_PMD, pmdp, pgtable); -+} -+ -+static inline unsigned long iee_test_and_clear_bit(long nr, volatile unsigned long *addr) -+{ -+ return iee_rw_gate(IEE_OP_TEST_CLEAR_BIT, nr, addr); -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-cred.h b/arch/x86/include/asm/iee-cred.h -new file mode 100644 -index 000000000000..44ae20ce05ad ---- /dev/null -+++ b/arch/x86/include/asm/iee-cred.h -@@ -0,0 +1,148 @@ -+#ifndef _LINUX_IEE_CRED_H -+#define _LINUX_IEE_CRED_H -+ -+#include -+#include -+ -+extern unsigned long long iee_rw_gate(int flag, ...); -+ -+static void __maybe_unused iee_copy_cred(const struct cred *old, struct cred *new) -+{ -+ iee_rw_gate(IEE_OP_COPY_CRED,old,new); -+} -+ -+static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_UID,cred,uid); -+} -+ -+static void __maybe_unused iee_set_cred_gid(struct cred *cred, kgid_t gid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_GID,cred,gid); -+} -+ -+static void __maybe_unused iee_set_cred_suid(struct cred *cred, kuid_t suid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_SUID,cred,suid); -+} -+ -+static void __maybe_unused iee_set_cred_sgid(struct cred *cred, kgid_t sgid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_SGID,cred,sgid); -+} -+ -+static void __maybe_unused iee_set_cred_euid(struct cred *cred, kuid_t euid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_EUID,cred,euid); -+} -+ -+static void __maybe_unused iee_set_cred_egid(struct cred *cred, kgid_t egid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_EGID,cred,egid); -+} -+ -+static void __maybe_unused iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_FSUID,cred,fsuid); -+} -+ -+static void __maybe_unused iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_FSGID,cred,fsgid); -+} -+ -+static void __maybe_unused iee_set_cred_user(struct cred *cred, struct user_struct *user) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_USER,cred,user); -+} -+ -+static void __maybe_unused iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_USER_NS,cred,user_ns); -+} -+ -+static void __maybe_unused iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS,cred,ucounts); -+} -+ -+static void __maybe_unused iee_set_cred_group_info(struct cred *cred, struct group_info *group_info) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO,cred,group_info); -+} -+ -+static void __maybe_unused iee_set_cred_securebits(struct cred *cred, unsigned securebits) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS,cred,securebits); -+} -+ -+static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER,cred,cap_inheritable); -+} -+ -+static void __maybe_unused iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM,cred,cap_permitted); -+} -+ -+static void __maybe_unused iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT,cred,cap_effective); -+} -+ -+static void __maybe_unused iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET,cred,cap_bset); -+} -+ -+static void __maybe_unused iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT,cred,cap_ambient); -+} -+ -+#ifdef CONFIG_KEYS -+static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING,cred,jit_keyring); -+} -+ -+static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING,cred,session_keyring); -+} -+ -+static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING,cred,process_keyring); -+} -+ -+static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING,cred,thread_keyring); -+} -+ -+static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING,cred,request_key_auth); -+} -+#endif -+ -+static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE,cred,i); -+} -+ -+#ifdef CONFIG_SECURITY -+static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_SECURITY,cred,security); -+} -+#endif -+ -+static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) -+{ -+ iee_rw_gate(IEE_OP_SET_CRED_RCU,cred,rcu); -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-def.h b/arch/x86/include/asm/iee-def.h -new file mode 100644 -index 000000000000..d9a712c2420e ---- /dev/null -+++ b/arch/x86/include/asm/iee-def.h -@@ -0,0 +1,124 @@ -+#ifndef _LINUX_IEE_DEF_H -+#define _LINUX_IEE_DEF_H -+ -+#ifdef CONFIG_CREDP -+#define AT_ADD 1 -+#define AT_INC_NOT_ZERO 2 -+#define AT_SUB_AND_TEST 3 -+#endif -+ -+#ifdef CONFIG_KEYP -+#define REFCOUNT_INC 1 -+#define REFCOUNT_SET 2 -+#define REFCOUNT_DEC_AND_TEST 3 -+#define REFCOUNT_INC_NOT_ZERO 4 -+ -+#define SET_BIT_OP 1 -+#define TEST_AND_CLEAR_BIT 2 -+#define TEST_AND_SET_BIT 3 -+#endif -+ -+enum { -+ #ifdef CONFIG_PTP -+ IEE_OP_SET_PTE, -+ IEE_OP_SET_PMD, -+ IEE_OP_SET_PUD, -+ IEE_OP_SET_P4D, -+ IEE_OP_SET_PGD, -+ #endif -+ IEE_OP_MEMCPY, -+ IEE_OP_MEMSET, -+ IEE_OP_SET_FREEPTR, -+ IEE_OP_SPLIT_HUGE_PMD, -+ IEE_OP_SET_TOKEN_PGD, -+ IEE_OP_INIT_TOKEN, -+ IEE_OP_INVALIDATE_TOKEN, -+ IEE_OP_VALIDATE_TOKEN, -+ IEE_OP_UNSET_TOKEN, -+ IEE_OP_SET_TOKEN, -+ IEE_OP_SET_TRACK, -+ IEE_OP_TEST_CLEAR_BIT, -+ IEE_SET_SENSITIVE_PTE, -+ IEE_UNSET_SENSITIVE_PTE, -+#ifdef CONFIG_CREDP -+ IEE_OP_COPY_CRED, // Parameters: struct cred *old, struct cred *new -+ IEE_OP_SET_CRED_UID, // Parameters: struct cred *cred, kuid_t uid -+ IEE_OP_SET_CRED_GID, // Parameters: struct cred *cred, kgid_t gid -+ IEE_OP_SET_CRED_SUID, // Parameters: struct cred *cred, kuid_t suid -+ IEE_OP_SET_CRED_SGID, // Parameters: struct cred *cred, kgid_t sgid -+ IEE_OP_SET_CRED_EUID, // Parameters: struct cred *cred, kuid_t euid -+ IEE_OP_SET_CRED_EGID, // Parameters: struct cred *cred, kgid_t egid -+ IEE_OP_SET_CRED_FSUID, // Parameters: struct cred *cred, kuid_t fsuid -+ IEE_OP_SET_CRED_FSGID, // Parameters: struct cred *cred, kgid_t fsgid -+ IEE_OP_SET_CRED_USER, // Parameters: struct cred *cred, struct user_struct *user -+ IEE_OP_SET_CRED_USER_NS, // Parameters: struct cred *cred, struct user_namespace *user_ns -+ IEE_OP_SET_CRED_GROUP_INFO, // Parameters: struct cred *cred, struct group_info *group_info -+ IEE_OP_SET_CRED_SECUREBITS, // Parameters: struct cred *cred, unsigned securebits -+ IEE_OP_SET_CRED_CAP_INHER, // Parameters: struct cred *cred, kernel_cap_t cap_inheritable -+ IEE_OP_SET_CRED_CAP_PERM, // Parameters: struct cred *cred, kernel_cap_t cap_permitted -+ IEE_OP_SET_CRED_CAP_EFFECT, // Parameters: struct cred *cred, kernel_cap_t cap_effective -+ IEE_OP_SET_CRED_CAP_BSET, // Parameters: struct cred *cred, kernel_cap_t cap_bset -+ IEE_OP_SET_CRED_CAP_AMBIENT, // Parameters: struct cred *cred, kernel_cap_t cap_ambient -+ IEE_OP_SET_CRED_JIT_KEYRING, // Parameters: struct cred *cred, unsigned char jit_keyring -+ IEE_OP_SET_CRED_SESS_KEYRING, // Parameters: struct cred *cred, struct key *session_keyring -+ IEE_OP_SET_CRED_PROC_KEYRING, // Parameters: struct cred *cred, struct key *process_keyring -+ IEE_OP_SET_CRED_THREAD_KEYRING, // Parameters: struct cred *cred, struct key *thread_keyring -+ IEE_OP_SET_CRED_REQ_KEYRING, // Parameters: struct cred *cred, struct key *request_key_auth -+ IEE_OP_SET_CRED_NON_RCU, // Parameters: struct cred *cred, int non_rcu -+ IEE_OP_SET_CRED_ATSET_USAGE, // Parameters: struct cred *cred, int i -+ IEE_OP_SET_CRED_ATOP_USAGE, // Parameters: struct cred *cred, int flag -+ IEE_OP_SET_CRED_SECURITY, // Parameters: struct cred *cred, void *security -+ IEE_OP_SET_CRED_RCU, // Parameters: struct cred *cred, struct rcu_head *rcu -+ IEE_OP_SET_CRED_UCOUNTS, // Parameters: struct cred *cred, struct ucounts *ucounts -+#endif -+#ifdef CONFIG_KEYP -+ IEE_OP_SET_KEY_UNION, -+ IEE_OP_SET_KEY_STRUCT, -+ IEE_OP_SET_KEY_PAYLOAD, -+ IEE_OP_SET_KEY_USAGE, -+ IEE_OP_SET_KEY_SERIAL, -+ IEE_OP_SET_KEY_WATCHERS, -+ IEE_OP_SET_KEY_USERS, -+ IEE_OP_SET_KEY_SECURITY, -+ IEE_OP_SET_KEY_EXPIRY, -+ IEE_OP_SET_KEY_REVOKED_AT, -+ IEE_OP_SET_KEY_LAST_USED_AT, -+ IEE_OP_SET_KEY_UID, -+ IEE_OP_SET_KEY_GID, -+ IEE_OP_SET_KEY_PERM, -+ IEE_OP_SET_KEY_QUOTALEN, -+ IEE_OP_SET_KEY_DATALEN, -+ IEE_OP_SET_KEY_STATE, -+ IEE_OP_SET_KEY_MAGIC, -+ IEE_OP_SET_KEY_FLAGS, -+ IEE_OP_SET_KEY_INDEX_KEY, -+ IEE_OP_SET_KEY_HASH, -+ IEE_OP_SET_KEY_LEN_DESC, -+ IEE_OP_SET_KEY_TYPE, -+ IEE_OP_SET_KEY_TAG, -+ IEE_OP_SET_KEY_DESCRIPTION, -+ IEE_OP_SET_KEY_RESTRICT_LINK, -+ IEE_OP_SET_KEY_FLAG_BIT, -+#endif -+#ifdef CONFIG_IEE_SELINUX_P -+ IEE_SEL_SET_STATUS_PG, // Parameters: struct page* new_page -+ IEE_SEL_SET_ENFORCING, // Parameters: bool value -+ IEE_SEL_SET_INITIALIZED, -+ IEE_SEL_SET_POLICY_CAP, // Parameters: unsigned int idx, int cap -+ IEE_SEL_RCU_ASSIGN_POLICY, // Parameters: struct selinux_policy* new_policy, struct selinux_policy* iee_new_policy -+#endif -+#ifdef CONFIG_KOI -+ IEE_READ_KOI_STACK, // Parameters: struct task_struct *tsk -+ IEE_WRITE_KOI_STACK, // Parameters: struct task_struct *tsk, unsigned long koi_stack -+ IEE_READ_TOKEN_TTBR1, // Parameters: struct task_struct *tsk -+ IEE_WRITE_TOKEN_TTBR1, // Parameters: struct task_struct *tsk, unsigned long current_ttbr1 -+ IEE_READ_KOI_KERNEL_STACK, // Parameters: struct task_struct *tsk -+ IEE_WRITE_KOI_KERNEL_STACK, // Parameters: struct task_struct *tsk, unsigned long kernel_stack -+ IEE_READ_KOI_STACK_BASE, // Parameters: struct task_struct *tsk -+ IEE_WRITE_KOI_STACK_BASE, // Parameters: struct task_struct *tsk, unsigned long koi_stack_base -+ IEE_SET_KOI_PGD, // Parameters: unsigned long koi_pgd_addr -+#endif -+ IEE_FLAG_END -+}; -+ -+#endif /* _LINUX_IEE_DEF_H */ -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-key.h b/arch/x86/include/asm/iee-key.h -new file mode 100644 -index 000000000000..fcac44558cce ---- /dev/null -+++ b/arch/x86/include/asm/iee-key.h -@@ -0,0 +1,147 @@ -+#ifndef _LINUX_IEE_KEY_H -+#define _LINUX_IEE_KEY_H -+ -+#include -+#include -+ -+extern unsigned long long iee_rw_gate(int flag, ...); -+ -+static void __maybe_unused iee_set_key_union(struct key *key, struct key_union *key_union) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_UNION, key, key_union); -+} -+ -+static void __maybe_unused iee_set_key_struct(struct key *key, struct key_struct *key_struct) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_STRUCT, key, key_struct); -+} -+ -+static void __maybe_unused iee_set_key_payload(struct key *key, union key_payload *key_payload) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_PAYLOAD, key, key_payload); -+} -+ -+extern bool iee_set_key_usage(struct key *key, int n, int flag); -+ -+static void __maybe_unused iee_set_key_serial(struct key *key, key_serial_t serial) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_SERIAL, key, serial); -+} -+ -+#ifdef CONFIG_KEY_NOTIFICATIONS -+static void __maybe_unused iee_set_key_watchers(struct key *key, struct watch_list *watchers) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_WATCHERS, key, watchers); -+} -+#endif -+ -+static void __maybe_unused iee_set_key_user(struct key *key, struct key_user *user) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_USERS, key, user); -+} -+ -+static void __maybe_unused iee_set_key_security(struct key *key, void *security) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_SECURITY, key, security); -+} -+ -+static void __maybe_unused iee_set_key_expiry(struct key *key, time64_t expiry) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_EXPIRY, key, expiry); -+} -+ -+static void __maybe_unused iee_set_key_revoked_at(struct key *key, time64_t revoked_at) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_REVOKED_AT, key, revoked_at); -+} -+ -+static void __maybe_unused iee_set_key_last_used_at(struct key *key, time64_t last_used_at) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_LAST_USED_AT, key, last_used_at); -+} -+ -+static void __maybe_unused iee_set_key_uid(struct key *key, kuid_t uid) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_UID, key, uid); -+} -+ -+static void __maybe_unused iee_set_key_gid(struct key *key, kgid_t gid) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_GID, key, gid); -+} -+ -+static void __maybe_unused iee_set_key_perm(struct key *key, key_perm_t perm) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_PERM, key, perm); -+} -+ -+static void __maybe_unused iee_set_key_quotalen(struct key *key, unsigned short quotalen) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_QUOTALEN, key, quotalen); -+} -+ -+static void __maybe_unused iee_set_key_datalen(struct key *key, unsigned short datalen) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_DATALEN, key, datalen); -+} -+ -+static void __maybe_unused iee_set_key_state(struct key *key, short state) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_STATE, key, state); -+} -+ -+#ifdef KEY_DEBUGGING -+static void __maybe_unused iee_set_key_magic(struct key *key, unsigned magic) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_MAGIC, key, magic); -+} -+#endif -+ -+static void __maybe_unused iee_set_key_flags(struct key *key, unsigned long flags) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_FLAGS, key, flags); -+} -+ -+static void __maybe_unused iee_set_key_index_key(struct key *key, struct keyring_index_key* index_key) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_INDEX_KEY, key, index_key); -+} -+ -+static void __maybe_unused iee_set_key_hash(struct key *key, unsigned long hash) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_HASH, key, hash); -+} -+ -+static void __maybe_unused iee_set_key_len_desc(struct key *key, unsigned long len_desc) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_LEN_DESC, key, len_desc); -+} -+ -+static void __maybe_unused iee_set_key_type(struct key *key, struct key_type *type) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_TYPE, key, type); -+} -+ -+static void __maybe_unused iee_set_key_domain_tag(struct key *key, struct key_tag *domain_tag) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_TAG, key, domain_tag); -+} -+ -+static void __maybe_unused iee_set_key_description(struct key *key, char *description) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_DESCRIPTION, key, description); -+} -+ -+static void __maybe_unused iee_set_key_restrict_link(struct key *key, struct key_restriction *restrict_link) -+{ -+ iee_rw_gate(IEE_OP_SET_KEY_RESTRICT_LINK, key, restrict_link); -+} -+ -+static bool __maybe_unused iee_set_key_flag_bit(struct key *key, long nr, int flag) -+{ -+ bool ret; -+ ret = iee_rw_gate(IEE_OP_SET_KEY_FLAG_BIT, key, nr, flag); -+ return ret; -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-koi.h b/arch/x86/include/asm/iee-koi.h -new file mode 100644 -index 000000000000..f55526f8aa60 ---- /dev/null -+++ b/arch/x86/include/asm/iee-koi.h -@@ -0,0 +1,5 @@ -+#if defined(CONFIG_KOI) && defined(CONFIG_IEE) -+#define IEE_SWITCH_TO_KERNEL 5 -+#define IEE_SWITCH_TO_KOI 6 -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-selinuxp.h b/arch/x86/include/asm/iee-selinuxp.h -new file mode 100644 -index 000000000000..29bcadb9979f ---- /dev/null -+++ b/arch/x86/include/asm/iee-selinuxp.h -@@ -0,0 +1,26 @@ -+#ifndef _LINUX_IEE_SELINUX_P_H -+#define _LINUX_IEE_SELINUX_P_H -+ -+#include -+#include "security.h" -+#include "ss/services.h" -+ -+static inline struct mutex* iee_get_selinux_policy_lock(void) -+{ -+ return (struct mutex*)(selinux_state.policy_mutex.owner.counter); -+} -+ -+static inline struct mutex* iee_get_selinux_status_lock(void) -+{ -+ return (struct mutex*)(selinux_state.status_lock.owner.counter); -+} -+ -+/* APIs for modifying selinux_state */ -+extern void iee_set_selinux_status_pg(struct page* new_page); -+extern void iee_set_sel_policy_cap(unsigned int idx, int cap); -+extern void iee_sel_rcu_assign_policy(struct selinux_policy* new_policy, -+ struct selinux_policy* iee_new_policy); -+ -+extern struct kmem_cache *policy_jar; -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-si.h b/arch/x86/include/asm/iee-si.h -new file mode 100644 -index 000000000000..19601dd5a6e4 ---- /dev/null -+++ b/arch/x86/include/asm/iee-si.h -@@ -0,0 +1,25 @@ -+#ifndef _LINUX_IEE_SI_H -+#define _LINUX_IEE_SI_H -+#define __iee_si_code __section(".iee.si_text") -+#define __iee_si_base __section(".iee.si_base") -+#define __iee_si_data __section(".iee.si_data") -+ -+extern bool iee_pgt_jar_init; -+extern bool iee_init_done; -+extern unsigned long iee_base_swapper_pg_dir; -+extern unsigned long __iee_si_text_start[]; -+extern unsigned long __iee_si_text_end[]; -+extern unsigned long __iee_si_data_start[]; -+extern unsigned long __iee_si_data_end[]; -+extern void iee_rwx_gate(int flag, ...); -+// Handler function for sensitive inst -+u64 iee_si_handler(int flag, ...); -+ -+#define IEE_SI_TEST 0 -+#define IEE_WRITE_CR0 1 -+#define IEE_WRITE_CR3 2 -+#define IEE_WRITE_CR4 3 -+#define IEE_LOAD_IDT 4 -+#define IEE_SWITCH_TO_KERNEL 5 -+#define IEE_SWITCH_TO_KOI 6 -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-slab.h b/arch/x86/include/asm/iee-slab.h -new file mode 100644 -index 000000000000..6c79bea4406a ---- /dev/null -+++ b/arch/x86/include/asm/iee-slab.h -@@ -0,0 +1,21 @@ -+#ifndef _LINUX_IEE_SLAB_H -+#define _LINUX_IEE_SLAB_H -+ -+/* -+ * Tracking user of a slab. -+ */ -+#include -+#define TRACK_ADDRS_COUNT 16 -+struct track { -+ unsigned long addr; /* Called from address */ -+#ifdef CONFIG_STACKDEPOT -+ depot_stack_handle_t handle; -+#endif -+ int cpu; /* Was running on cpu */ -+ int pid; /* Pid context */ -+ unsigned long when; /* When did the operation occur */ -+}; -+enum track_item { TRACK_ALLOC, TRACK_FREE }; -+typedef struct { unsigned long v; } freeptr_t; -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee-token.h b/arch/x86/include/asm/iee-token.h -new file mode 100644 -index 000000000000..cd38b16beb25 ---- /dev/null -+++ b/arch/x86/include/asm/iee-token.h -@@ -0,0 +1,32 @@ -+#ifndef _LINUX_IEE_TOKEN_H -+#define _LINUX_IEE_TOKEN_H -+ -+#include -+#include -+ -+extern unsigned long long iee_rw_gate(int flag, ...); -+struct task_token; -+struct task_struct; -+struct mm_struct; -+ -+static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) -+{ -+ iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); -+} -+ -+static inline void iee_init_token(struct task_struct *tsk, void *iee_stack, void *tmp_page) -+{ -+ iee_rw_gate(IEE_OP_INIT_TOKEN, tsk, iee_stack, tmp_page); -+} -+ -+static inline void iee_invalidate_token(struct task_struct *tsk) -+{ -+ iee_rw_gate(IEE_OP_INVALIDATE_TOKEN, tsk); -+} -+ -+static inline void iee_validate_token(struct task_struct *tsk) -+{ -+ iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/iee.h b/arch/x86/include/asm/iee.h -new file mode 100644 -index 000000000000..9b116751df6e ---- /dev/null -+++ b/arch/x86/include/asm/iee.h -@@ -0,0 +1,5 @@ -+#ifndef _LINUX_IEE_H -+#define _LINUX_IEE_H -+#define __iee_code __section(".iee.text") -+#define __iee_header __section(".iee.text.header") -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/koi.h b/arch/x86/include/asm/koi.h -new file mode 100644 -index 000000000000..f0f3ae1623f8 ---- /dev/null -+++ b/arch/x86/include/asm/koi.h -@@ -0,0 +1,432 @@ -+#include "asm/pgtable_types.h" -+#include "linux/percpu-defs.h" -+#include "linux/spinlock_types.h" -+#include "linux/hashtable.h" -+#include "asm/tlbflush.h" -+ -+#define MAX_VAR_NAME 64 -+#define DRIVER_ISOLATION_VAR_ARRAY_SIZE 32 -+#define DRIVER_ISOLATION_MAX_VAL 256 -+ -+#define HASH_TABLE_BIT 10 -+ -+#define PTI_USER_PCID_MASK (1 << X86_CR3_PTI_PCID_USER_BIT) -+ -+DECLARE_PER_CPU_PAGE_ALIGNED(unsigned long, koi_kern_cr3); -+ -+extern struct hlist_head koi_mem_htbl[1024]; -+ -+extern unsigned long koi_offset; -+ -+extern unsigned long koi_cr3_ctor(struct module *mod); -+ -+extern int koi_share_kstack(struct module *mod); -+ -+void koi_map_kostack(struct module *mod); -+ -+#ifdef CONFIG_IEE -+#include "asm/page.h" -+extern unsigned long IEE_OFFSET; -+#endif -+/** -+* struct koi_mem_hash_node - -+*@mod:pointer to driver module -+*@mem_list_head:free memory list head -+*@ko_mm: mm_struct in each driver -+*@pgdp:entry to Page Global Directory :pgd -+*@node:hash linked list node -+*@addr_htbl[1 << (HASH_TABLE_BIT)]: -+*@rcu: -+*/ -+struct koi_mem_hash_node { -+ struct module *mod; -+ struct mm_struct *ko_mm; -+ pgd_t *pgdp; -+ unsigned long ko_cr3; -+ struct hlist_node node; -+ bool is_valid; -+ spinlock_t mod_lock; -+}; -+ -+struct shared_variable_descriptor { -+ unsigned int id; -+ unsigned int type; -+ char name[MAX_VAR_NAME]; -+ unsigned long offset; -+ unsigned int size; -+ unsigned int self_ptr_ids[DRIVER_ISOLATION_VAR_ARRAY_SIZE]; -+}; -+ -+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, -+ unsigned long addr, unsigned long end, pteval_t prot); -+ -+void koi_create_pagetable(struct module *mod); -+void koi_destroy_pagetable(struct module *mod); -+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size); -+void koi_unmap_mem(struct module *mod, unsigned long addr, unsigned long size); -+ -+#ifndef CONFIG_IEE -+#define __koi_switch_to_ko(mod) \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ struct task_token *token = (struct task_token *)(__phys_to_koi(__pa(current))); \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = __read_cr3(); \ -+ this_cpu_write(koi_kern_cr3, new_cr3); \ -+ new_cr3 = koi_cr3_ctor(mod); \ -+ token->current_ttbr1 = new_cr3 & (~X86_CR3_PCID_MASK); \ -+ native_write_cr3(new_cr3); \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+ -+#define koi_switch_to_ko() \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ struct task_token *token = (struct task_token *)(__phys_to_koi(__pa(current))); \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = __read_cr3(); \ -+ this_cpu_write(koi_kern_cr3, new_cr3); \ -+ new_cr3 = koi_cr3_ctor(THIS_MODULE); \ -+ token->current_ttbr1 = new_cr3 & (~X86_CR3_PCID_MASK); \ -+ native_write_cr3(new_cr3); \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+ -+#define koi_switch_to_kernel() \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = this_cpu_read(koi_kern_cr3); \ -+ /*pcid = __read_cr3(); \ -+ pcid &= X86_CR3_PCID_MASK; \ -+ pcid &= ~PTI_USER_PCID_MASK;*/ \ -+ native_write_cr3(new_cr3); \ -+ struct task_token *token = (struct task_token *)(__phys_to_koi(__pa(current))); \ -+ token->current_ttbr1 = new_cr3; \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+#else -+#define __koi_switch_to_ko(mod) \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = __read_cr3(); \ -+ this_cpu_write(koi_kern_cr3, new_cr3); \ -+ new_cr3 = koi_cr3_ctor(mod); \ -+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, new_cr3 & (~X86_CR3_PCID_MASK)); \ -+ /* iee_rwx_gate(IEE_SWITCH_TO_KOI, new_cr3); */ \ -+ asm volatile("mov %0,%%cr3": : "r" (new_cr3) : "memory"); \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+ -+#define koi_switch_to_ko() \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = __read_cr3(); \ -+ this_cpu_write(koi_kern_cr3, new_cr3); \ -+ new_cr3 = koi_cr3_ctor(THIS_MODULE); \ -+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, new_cr3 & (~X86_CR3_PCID_MASK)); \ -+ /* iee_rwx_gate(IEE_SWITCH_TO_KOI, new_cr3); */ \ -+ asm volatile("mov %0,%%cr3": : "r" (new_cr3) : "memory"); \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+ -+#define koi_switch_to_kernel() \ -+ do { \ -+ unsigned long flags, new_cr3; \ -+ asm volatile( \ -+ "pushf\n\t" \ -+ "pop %0\n\t" \ -+ "cli\n\t" \ -+ : "=r"(flags) \ -+ : \ -+ : "memory" \ -+ ); \ -+ new_cr3 = this_cpu_read(koi_kern_cr3); \ -+ /*pcid = __read_cr3(); \ -+ pcid &= X86_CR3_PCID_MASK; \ -+ pcid &= ~PTI_USER_PCID_MASK;*/ \ -+ asm volatile("mov %0,%%cr3": : "r" (new_cr3) : "memory"); \ -+ /* iee_rwx_gate(IEE_SWITCH_TO_KERNEL, new_cr3);*/ \ -+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, new_cr3); \ -+ if (!arch_irqs_disabled_flags(flags)) \ -+ arch_local_irq_enable(); \ -+ } while (0); -+#endif -+ -+ -+ -+ -+#ifdef CONFIG_KOI -+ -+extern void *koi_kcalloc_wrapper(struct module *mod, size_t n, size_t size, gfp_t flags); -+extern void *koi_kmalloc_array_wrapper(struct module *mod, size_t n, size_t size, gfp_t flags); -+extern void *koi_vmalloc_wrapper(struct module *mod, unsigned long size); -+extern void *koi_kmalloc_wrapper(struct module *mod, size_t size, gfp_t flags); -+extern void *koi_kzalloc_node_wrapper(struct module *mod, size_t size, gfp_t flags, int node); -+extern void *koi_kzalloc_wrapper(struct module *mod, size_t size, gfp_t flags); -+ -+ -+#define koi_copy_to_user_wrapper(to, from, n) \ -+({ \ -+ koi_switch_to_kernel(); \ -+ long long ret = copy_to_user(to, from, n); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+}) -+ -+#define koi_copy_from_user_wrapper(to, from, n) \ -+({ \ -+ koi_switch_to_kernel(); \ -+ long long ret = copy_from_user(to, from, n); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+}) -+ -+#define koi_kasprintf_wrapper(gfp, fmt, args...)\ -+ ({ \ -+ koi_switch_to_kernel(); \ -+ void *ret = kasprintf(gfp, fmt, ##args); \ -+ koi_map_mem(THIS_MODULE, (unsigned long)ret, sizeof(void *)); \ -+ koi_switch_to_ko(); \ -+ ret;\ -+ }) -+ -+#define koi_sprintf_wrapper(buf, fmt, args...) \ -+({ \ -+ koi_switch_to_kernel(); \ -+ int ret = sprintf(buf, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+}) -+ -+#define koi_scnprintf_wrapper(buf, size, fmt, args...) \ -+ ({ \ -+ int ret; \ -+ koi_switch_to_kernel(); \ -+ ret = scnprintf(buf, size, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+ }) -+ -+#define koi_sscanf_wrapper(buf, fmt, args...) \ -+ ({ \ -+ int ret; \ -+ koi_switch_to_kernel(); \ -+ ret = sscanf(buf, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+ }) -+ -+#define koi_printk_wrapper(arg_0, args...) \ -+ ({ \ -+ int ret; \ -+ koi_switch_to_kernel(); \ -+ ret = printk(arg_0, ##args); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+ }) -+ -+#define PTR_ERR_wrapper(arg) \ -+({ \ -+ long ret; \ -+ koi_switch_to_kernel(); \ -+ ret = PTR_ERR(arg); \ -+ koi_switch_to_ko(); \ -+ ret; \ -+}) -+#define koi_rcu_read_lock_wrapper() \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ rcu_read_lock(); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_rcu_read_unlock_wrapper() \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ rcu_read_unlock(); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_lock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_lock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_unlock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_unlock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_mutex_init_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ mutex_init(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_unlock_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_irq_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_irq(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+ -+#define koi_spin_unlock_irq_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock_irq(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_lock_irqsave_wrapper(lock, flags) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_irqsave(lock, flags); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+ -+#define koi_spin_unlock_irqrestore_wrapper(lock, flags) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock_irqrestore(lock, flags); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+ -+#define koi_spin_lock_bh_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_lock_bh(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_spin_unlock_bh_wrapper(lock) \ -+ do { \ -+ koi_switch_to_kernel(); \ -+ spin_unlock_bh(lock); \ -+ koi_switch_to_ko(); \ -+ } while(0); -+ -+#define koi_dev_err_wrapper(dev, fmt, args...) \ -+ ({ \ -+ koi_switch_to_kernel(); \ -+ dev_err(dev, fmt, ##args); \ -+ koi_switch_to_ko(); \ -+ }) -+ -+#else -+ -+#define koi_copy_to_user_wrapper copy_to_user -+ -+#define koi_copy_from_user_wrapper copy_from_user -+ -+#define koi_kasprintf_wrapper kasprintf -+ -+#define koi_scnprintf_wrapper scnprintf -+ -+#define koi_sscanf_wrapper sscanf -+ -+#define koi_sprintf_wrapper sprintf -+ -+#define koi_rcu_read_lock_wrapper rcu_read_lock -+ -+#define koi_rcu_read_unlock_wrapper rcu_read_unlock -+ -+#define koi_mutex_lock_wrapper mutex_lock -+ -+#define koi_mutex_unlock_wrapper mutex_unlock -+ -+#define koi_mutex_init_wrapper mutex_init -+ -+#define koi_spin_lock_irq_wrapper spin_lock_irq -+ -+#define koi_spin_unlock_irq_wrapper spin_unlock_irq -+ -+#define koi_spin_lock_wrapper spin_lock -+ -+#define koi_spin_unlock_wrapper spin_unlock -+ -+#define koi_spin_lock_irqsave_wrapper spin_lock_irqsave -+ -+#define koi_spin_unlock_irqrestore_wrapper spin_lock_irqrestore -+ -+#define koi_spin_lock_bh_wrapper spin_lock_bh -+ -+#define koi_spin_unlock_bh_wrapper spin_unlock_bh -+ -+#define koi_kzalloc_wrapper(mod, size, flags) kzalloc(size, flags) -+ -+#define koi_kzalloc_node_wrapper(mod, size, flags, node) kzalloc_node(size, flags, node) -+ -+#define koi_kmalloc_wrapper(mod, size, flags) kmalloc(size, flags) -+ -+#define koi_vmalloc_wrapper(mod, size) vmalloc(size) -+ -+#define koi_kmalloc_array_wrapper(mod, n, size, flags) kmalloc_array(n, size, flags) -+ -+#define koi_kcalloc_wrapper(mod, n, size, flags) kcalloc(n, size, flags) -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h -index 1b93ff80b43b..4b27230dff49 100644 ---- a/arch/x86/include/asm/page.h -+++ b/arch/x86/include/asm/page.h -@@ -61,6 +61,26 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, - #define __boot_va(x) __va(x) - #define __boot_pa(x) __pa(x) - -+#ifdef CONFIG_IEE -+extern unsigned long IEE_OFFSET; -+#ifndef __iee_pa -+#define __iee_pa(x) (__pa(x - IEE_OFFSET)) -+#endif -+#ifndef __phys_to_iee -+#define __phys_to_iee(x) ((void *)(__va(x) + IEE_OFFSET)) -+#endif -+#else -+#ifdef CONFIG_KOI -+extern unsigned long KOI_OFFSET; -+#ifndef __koi_pa -+#define __koi_pa(x) (__pa(x - KOI_OFFSET)) -+#endif -+#ifndef __phys_to_koi -+#define __phys_to_koi(x) ((void *)(__va(x) + KOI_OFFSET)) -+#endif -+#endif -+#endif /* CONFIG_IEE*/ -+ - /* - * virt_to_page(kaddr) returns a valid pointer if and only if - * virt_addr_valid(kaddr) returns true. -diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h -index 9dab85aba7af..e14b0b574e0f 100644 ---- a/arch/x86/include/asm/page_64.h -+++ b/arch/x86/include/asm/page_64.h -@@ -19,6 +19,10 @@ extern unsigned long vmalloc_base; - extern unsigned long vmemmap_base; - extern unsigned long physmem_end; - -+#ifdef CONFIG_PTP -+extern unsigned long iee_ptdesc_base; -+#endif -+ - static __always_inline unsigned long __phys_addr_nodebug(unsigned long x) - { - unsigned long y = x - __START_KERNEL_map; -diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h -index 86bd4311daf8..c8656ccb03c1 100644 ---- a/arch/x86/include/asm/page_types.h -+++ b/arch/x86/include/asm/page_types.h -@@ -29,6 +29,13 @@ - - #define PAGE_OFFSET ((unsigned long)__PAGE_OFFSET) - -+#ifdef CONFIG_IEE -+#define SET_UPAGE(x) __pg(pgprot_val(x) | _PAGE_USER) -+#endif /* CONFIG_IEE*/ -+#if defined (CONFIG_IEE) || defined (CONFIG_KOI) -+#define SET_NG(x) __pg(pgprot_val(x) & (~_PAGE_GLOBAL)) -+#endif -+ - #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC - - #define __PHYSICAL_START ALIGN(CONFIG_PHYSICAL_START, \ -diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h -index c7ec5bb88334..cd9505d0c769 100644 ---- a/arch/x86/include/asm/pgalloc.h -+++ b/arch/x86/include/asm/pgalloc.h -@@ -12,6 +12,10 @@ - - static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } - -+#ifdef CONFIG_KOI -+pgd_t *koi_pgd_alloc(void); -+#endif -+ - #ifdef CONFIG_PARAVIRT_XXL - #include - #else -@@ -147,13 +151,78 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4 - set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); - } - -+#ifdef CONFIG_PTP -+#include -+ -+static inline void iee_pmd_populate_kernel_pre_init(struct mm_struct *mm, -+ pmd_t *pmd, pte_t *pte) -+{ -+ paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); -+ iee_set_pmd_pre_init(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); -+} -+static inline void iee_pmd_populate_kernel_safe_pre_init(struct mm_struct *mm, -+ pmd_t *pmd, pte_t *pte) -+{ -+ paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); -+ iee_set_pmd_safe_pre_init(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); -+} -+ -+static inline void iee_pud_populate_pre_init(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) -+{ -+ paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); -+ iee_set_pud_pre_init(pud, __pud(_PAGE_TABLE | __pa(pmd))); -+} -+ -+static inline void iee_pud_populate_safe_pre_init(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) -+{ -+ paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); -+ iee_set_pud_safe_pre_init(pud, __pud(_PAGE_TABLE | __pa(pmd))); -+} -+ -+static inline void iee_p4d_populate_pre_init(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) -+{ -+ paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); -+ iee_set_p4d_pre_init(p4d, __p4d(_PAGE_TABLE | __pa(pud))); -+} -+ -+static inline void iee_p4d_populate_safe_pre_init(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) -+{ -+ paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); -+ iee_set_p4d_safe_pre_init(p4d, __p4d(_PAGE_TABLE | __pa(pud))); -+} -+ -+static inline void iee_pgd_populate_pre_init(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) -+{ -+ if (!pgtable_l5_enabled()) -+ return; -+ paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); -+ iee_set_pgd_pre_init(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); -+} -+ -+static inline void iee_pgd_populate_safe_pre_init(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) -+{ -+ if (!pgtable_l5_enabled()) -+ return; -+ paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); -+ iee_set_pgd_safe_pre_init(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); -+} -+ -+#endif -+ - static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) - { - gfp_t gfp = GFP_KERNEL_ACCOUNT; - - if (mm == &init_mm) - gfp &= ~__GFP_ACCOUNT; -+ #ifdef CONFIG_PTP -+ if (iee_pgt_jar_init) -+ return (p4d_t *)get_iee_pgtable_page(gfp); -+ else -+ return (p4d_t *)get_zeroed_page(gfp); -+ #else - return (p4d_t *)get_zeroed_page(gfp); -+ #endif - } - - static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) -@@ -162,7 +231,11 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) - return; - - BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page(p4d); -+ #else - free_page((unsigned long)p4d); -+ #endif - } - - extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); -diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h -index 993d49cd379a..82fac64eccd5 100644 ---- a/arch/x86/include/asm/pgtable.h -+++ b/arch/x86/include/asm/pgtable.h -@@ -94,6 +94,14 @@ extern pmdval_t early_pmd_flags; - #define pud_clear(pud) native_pud_clear(pud) - #endif - -+#ifdef CONFIG_PTP -+#define iee_set_pte_pre_init(ptep, pte) iee_early_set_pte(ptep, pte) -+#define iee_set_pmd_pre_init(pmdp, pmd) iee_early_set_pmd(pmdp, pmd) -+#define iee_set_pgd_pre_init(pgdp, pgd) iee_early_set_pgd(pgdp, pgd) -+#define iee_set_p4d_pre_init(p4dp, p4d) iee_early_set_p4d(p4dp, p4d) -+#define iee_set_pud_pre_init(pudp, pud) iee_early_set_pud(pudp, pud) -+#endif -+ - #define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) - #define pmd_clear(pmd) native_pmd_clear(pmd) - -@@ -251,6 +259,14 @@ static inline unsigned long pgd_pfn(pgd_t pgd) - return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT; - } - -+#ifdef CONFIG_IEE -+#define __pte_to_phys(pte) (pte_pfn(pte) << PAGE_SHIFT) -+#define __pmd_to_phys(pmd) (__pte_to_phys(__pte(pmd_val(pmd)))) -+#define __pud_to_phys(pud) (__pte_to_phys(__pte(pud_val(pud)))) -+#define __p4d_to_phys(p4d) (__pte_to_phys(__pte(p4d_val(p4d)))) -+#define __pgd_to_phys(pgd) (__pte_to_phys(__pte(pgd_val(pgd)))) -+#endif -+ - #define p4d_leaf p4d_large - static inline int p4d_large(p4d_t p4d) - { -@@ -928,6 +944,13 @@ static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) - #include - #include - -+#ifdef CONFIG_KOI -+static inline int pte_valid(pte_t pte) -+{ -+ return pte.pte & _PAGE_PRESENT; -+} -+#endif -+ - static inline int pte_none(pte_t pte) - { - return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK)); -@@ -1189,6 +1212,20 @@ static inline int pgd_none(pgd_t pgd) - - extern int direct_gbpages; - void init_mem_mapping(void); -+#ifdef CONFIG_IEE -+void init_iee_mapping(void); -+unsigned long init_memory_mapping_for_iee(unsigned long start, -+ unsigned long end, pgprot_t prot); -+#else -+#ifdef CONFIG_KOI -+void init_koi_mapping(void); -+unsigned long init_memory_mapping_for_koi(unsigned long start, -+ unsigned long end, pgprot_t prot); -+#endif -+#endif /* CONFIG_IEE*/ -+#ifdef CONFIG_PTP -+void init_iee(void); -+#endif - void early_alloc_pgt_buf(void); - extern void memblock_find_dma_reserve(void); - void __init poking_init(void); -@@ -1289,6 +1326,11 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, - return pte; - } - -+#ifdef CONFIG_PTP -+extern pgprotval_t iee_set_try_cmpxchg(pgprotval_t *pgprotp, pgprotval_t old_pgprotval, pgprotval_t new_pgprotval); -+extern pgprotval_t iee_set_xchg(pgprotval_t *pgprotp, pgprotval_t pgprotval); -+#endif -+ - #define __HAVE_ARCH_PTEP_SET_WRPROTECT - static inline void ptep_set_wrprotect(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -@@ -1303,7 +1345,12 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, - old_pte = READ_ONCE(*ptep); - do { - new_pte = pte_wrprotect(old_pte); -- } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); -+ } -+ #ifdef CONFIG_PTP -+ while (!iee_set_try_cmpxchg(__phys_to_iee(__pa(ptep)), pte_val(old_pte), pte_val(new_pte))); -+ #else -+ while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); -+ #endif - } - - #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) -@@ -1365,7 +1412,12 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, - old_pmd = READ_ONCE(*pmdp); - do { - new_pmd = pmd_wrprotect(old_pmd); -- } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd)); -+ } -+ #ifdef CONFIG_PTP -+ while (!iee_set_try_cmpxchg(__phys_to_iee(__pa(pmdp)), pmd_val(old_pmd), pmd_val(new_pmd))); -+ #else -+ while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd)); -+ #endif - } - - #ifndef pmdp_establish -@@ -1375,10 +1427,19 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, - { - page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); - if (IS_ENABLED(CONFIG_SMP)) { -+ #ifdef CONFIG_PTP -+ pmdval_t pmdval = iee_set_xchg(__phys_to_iee(__pa(pmdp)), pmd_val(pmd)); -+ return native_make_pmd(pmdval); -+ #else - return xchg(pmdp, pmd); -+ #endif - } else { - pmd_t old = *pmdp; -+ #ifdef CONFIG_PTP -+ set_pmd(pmdp, pmd); -+ #else - WRITE_ONCE(*pmdp, pmd); -+ #endif - return old; - } - } -@@ -1466,6 +1527,17 @@ static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) - */ - static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) - { -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_MEMCPY, dst, src, count * sizeof(pgd_t)); -+#ifdef CONFIG_PAGE_TABLE_ISOLATION -+ if (!static_cpu_has(X86_FEATURE_PTI)) -+ return; -+ /* Clone the user space pgd as well */ -+ iee_rw_gate(IEE_OP_MEMCPY, kernel_to_user_pgdp(dst), -+ kernel_to_user_pgdp(src), count * sizeof(pgd_t)); -+#endif -+ #else -+ - memcpy(dst, src, count * sizeof(pgd_t)); - #ifdef CONFIG_PAGE_TABLE_ISOLATION - if (!static_cpu_has(X86_FEATURE_PTI)) -@@ -1474,6 +1546,7 @@ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) - memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), - count * sizeof(pgd_t)); - #endif -+ #endif - } - - #define PTE_SHIFT ilog2(PTRS_PER_PTE) -diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h -index a629b1b9f65a..10134a82fa2c 100644 ---- a/arch/x86/include/asm/pgtable_64.h -+++ b/arch/x86/include/asm/pgtable_64.h -@@ -16,6 +16,22 @@ - #include - #include - -+#ifdef CONFIG_KOI -+ -+#ifndef __PAGETABLE_PUD_FOLDED -+#define pud_val(x) native_pud_val(x) -+#define __pud(x) native_make_pud(x) -+#endif -+ -+#ifndef __PAGETABLE_PMD_FOLDED -+#define pmd_val(x) native_pmd_val(x) -+#define __pmd(x) native_make_pmd(x) -+#endif -+ -+#define pte_val(x) native_pte_val(x) -+#define __pte(x) native_make_pte(x) -+#endif -+ - extern p4d_t level4_kernel_pgt[512]; - extern p4d_t level4_ident_pgt[512]; - extern pud_t level3_kernel_pgt[512]; -@@ -31,6 +47,12 @@ extern pgd_t init_top_pgt[]; - extern void paging_init(void); - static inline void sync_initial_page_table(void) { } - -+#ifdef CONFIG_PTP -+#include -+extern unsigned long long iee_rw_gate(int flag, ...); -+extern pgprotval_t iee_set_xchg(pgprotval_t *pgprotp, pgprotval_t pgprotval); -+#endif -+ - #define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pte_val(e)) -@@ -62,9 +84,23 @@ static inline bool mm_p4d_folded(struct mm_struct *mm) - void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte); - void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); - -+ -+#ifdef CONFIG_KOI -+static int pmd_present(pmd_t pmd); -+static int pud_present(pud_t pud); -+#endif - static inline void native_set_pte(pte_t *ptep, pte_t pte) - { -+#ifdef CONFIG_KOI -+ if (pte_flags(pte) & _PAGE_PRESENT) { -+ pte = __pte(pte_val(pte) & ~_PAGE_GLOBAL); -+ } -+#endif -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); -+ #else - WRITE_ONCE(*ptep, pte); -+ #endif - } - - static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, -@@ -80,7 +116,16 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) - - static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) - { -+#ifdef CONFIG_KOI -+ if (pmd_present(pmd) && pmd_leaf(pmd)) { -+ pmd = __pmd(pmd_val(pmd) & ~_PAGE_GLOBAL); -+ } -+#endif -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); -+ #else - WRITE_ONCE(*pmdp, pmd); -+ #endif - } - - static inline void native_pmd_clear(pmd_t *pmd) -@@ -91,7 +136,12 @@ static inline void native_pmd_clear(pmd_t *pmd) - static inline pte_t native_ptep_get_and_clear(pte_t *xp) - { - #ifdef CONFIG_SMP -+ #ifdef CONFIG_PTP -+ pteval_t pteval = iee_set_xchg(__phys_to_iee(__pa(xp)), 0); -+ return native_make_pte(pteval); -+ #else - return native_make_pte(xchg(&xp->pte, 0)); -+ #endif - #else - /* native_local_ptep_get_and_clear, - but duplicated because of cyclic dependency */ -@@ -104,7 +154,12 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) - static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) - { - #ifdef CONFIG_SMP -+ #ifdef CONFIG_PTP -+ pmdval_t pmdval = iee_set_xchg(__phys_to_iee(__pa(xp)), 0); -+ return native_make_pmd(pmdval); -+ #else - return native_make_pmd(xchg(&xp->pmd, 0)); -+ #endif - #else - /* native_local_pmdp_get_and_clear, - but duplicated because of cyclic dependency */ -@@ -116,7 +171,17 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) - - static inline void native_set_pud(pud_t *pudp, pud_t pud) - { -+#ifdef CONFIG_KOI -+ if ((pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == -+ (_PAGE_PSE | _PAGE_PRESENT)) { -+ pud = __pud(pud_val(pud) & ~_PAGE_GLOBAL); -+ } -+#endif -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); -+ #else - WRITE_ONCE(*pudp, pud); -+ #endif - } - - static inline void native_pud_clear(pud_t *pud) -@@ -127,7 +192,12 @@ static inline void native_pud_clear(pud_t *pud) - static inline pud_t native_pudp_get_and_clear(pud_t *xp) - { - #ifdef CONFIG_SMP -+ #ifdef CONFIG_PTP -+ pudval_t pudval = iee_set_xchg(__phys_to_iee(__pa(xp)), 0); -+ return native_make_pud(pudval); -+ #else - return native_make_pud(xchg(&xp->pud, 0)); -+ #endif - #else - /* native_local_pudp_get_and_clear, - * but duplicated because of cyclic dependency -@@ -144,13 +214,21 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) - pgd_t pgd; - - if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); -+ #else - WRITE_ONCE(*p4dp, p4d); -+ #endif - return; - } - - pgd = native_make_pgd(native_p4d_val(p4d)); - pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_P4D, p4dp, native_make_p4d(native_pgd_val(pgd))); -+ #else - WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); -+ #endif - } - - static inline void native_p4d_clear(p4d_t *p4d) -@@ -160,7 +238,12 @@ static inline void native_p4d_clear(p4d_t *p4d) - - static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) - { -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PGD, pgdp, -+ pti_set_user_pgtbl(pgdp, pgd)); -+ #else - WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); -+ #endif - } - - static inline void native_pgd_clear(pgd_t *pgd) -@@ -168,6 +251,42 @@ static inline void native_pgd_clear(pgd_t *pgd) - native_set_pgd(pgd, native_make_pgd(0)); - } - -+#ifdef CONFIG_PTP -+static inline void iee_early_set_pte(pte_t *ptep, pte_t pte) -+{ -+ WRITE_ONCE(*ptep, pte); -+} -+ -+static inline void iee_early_set_pmd(pmd_t *pmdp, pmd_t pmd) -+{ -+ WRITE_ONCE(*pmdp, pmd); -+} -+ -+static inline void iee_early_set_pud(pud_t *pudp, pud_t pud) -+{ -+ WRITE_ONCE(*pudp, pud); -+} -+ -+static inline void iee_early_set_p4d(p4d_t *p4dp, p4d_t p4d) -+{ -+ pgd_t pgd; -+ -+ if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { -+ WRITE_ONCE(*p4dp, p4d); -+ return; -+ } -+ -+ pgd = native_make_pgd(native_p4d_val(p4d)); -+ pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); -+ WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); -+} -+ -+static inline void iee_early_set_pgd(pgd_t *pgdp, pgd_t pgd) -+{ -+ WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); -+} -+#endif -+ - /* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. -diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h -index 35c416f06155..bd0d82bb9b18 100644 ---- a/arch/x86/include/asm/pgtable_64_types.h -+++ b/arch/x86/include/asm/pgtable_64_types.h -@@ -140,6 +140,11 @@ extern unsigned int ptrs_per_p4d; - # define VMEMMAP_START __VMEMMAP_BASE_L4 - #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */ - -+#ifdef CONFIG_PTP -+#define __VPTDMAP_BASE_L4 0xffffeb0000000000UL -+#define __VPTDMAP_BASE_L5 0xffd6000000000000UL -+#endif -+ - #ifdef CONFIG_RANDOMIZE_MEMORY - # define PHYSMEM_END physmem_end - #endif -diff --git a/arch/x86/include/asm/pgtable_slab.h b/arch/x86/include/asm/pgtable_slab.h -new file mode 100644 -index 000000000000..03452a3d0569 ---- /dev/null -+++ b/arch/x86/include/asm/pgtable_slab.h -@@ -0,0 +1,13 @@ -+#ifndef _LINUX_PGTABLE_SLAB_H -+#define _LINUX_PGTABLE_SLAB_H -+ -+extern void __init iee_pgtable_init(void); -+extern void iee_ptdesc_init(struct page *page); -+extern void iee_ptdesc_free(struct page *page); -+ -+extern void *get_iee_pgtable_page(gfp_t gfpflags); -+extern void free_iee_pgtable_page(void *obj); -+extern void *get_iee_pgd_page(gfp_t gfpflags); -+extern void free_iee_pgd_page(void *obj); -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h -index 48f8dd47cf68..47f697ce730c 100644 ---- a/arch/x86/include/asm/special_insns.h -+++ b/arch/x86/include/asm/special_insns.h -@@ -10,6 +10,10 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#endif -+ - /* - * The compiler should not reorder volatile asm statements with respect to each - * other: they should execute in program order. However GCC 4.9.x and 5.x have -@@ -51,9 +55,20 @@ static inline unsigned long __native_read_cr3(void) - - static inline void native_write_cr3(unsigned long val) - { -+ #ifdef CONFIG_IEE -+ iee_rwx_gate(IEE_WRITE_CR3, val); -+ #else - asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); -+ #endif - } - -+#ifdef CONFIG_IEE -+static inline void native_write_cr3_pre_init(unsigned long val) -+{ -+ asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); -+} -+#endif -+ - static inline unsigned long native_read_cr4(void) - { - unsigned long val; -diff --git a/arch/x86/include/asm/stack_slab.h b/arch/x86/include/asm/stack_slab.h -new file mode 100644 -index 000000000000..514c09e1c415 ---- /dev/null -+++ b/arch/x86/include/asm/stack_slab.h -@@ -0,0 +1,8 @@ -+#ifndef _LINUX_STACK_SLAB_H -+#define _LINUX_STACK_SLAB_H -+ -+extern void __init iee_stack_init(void); -+extern void *get_iee_stack(void); -+extern void free_iee_stack(void *obj); -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h -index 580636cdc257..f18a855dfa13 100644 ---- a/arch/x86/include/asm/tlb.h -+++ b/arch/x86/include/asm/tlb.h -@@ -34,4 +34,10 @@ static inline void __tlb_remove_table(void *table) - free_page_and_swap_cache(table); - } - -+#ifdef CONFIG_PTP -+static inline void __iee_tlb_remove_table(void *_table) { -+ struct page *page = (struct page *)_table; -+ free_iee_pgtable_page((void *)page_to_virt(page)); -+} -+#endif - #endif /* _ASM_X86_TLB_H */ -diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile -index 2b86fa2d8c64..d8d4e64af538 100644 ---- a/arch/x86/kernel/Makefile -+++ b/arch/x86/kernel/Makefile -@@ -151,6 +151,10 @@ obj-$(CONFIG_X86_CET) += cet.o - - obj-$(CONFIG_X86_USER_SHADOW_STACK) += shstk.o - -+obj-$(CONFIG_IEE) += iee/ -+ -+obj-$(CONFIG_HIVE) += sfi_bpf_arch.o -+ - ### - # 64 bit specific files - ifeq ($(CONFIG_X86_64),y) -@@ -161,6 +165,7 @@ ifeq ($(CONFIG_X86_64),y) - obj-$(CONFIG_MMCONF_FAM10H) += mmconf-fam10h_64.o - obj-y += vsmp_64.o - obj-$(CONFIG_INTEL_IOMMU) += zhaoxin_kh40000.o -+ obj-$(CONFIG_KOI) += koi/ - endif - - obj-$(CONFIG_HYGON_CSV) += csv.o -diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c -index dc3576303f1a..d528f34a0541 100644 ---- a/arch/x86/kernel/asm-offsets.c -+++ b/arch/x86/kernel/asm-offsets.c -@@ -119,6 +119,12 @@ static void __used common(void) - #ifdef CONFIG_CALL_DEPTH_TRACKING - OFFSET(X86_call_depth, pcpu_hot, call_depth); - #endif -+#ifdef CONFIG_IEE -+ DEFINE(iee_from_token_offset, offsetof(struct task_token, iee_stack)); -+ DEFINE(tmp_page_from_token_offset, offsetof(struct task_token, tmp_page)); -+ DEFINE(kernel_from_token_offset,offsetof(struct task_token, kernel_stack)); -+ DEFINE(pgd_from_token_offset, offsetof(struct task_token, pgd)); -+#endif - #if IS_ENABLED(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64) - /* Offset for fields in aria_ctx */ - BLANK(); -diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c -index 26d7a26ef2d2..9519b6ff2530 100644 ---- a/arch/x86/kernel/cpu/common.c -+++ b/arch/x86/kernel/cpu/common.c -@@ -68,6 +68,10 @@ - - #include "cpu.h" - -+#ifdef CONFIG_IEE -+#include -+#endif -+ - u32 elf_hwcap2 __read_mostly; - - /* Number of siblings per CPU package */ -@@ -406,11 +410,18 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) - static const unsigned long cr4_pinned_mask = - X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET; -+#ifdef CONFIG_IEE -+DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); -+#else - static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); -+#endif - static unsigned long cr4_pinned_bits __ro_after_init; - - void native_write_cr0(unsigned long val) - { -+#ifdef CONFIG_IEE -+ iee_rwx_gate(IEE_WRITE_CR0, val); -+#else - unsigned long bits_missing = 0; - - set_register: -@@ -425,6 +436,7 @@ void native_write_cr0(unsigned long val) - /* Warn after we've set the missing bits. */ - WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); - } -+#endif - } - EXPORT_SYMBOL(native_write_cr0); - -diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c -index 16f9814c9be0..729b426ab161 100644 ---- a/arch/x86/kernel/espfix_64.c -+++ b/arch/x86/kernel/espfix_64.c -@@ -34,6 +34,9 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif - /* - * Note: we only need 6*8 = 48 bytes for the espfix stack, but round - * it up to a cache line to avoid unnecessary sharing. -@@ -106,6 +109,11 @@ void __init init_espfix_bsp(void) - pgd_t *pgd; - p4d_t *p4d; - -+ #ifdef CONFIG_PTP -+ iee_set_logical_mem_ro((unsigned long)espfix_pud_page); -+ set_iee_page((unsigned long)__va(__pa_symbol(espfix_pud_page)), 0); -+ #endif -+ - /* Install the espfix pud into the kernel page directory */ - pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)]; - p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR); -@@ -158,6 +166,9 @@ void init_espfix_ap(int cpu) - - pmd_p = (pmd_t *)page_address(page); - pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask)); -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pmd_p, 0); -+ #endif - paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT); - for (n = 0; n < ESPFIX_PUD_CLONES; n++) - set_pud(&pud_p[n], pud); -@@ -170,6 +181,9 @@ void init_espfix_ap(int cpu) - - pte_p = (pte_t *)page_address(page); - pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask)); -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pte_p, 0); -+ #endif - paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT); - for (n = 0; n < ESPFIX_PMD_CLONES; n++) - set_pmd(&pmd_p[n], pmd); -diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c -index 1defe865de67..148ea7acf075 100644 ---- a/arch/x86/kernel/head64.c -+++ b/arch/x86/kernel/head64.c -@@ -44,6 +44,10 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#endif -+ - /* - * Manage page tables very early on. - */ -@@ -68,6 +72,11 @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; - EXPORT_SYMBOL(vmemmap_base); - #endif - -+#ifdef CONFIG_PTP -+unsigned long iee_ptdesc_base __ro_after_init = __VPTDMAP_BASE_L4; -+EXPORT_SYMBOL(iee_ptdesc_base); -+#endif -+ - /* - * GDT used on the boot CPU before switching to virtual addresses. - */ -@@ -118,6 +127,10 @@ static bool __head check_la57_support(unsigned long physaddr) - *fixup_long(&vmalloc_base, physaddr) = __VMALLOC_BASE_L5; - *fixup_long(&vmemmap_base, physaddr) = __VMEMMAP_BASE_L5; - -+ #ifdef CONFIG_PTP -+ *fixup_long(&iee_ptdesc_base, physaddr) = __VPTDMAP_BASE_L5; -+ #endif -+ - return true; - } - #else -@@ -720,7 +733,11 @@ static void startup_64_load_idt(unsigned long physbase) - } - - desc->address = (unsigned long)idt; -+ #ifdef CONFIG_IEE -+ iee_load_idt_pre_init(desc); -+ #else - native_load_idt(desc); -+ #endif - } - - /* This is used when running on kernel addresses */ -diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c -index fc77a96040b7..378d35807da9 100644 ---- a/arch/x86/kernel/idt.c -+++ b/arch/x86/kernel/idt.c -@@ -163,13 +163,17 @@ static const __initconst struct idt_data apic_idts[] = { - }; - - /* Must be page-aligned because the real IDT is used in the cpu entry area */ --static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; -+gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; - --static struct desc_ptr idt_descr __ro_after_init = { -+struct desc_ptr idt_descr __ro_after_init = { - .size = IDT_TABLE_SIZE - 1, - .address = (unsigned long) idt_table, - }; - -+#ifdef CONFIG_KOI -+const unsigned long koi_idt_descr_addr = (unsigned long)&idt_descr; -+#endif -+ - void load_current_idt(void) - { - lockdep_assert_irqs_disabled(); -diff --git a/arch/x86/kernel/iee/Makefile b/arch/x86/kernel/iee/Makefile -new file mode 100644 -index 000000000000..b4f188f159d8 ---- /dev/null -+++ b/arch/x86/kernel/iee/Makefile -@@ -0,0 +1,4 @@ -+obj-$(CONFIG_IEE) += iee.o iee-gate.o iee-func.o stack-slab.o -+ccflags-$(CONFIG_IEE_SELINUX_P) := -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include -+obj-$(CONFIG_IEE_SELINUX_P) += iee-selinuxp.o -+obj-$(CONFIG_PTP) += pgtable-slab.o -diff --git a/arch/x86/kernel/iee/iee-func.c b/arch/x86/kernel/iee/iee-func.c -new file mode 100644 -index 000000000000..d742691e7e32 ---- /dev/null -+++ b/arch/x86/kernel/iee/iee-func.c -@@ -0,0 +1,365 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+static inline void iee_set_sensitive_pte(pte_t *lm_ptep, int order, int use_block_pmd) -+{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_SET_SENSITIVE_PTE, lm_ptep, order, use_block_pmd); -+#else -+ int i; -+ if (use_block_pmd) { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) & (~__RW) & (~___D))); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) & (~__RW) & (~___D))); -+ WRITE_ONCE(*lm_ptep, pte); -+ lm_ptep++; -+ } -+ } -+#endif -+} -+ -+static inline void iee_unset_sensitive_pte(pte_t *lm_ptep, int order, int use_block_pmd) -+{ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_UNSET_SENSITIVE_PTE, lm_ptep, order, use_block_pmd); -+#else -+ int i; -+ if (use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) | __RW | ___D)); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | __RW | ___D); -+ WRITE_ONCE(*lm_ptep, pte); -+ lm_ptep++; -+ } -+ } -+#endif -+} -+ -+static void do_split_huge_pmd(pmd_t* pmdp) -+{ -+ // pte_t *pgtable = pte_alloc_one_kernel(&init_mm); -+ gfp_t gfp = (GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM) | __GFP_COMP; -+ struct page *new_page = alloc_pages(gfp, 0); -+ if (!new_page) -+ panic("IEE: failed to alloc pgtable\n"); -+ pte_t *pgtable = (pte_t *)page_to_virt(new_page); -+ #ifdef CONFIG_PTP -+ if (iee_pgt_jar_init) -+ iee_ptdesc_init(new_page); -+ iee_split_huge_pmd(pmdp, pgtable); -+ #else -+ int i; -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)((unsigned long)pgtable); -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = pmd_pgprot(*pmdp); -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+ #endif -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if (pmd_leaf(READ_ONCE(*pmdp))) { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); -+ if(pgtable) -+ { -+ memset(pgtable, 0, PAGE_SIZE); -+ #ifdef CONFIG_PTP -+ if (iee_pgt_jar_init) -+ iee_ptdesc_free(new_page); -+ #endif -+ __free_pages(new_page, compound_order(new_page)); -+ // pte_free_kernel(&init_mm, pgtable); -+ } -+} -+ -+// Input is the lm vaddr of sensitive data. -+void set_iee_page(unsigned long addr, int order) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ int use_block_pmd = 0; -+ -+ if (pmd_leaf(*pmdp) && order < 9) { -+ do_split_huge_pmd(pmdp); -+ } else if (pmd_leaf(*pmdp)) { -+ use_block_pmd = 1; -+ } -+ pte_t *lm_ptep; -+ if (use_block_pmd) { -+ lm_ptep = (pte_t *)pmdp; -+ } else { -+ lm_ptep = pte_offset_kernel(pmdp, addr); -+ } -+ -+ iee_set_sensitive_pte(lm_ptep, order, use_block_pmd); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); -+} -+ -+// Input is the lm vaddr of sensitive data. -+void unset_iee_page(unsigned long addr, int order) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ pte_t *lm_ptep; -+ int use_block_pmd = 0; -+ // Use Block Descriptor. -+ if(pmd_leaf(*pmdp)) -+ { -+ use_block_pmd = 1; -+ lm_ptep = (pte_t *)pmdp; -+ } -+ else -+ lm_ptep = pte_offset_kernel(pmdp, addr); -+ -+ iee_unset_sensitive_pte(lm_ptep, order, use_block_pmd); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); -+} -+ -+void iee_set_logical_mem_ro(unsigned long addr) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ pud_t *pudp = pud_offset(p4dp, addr); -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ if (pmd_leaf(*pmdp)) { -+ do_split_huge_pmd(pmdp); -+ } -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte((pte_val(pte) & (~__RW) & (~___D))); -+ set_pte(ptep, pte); -+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); -+} -+ -+void set_iee_page_valid(unsigned long addr) {} -+ -+void set_iee_page_invalid(unsigned long addr) {} -+ -+void iee_set_token_page_valid(void *token, void *token_page, unsigned int order) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *token_pmdp = pmd_offset(pudp, (unsigned long)token); -+ pte_t *token_ptep = pte_offset_kernel(token_pmdp, (unsigned long)token); -+ -+ if(token_page == NULL) -+ panic("Token of task_struct was unset.\n"); -+ -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_page); -+ p4dp = p4d_offset(pgdp, (unsigned long)token_page); -+ pudp = pud_offset(p4dp, (unsigned long)token_page); -+ pmd_t *token_page_pmdp = pmd_offset(pudp, (unsigned long)token_page); -+ pte_t *token_page_ptep; -+ -+ int use_block_pmd = 0; -+ if (pmd_leaf(*token_page_pmdp) && order < 9) { -+ do_split_huge_pmd(token_page_pmdp); -+ } else if (pmd_leaf(*token_page_pmdp)) { -+ use_block_pmd = 1; -+ } -+ -+ if (use_block_pmd) { -+ token_page_ptep = (pte_t *)token_page_pmdp; -+ } else { -+ token_page_ptep = pte_offset_kernel(token_page_pmdp, (unsigned long)token_page); -+ } -+ -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_TOKEN, token_ptep, token_page_ptep, token_page, order, use_block_pmd); -+#else -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) & ~__RW) & ~___D); -+ WRITE_ONCE(*pmdp, pmd); -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__pa(token_page + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ token_ptep++; -+ } -+ } -+ else { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__pa(token_page + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte((pte_val(pte) & ~__RW) & ~___D); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+#endif -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)token_page, (unsigned long)(token_page + (PAGE_SIZE * (1 << order)))); -+} -+ -+void iee_set_token_page_invalid(void *token, void *__unused, unsigned long order) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *token_pmdp = pmd_offset(pudp, (unsigned long)token); -+ pte_t *token_ptep = pte_offset_kernel(token_pmdp, (unsigned long)token); -+ void *token_page = page_address(pte_page(*token_ptep)); -+ -+ if(token_page == NULL) -+ panic("Token of task_struct was unset.\n"); -+ -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_page); -+ p4dp = p4d_offset(pgdp, (unsigned long)token_page); -+ pudp = pud_offset(p4dp, (unsigned long)token_page); -+ pmd_t *token_page_pmdp = pmd_offset(pudp, (unsigned long)token_page); -+ pte_t *token_page_ptep; -+ int use_block_pmd = 0; -+ if (pmd_leaf(*token_page_pmdp)) { -+ use_block_pmd = 1; -+ token_page_ptep = (pte_t *)token_page_pmdp; -+ } else { -+ token_page_ptep = pte_offset_kernel(token_page_pmdp, (unsigned long)token_page); -+ } -+#ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_UNSET_TOKEN, token_ptep, token_page_ptep, token, token_page, order); -+#else -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | ___D | __RW); -+ WRITE_ONCE(*pmdp, pmd); -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__iee_pa(token + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ token_ptep++; -+ } -+ } -+ else -+ { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__iee_pa(token + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte(pte_val(pte) | ___D | __RW); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+#endif -+ free_pages((unsigned long)token_page, order); -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)token_page, (unsigned long)(token_page + (PAGE_SIZE * (1 << order)))); -+} -+ -+void __init iee_set_kernel_upage(unsigned long addr) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ pgd_t pgd = READ_ONCE(*pgdp); -+ pgd = __pgd((pgd_val(pgd) | _USR) & ~___G); -+ set_pgd(pgdp, pgd); -+ -+ p4d_t *p4dp = p4d_offset(pgdp, addr); -+ p4d_t p4d = READ_ONCE(*p4dp); -+ p4d = __p4d((p4d_val(p4d) | _USR) & ~___G); -+ set_p4d(p4dp, p4d); -+ -+ pud_t *pudp = pud_offset(p4dp, addr); -+ if (pud_leaf(*pudp)) { -+ panic("Huge pud page set upage!\n"); -+ } -+ pud_t pud = READ_ONCE(*pudp); -+ pud = __pud((pud_val(pud) | _USR) & ~___G); -+ set_pud(pudp, pud); -+ -+ pmd_t *pmdp = pmd_offset(pudp, addr); -+ if (pmd_leaf(*pmdp)) { -+ do_split_huge_pmd(pmdp); -+ } -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) | _USR) & ~___G); -+ set_pmd(pmdp, pmd); -+ -+ pte_t *ptep = pte_offset_kernel(pmdp, addr); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte((pte_val(pte) | _USR) & ~___G); -+ set_pte(ptep, pte); -+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -+} -+ -+void set_iee_stack_page(unsigned long addr, int order) -+{ -+ set_iee_page(addr ,order); -+} -+ -+void unset_iee_stack_page(unsigned long addr, int order) -+{ -+ unset_iee_page(addr, order); -+} -+ -+void __init iee_rest_init(void) -+{ -+ // Prepare data for iee rwx gate -+ unsigned long addr; -+ /* Map .iee.text as U RWX pages */ -+ addr = (unsigned long)__iee_si_text_start; -+ for (; addr < (unsigned long)__iee_si_text_end; addr += PAGE_SIZE){ -+ iee_set_kernel_upage((unsigned long)addr); -+ } -+ iee_init_done = true; -+ iee_base_swapper_pg_dir = __sme_pa(swapper_pg_dir); -+ /* Map .iee.data as RO pages */ -+ set_memory_ro((unsigned long)__iee_si_data_start, ((unsigned long)__iee_si_data_end - (unsigned long)__iee_si_data_start) / PAGE_SIZE); -+ // All initialization is done. Do some simple tests. -+ pr_err("IEE: testing iee_exec_entry si_test..."); -+ iee_rwx_gate(IEE_SI_TEST); -+ pr_err("IEE: testing iee_exec_entry si_test..."); -+} -\ No newline at end of file -diff --git a/arch/x86/kernel/iee/iee-gate.S b/arch/x86/kernel/iee/iee-gate.S -new file mode 100644 -index 000000000000..c80e3f6b45b3 ---- /dev/null -+++ b/arch/x86/kernel/iee/iee-gate.S -@@ -0,0 +1,209 @@ -+#include -+#include -+#include -+#include -+#include -+ -+#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) -+ -+#ifdef CONFIG_PTP -+SYM_FUNC_START(iee_set_xchg) -+ /* save RFLAGS, close irq */ -+ pushfq -+ cli -+ /* stac, disable SMAP */ -+ stac -+ -+ xchg %rsi, (%rdi) -+ mov %rsi, %rax -+ -+ /* clac, enable SMAP */ -+ clac -+ /* restore RFLAGS*/ -+ popfq -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_set_xchg) -+ -+SYM_FUNC_START(iee_set_try_cmpxchg) -+ /* save RFLAGS, close irq */ -+ pushfq -+ cli -+ /* stac, disable SMAP */ -+ stac -+ -+ mov %rsi, %rax -+ lock cmpxchgq %rdx, (%rdi) -+ -+ /* clac, enable SMAP */ -+ clac -+ /* restore RFLAGS*/ -+ popfq -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_set_try_cmpxchg) -+#endif /* CONFIG_PTP */ -+ -+SYM_FUNC_START(iee_rw_gate) -+ /* save Interrupt flag */ -+ pushf -+ /* close irq*/ -+ cli -+ -+ /* stac, disable SMAP */ -+ stac -+ -+ pushq %r12 -+ -+ /* switch to iee stack */ -+ movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %r12 /* r12 -> task_struct */ -+ addq iee_offset(%rip), %r12 -+ movq %rsp, kernel_from_token_offset(%r12) -+ movq iee_from_token_offset(%r12), %rsp -+ -+ /* call iee func */ -+ leaq iee_funcs(%rip), %rax -+ call *(%rax, %rdi, 8) -+ -+ /* switch to kernel stack */ -+ movq kernel_from_token_offset(%r12), %rsp -+ -+ popq %r12 -+ -+ /* clac, enable SMAP */ -+ clac -+ -+ /* restore irq*/ -+ popf -+ -+ lfence /* Serializing instruction before the ret */ -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_rw_gate) -+#if defined(CONFIG_CREDP) || defined(CONFIG_KEYP) || defined(CONFIG_PTP) || defined(CONFIG_KOI) -+EXPORT_SYMBOL(iee_rw_gate) -+#endif -+ -+SYM_FUNC_START(iee_read_token_stack) -+ /* save Interrupt flag */ -+ pushf -+ /* close irq*/ -+ cli -+ -+ push %r12 -+ -+ /* stac, disable SMAP */ -+ stac -+ -+ addq iee_offset(%rip), %rdi -+ movq iee_from_token_offset(%rdi), %rax -+ -+ /* clac, enable SMAP */ -+ clac -+ -+ pop %r12 -+ -+ /* restore irq*/ -+ popf -+ -+ lfence /* Serializing instruction before the ret */ -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_read_token_stack) -+ -+SYM_FUNC_START(iee_read_tmp_page) -+ /* save Interrupt flag */ -+ pushf -+ /* close irq*/ -+ cli -+ -+ push %r12 -+ -+ /* stac, disable SMAP */ -+ stac -+ -+ addq iee_offset(%rip), %rdi -+ movq tmp_page_from_token_offset(%rdi), %rax -+ -+ /* clac, enable SMAP */ -+ clac -+ -+ pop %r12 -+ -+ /* restore irq*/ -+ popf -+ lfence /* Serializing instruction before the ret */ -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_read_tmp_page) -+ -+SYM_FUNC_START(iee_read_freeptr) -+ /* save Interrupt flag */ -+ pushf -+ /* close irq*/ -+ cli -+ -+ push %r12 -+ -+ /* stac, disable SMAP */ -+ stac -+ -+ addq iee_offset(%rip), %rdi -+ movq (%rdi), %rax -+ -+ /* clac, enable SMAP */ -+ clac -+ -+ pop %r12 -+ -+ /* restore irq*/ -+ popf -+ lfence /* Serializing instruction before the ret */ -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_read_freeptr) -+ -+SYM_FUNC_START(iee_rwx_gate) -+ pushq %r12 -+ -+ /* save Interrupt flag*/ -+ pushfq -+ /* close irq */ -+ cli -+ -+ /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ -+ movq %cr4, %rax /* rax -> cr4 */ -+ andq $(~X86_CR4_SMEP_SMAP), %rax -+ movq %rax, %cr4 -+ -+ movq %rsp, %r12 -+ -+ /* If iee hasn't been initialized, skip stack switch. */ -+ cmpb $0, iee_init_done(%rip) -+ jz 2f -+#ifdef CONFIG_KOI -+ cmpq $IEE_SWITCH_TO_KOI, %rdi -+ jz 2f -+ cmpq $IEE_SWITCH_TO_KERNEL, %rdi -+ jz 2f -+#endif -+ /* switch to iee stack */ -+ movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %rax /* rax -> task_struct */ -+ addq iee_offset(%rip), %rax -+ movq %rsp, kernel_from_token_offset(%rax) -+ movq iee_from_token_offset(%rax), %rsp -+ -+2: call iee_si_handler -+ -+ /* switch to kernel stack. If iee hasn't been initialized, skip switch*/ -+ movq %r12, %rsp -+ -+ /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ -+ movq %cr4, %rax /* rax -> cr4 */ -+1: orq $X86_CR4_SMEP_SMAP, %rax -+ movq %rax, %cr4 -+ andq $(X86_CR4_SMEP_SMAP), %rax -+ cmpq $(X86_CR4_SMEP_SMAP), %rax -+ jnz 1 -+ -+ /* restore irq*/ -+ popfq -+ -+ popq %r12 -+ jmp __x86_return_thunk /* ret */ -+SYM_FUNC_END(iee_rwx_gate) -+EXPORT_SYMBOL(iee_rwx_gate) -\ No newline at end of file -diff --git a/arch/x86/kernel/iee/iee-selinuxp.c b/arch/x86/kernel/iee/iee-selinuxp.c -new file mode 100644 -index 000000000000..47d6eeb7a784 ---- /dev/null -+++ b/arch/x86/kernel/iee/iee-selinuxp.c -@@ -0,0 +1,36 @@ -+#include -+#include -+extern unsigned long long iee_rw_gate(int flag, ...); -+ -+#ifdef CONFIG_IEE_SELINUX_P -+inline void iee_set_selinux_status_pg(struct page *new_page) -+{ -+ iee_rw_gate(IEE_SEL_SET_STATUS_PG, new_page); -+} -+ -+inline void enforcing_set(bool value) -+{ -+ iee_rw_gate(IEE_SEL_SET_ENFORCING, value); -+} -+ -+inline void selinux_mark_initialized(void) -+{ -+ iee_rw_gate(IEE_SEL_SET_INITIALIZED); -+} -+ -+inline void iee_set_sel_policy_cap(unsigned int idx, int cap) -+{ -+ iee_rw_gate(IEE_SEL_SET_POLICY_CAP, idx, cap); -+} -+ -+/* -+ * Please make sure param iee_new_policy is from policy_jar memcache. -+ * Need to free new_policy after calling this func as it's only used to -+ * trans data from kernel. -+ */ -+inline void iee_sel_rcu_assign_policy(struct selinux_policy *new_policy, -+ struct selinux_policy *iee_new_policy) -+{ -+ iee_rw_gate(IEE_SEL_RCU_ASSIGN_POLICY, new_policy, iee_new_policy); -+} -+#endif -diff --git a/arch/x86/kernel/iee/iee.c b/arch/x86/kernel/iee/iee.c -new file mode 100644 -index 000000000000..107167a0c5ea ---- /dev/null -+++ b/arch/x86/kernel/iee/iee.c -@@ -0,0 +1,1048 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#ifdef CONFIG_PTP -+void __iee_code _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte); -+void __iee_code _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); -+void __iee_code _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud); -+void __iee_code _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d); -+void __iee_code _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd); -+#endif -+void __iee_code _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); -+void __iee_code _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); -+void __iee_code _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); -+void __iee_code _iee_split_huge_pmd(unsigned long __unused, pmd_t *pmdp, pte_t *pgtable); -+void __iee_code _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd); -+void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk, void *iee_stack, void *tmp_page); -+void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); -+void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct *tsk); -+void __iee_code _iee_unset_token(unsigned long __unused, pte_t *token_ptep, pte_t *token_page_ptep, void *token, void *token_page, unsigned long order, int use_block_pmd); -+void __iee_code _iee_set_token(unsigned long __unused, pte_t *token_ptep, pte_t *new_ptep, void *new, unsigned long order, int use_block_pmd); -+void __iee_code _iee_set_track(unsigned long __unused, struct track *ptr, struct track *data); -+unsigned long __iee_code _iee_test_and_clear_bit(unsigned long __unused, long nr, volatile unsigned long *addr); -+void __iee_code _iee_set_sensitive_pte(unsigned long __unused, pte_t *lm_ptep, int order, int use_block_pmd); -+void __iee_code _iee_unset_sensitive_pte(unsigned long __unused, pte_t *lm_ptep, int order, int use_block_pmd); -+ -+#ifdef CONFIG_CREDP -+extern struct cred init_cred; -+void __iee_code _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); -+void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); -+void __iee_code _iee_copy_cred(unsigned long __unused, struct cred *old, struct cred *new); -+void __iee_code _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); -+void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid); -+void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid); -+void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid); -+void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid); -+void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid); -+void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user); -+void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns); -+void __iee_code _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info); -+void __iee_code _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned securebits); -+void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable); -+void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted); -+void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective); -+void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset); -+void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient); -+void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring); -+void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring); -+void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring); -+void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring); -+void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth); -+void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu); -+void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i); -+unsigned long __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr); -+void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security); -+void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu); -+void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); -+#endif -+ -+#ifdef CONFIG_KEYP -+struct watch_list; -+void __iee_code _iee_set_key_union(unsigned long __unused, struct key *key, struct key_union *key_union); -+void __iee_code _iee_set_key_struct(unsigned long __unused, struct key *key, struct key_struct *key_struct); -+void __iee_code _iee_set_key_payload(unsigned long __unused, struct key *key, union key_payload *key_payload); -+unsigned long __iee_code _iee_set_key_usage(unsigned long __unused, struct key *key, int n, int flag); -+void __iee_code _iee_set_key_serial(unsigned long __unused, struct key *key, key_serial_t serial); -+void __iee_code _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers); -+void __iee_code _iee_set_key_user(unsigned long __unused, struct key *key, struct key_user *user); -+void __iee_code _iee_set_key_security(unsigned long __unused, struct key *key, void *security); -+void __iee_code _iee_set_key_expiry(unsigned long __unused, struct key *key, time64_t expiry); -+void __iee_code _iee_set_key_revoked_at(unsigned long __unused, struct key *key, time64_t revoked_at); -+void __iee_code _iee_set_key_last_used_at(unsigned long __unused, struct key *key, time64_t last_used_at); -+void __iee_code _iee_set_key_uid(unsigned long __unused, struct key *key, kuid_t uid); -+void __iee_code _iee_set_key_gid(unsigned long __unused, struct key *key, kgid_t gid); -+void __iee_code _iee_set_key_perm(unsigned long __unused, struct key *key, key_perm_t perm); -+void __iee_code _iee_set_key_quotalen(unsigned long __unused, struct key *key, unsigned short quotalen); -+void __iee_code _iee_set_key_datalen(unsigned long __unused, struct key *key, unsigned short datalen); -+void __iee_code _iee_set_key_state(unsigned long __unused, struct key *key, short state); -+void __iee_code _iee_set_key_magic(unsigned long __unused, struct key *key, unsigned magic); -+void __iee_code _iee_set_key_flags(unsigned long __unused, struct key *key, unsigned long flags); -+void __iee_code _iee_set_key_index_key(unsigned long __unused, struct key *key, struct keyring_index_key* index_key); -+void __iee_code _iee_set_key_hash(unsigned long __unused, struct key *key, unsigned long hash); -+void __iee_code _iee_set_key_len_desc(unsigned long __unused, struct key *key, unsigned long len_desc); -+void __iee_code _iee_set_key_type(unsigned long __unused, struct key *key, struct key_type *type); -+void __iee_code _iee_set_key_domain_tag(unsigned long __unused, struct key *key, struct key_tag *domain_tag); -+void __iee_code _iee_set_key_description(unsigned long __unused, struct key *key, char *description); -+void __iee_code _iee_set_key_restrict_link(unsigned long __unused, struct key *key, struct key_restriction *restrict_link); -+unsigned long __iee_code _iee_set_key_flag_bit(unsigned long __unused, struct key *key, long nr, int flag); -+#endif -+#ifdef CONFIG_KOI -+unsigned long __iee_code _iee_read_koi_stack(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack); -+unsigned long __iee_code _iee_read_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_token_ttbr1(unsigned long iee_offset, struct task_struct *tsk, unsigned long current_ttbr1); -+unsigned long __iee_code _iee_read_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_kernel_stack(unsigned long iee_offset, struct task_struct *tsk, unsigned long kernel_stack); -+unsigned long __iee_code _iee_read_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk); -+void __iee_code _iee_write_koi_stack_base(unsigned long iee_offset, struct task_struct *tsk, unsigned long koi_stack_base); -+void __iee_code _iee_set_koi_pgd(unsigned long iee_offset, unsigned long koi_pgd_addr); -+#endif -+ -+#ifdef CONFIG_IEE_SELINUX_P -+#include -+void __iee_code _iee_set_selinux_status_pg(unsigned long __unused, struct page* new_page); -+void __iee_code _iee_set_selinux_enforcing(unsigned long __unused, bool value); -+void __iee_code _iee_mark_selinux_initialized(unsigned long __unused); -+void __iee_code _iee_set_sel_policy_cap(unsigned long __unused, unsigned int idx, int cap); -+void __iee_code _iee_sel_rcu_assign_policy(unsigned long __unused, struct selinux_policy* new_policy, struct selinux_policy* iee_new_policy); -+#endif -+ -+typedef void (*iee_func)(void); -+iee_func iee_funcs[] = { -+ #ifdef CONFIG_PTP -+ (iee_func)_iee_set_pte, -+ (iee_func)_iee_set_pmd, -+ (iee_func)_iee_set_pud, -+ (iee_func)_iee_set_p4d, -+ (iee_func)_iee_set_pgd, -+ #endif -+ (iee_func)_iee_memcpy, -+ (iee_func)_iee_memset, -+ (iee_func)_iee_set_freeptr, -+ (iee_func)_iee_split_huge_pmd, -+ (iee_func)_iee_set_token_pgd, -+ (iee_func)_iee_init_token, -+ (iee_func)_iee_invalidate_token, -+ (iee_func)_iee_validate_token, -+ (iee_func)_iee_unset_token, -+ (iee_func)_iee_set_token, -+ (iee_func)_iee_set_track, -+ (iee_func)_iee_test_and_clear_bit, -+ (iee_func)_iee_set_sensitive_pte, -+ (iee_func)_iee_unset_sensitive_pte, -+#ifdef CONFIG_CREDP -+ (iee_func)_iee_copy_cred, -+ (iee_func)_iee_set_cred_uid, -+ (iee_func)_iee_set_cred_gid, -+ (iee_func)_iee_set_cred_suid, -+ (iee_func)_iee_set_cred_sgid, -+ (iee_func)_iee_set_cred_euid, -+ (iee_func)_iee_set_cred_egid, -+ (iee_func)_iee_set_cred_fsuid, -+ (iee_func)_iee_set_cred_fsgid, -+ (iee_func)_iee_set_cred_user, -+ (iee_func)_iee_set_cred_user_ns, -+ (iee_func)_iee_set_cred_group_info, -+ (iee_func)_iee_set_cred_securebits, -+ (iee_func)_iee_set_cred_cap_inheritable, -+ (iee_func)_iee_set_cred_cap_permitted, -+ (iee_func)_iee_set_cred_cap_effective, -+ (iee_func)_iee_set_cred_cap_bset, -+ (iee_func)_iee_set_cred_cap_ambient, -+ (iee_func)_iee_set_cred_jit_keyring, -+ (iee_func)_iee_set_cred_session_keyring, -+ (iee_func)_iee_set_cred_process_keyring, -+ (iee_func)_iee_set_cred_thread_keyring, -+ (iee_func)_iee_set_cred_request_key_auth, -+ (iee_func)_iee_set_cred_non_rcu, -+ (iee_func)_iee_set_cred_atomic_set_usage, -+ (iee_func)_iee_set_cred_atomic_op_usage, -+ (iee_func)_iee_set_cred_security, -+ (iee_func)_iee_set_cred_rcu, -+ (iee_func)_iee_set_cred_ucounts, -+#endif -+#ifdef CONFIG_KEYP -+ (iee_func)_iee_set_key_union, -+ (iee_func)_iee_set_key_struct, -+ (iee_func)_iee_set_key_payload, -+ (iee_func)_iee_set_key_usage, -+ (iee_func)_iee_set_key_serial, -+ (iee_func)_iee_set_key_watchers, -+ (iee_func)_iee_set_key_user, -+ (iee_func)_iee_set_key_security, -+ (iee_func)_iee_set_key_expiry, -+ (iee_func)_iee_set_key_revoked_at, -+ (iee_func)_iee_set_key_last_used_at, -+ (iee_func)_iee_set_key_uid, -+ (iee_func)_iee_set_key_gid, -+ (iee_func)_iee_set_key_perm, -+ (iee_func)_iee_set_key_quotalen, -+ (iee_func)_iee_set_key_datalen, -+ (iee_func)_iee_set_key_state, -+ (iee_func)_iee_set_key_magic, -+ (iee_func)_iee_set_key_flags, -+ (iee_func)_iee_set_key_index_key, -+ (iee_func)_iee_set_key_hash, -+ (iee_func)_iee_set_key_len_desc, -+ (iee_func)_iee_set_key_type, -+ (iee_func)_iee_set_key_domain_tag, -+ (iee_func)_iee_set_key_description, -+ (iee_func)_iee_set_key_restrict_link, -+ (iee_func)_iee_set_key_flag_bit, -+#endif -+#ifdef CONFIG_IEE_SELINUX_P -+ (iee_func)_iee_set_selinux_status_pg, -+ (iee_func)_iee_set_selinux_enforcing, -+ (iee_func)_iee_mark_selinux_initialized, -+ (iee_func)_iee_set_sel_policy_cap, -+ (iee_func)_iee_sel_rcu_assign_policy, -+#endif -+#ifdef CONFIG_KOI -+ (iee_func)_iee_read_koi_stack, -+ (iee_func)_iee_write_koi_stack, -+ (iee_func)_iee_read_token_ttbr1, -+ (iee_func)_iee_write_token_ttbr1, -+ (iee_func)_iee_read_koi_kernel_stack, -+ (iee_func)_iee_write_koi_kernel_stack, -+ (iee_func)_iee_read_koi_stack_base, -+ (iee_func)_iee_write_koi_stack_base, -+ (iee_func)_iee_set_koi_pgd, -+#endif -+ NULL -+}; -+ -+#ifdef CONFIG_KOI -+unsigned long __iee_code _iee_read_koi_stack(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ return (unsigned long)token->koi_stack; -+} -+ -+void __iee_code _iee_write_koi_stack(unsigned long __unused, struct task_struct *tsk, unsigned long koi_stack) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->koi_stack = (void *) koi_stack; -+} -+ -+unsigned long __iee_code _iee_read_token_ttbr1(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ return token->current_ttbr1; -+} -+ -+void __iee_code _iee_write_token_ttbr1(unsigned long __unused, struct task_struct *tsk, unsigned long current_ttbr1) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->current_ttbr1 = current_ttbr1; -+} -+ -+unsigned long __iee_code _iee_read_koi_kernel_stack(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ return (unsigned long) token->koi_kernel_stack; -+} -+ -+void __iee_code _iee_write_koi_kernel_stack(unsigned long __unused, struct task_struct *tsk, unsigned long kernel_stack) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->koi_kernel_stack = (void *) kernel_stack; -+} -+ -+unsigned long __iee_code _iee_read_koi_stack_base(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ return (unsigned long)token->koi_stack_base; -+} -+ -+void __iee_code _iee_write_koi_stack_base(unsigned long __unused, struct task_struct *tsk, unsigned long koi_stack_base) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->koi_stack_base = (void *) koi_stack_base; -+} -+ -+static inline void iee_set_koi_pgd_writeable(unsigned long koi_pgd_addr, unsigned long iee_si_addr) -+{ -+ return; -+} -+/* -+ * Set IEE SI codes U RWX here to avoid IEE DEP checking fault. -+ * Mark koi pgd in the same time. -+ */ -+void __iee_code _iee_set_koi_pgd(unsigned long iee_offset, unsigned long koi_pgd_addr) -+{ -+ return; -+} -+#endif -+ -+#ifdef CONFIG_PTP -+void __iee_code _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte) -+{ -+ WRITE_ONCE(*(pte_t *)(__phys_to_iee(__pa(ptep))), pte); -+} -+ -+void __iee_code _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) -+{ -+ WRITE_ONCE(*(pmd_t *)(__phys_to_iee(__pa(pmdp))), pmd); -+} -+ -+void __iee_code _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud) -+{ -+ WRITE_ONCE(*(pud_t *)(__phys_to_iee(__pa(pudp))), pud); -+} -+ -+void __iee_code _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d) -+{ -+ WRITE_ONCE(*(p4d_t *)(__phys_to_iee(__pa(p4dp))), p4d); -+} -+ -+void __iee_code _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd) -+{ -+ WRITE_ONCE(*(pgd_t *)(__phys_to_iee(__pa(pgdp))), pgd); -+} -+#endif -+ -+void __iee_code _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) -+{ -+ char *_dst, *_src; -+ -+ _dst = (char *)(__phys_to_iee(__pa(dst))); -+ _src = (char *)src; -+ -+ while (n--) -+ *_dst++ = *_src++; -+} -+ -+void __iee_code _iee_set_track(unsigned long __unused, struct track *ptr, struct track *data) -+{ -+ _iee_memcpy(__unused, ptr, data, sizeof(struct track)); -+} -+ -+void __iee_code _iee_memset(unsigned long __unused, void *ptr, int data, size_t n) -+{ -+ char *_ptr; -+ -+ _ptr = (char *)(__phys_to_iee(__pa(ptr))); -+ -+ while (n--) -+ *_ptr++ = data; -+} -+ -+void __iee_code _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr) -+{ -+ pptr = (void **)(__phys_to_iee(__pa(pptr))); -+ *pptr = ptr; -+} -+ -+void __iee_code _iee_split_huge_pmd(unsigned long __unused, pmd_t *pmdp, pte_t *pgtable) -+{ -+ int i; -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)(__phys_to_iee(__pa(pgtable))); -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = pmd_pgprot(*pmdp); -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+} -+ -+void __iee_code _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) -+{ -+ struct task_token *token; -+ -+ token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->pgd = pgd; -+} -+ -+void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk, void *iee_stack, void *tmp_page) -+{ -+ struct task_token *token; -+ -+ token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->iee_stack = iee_stack; -+ token->tmp_page = tmp_page; -+} -+ -+void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->pgd = NULL; -+ token->valid = false; -+ token->kernel_stack = NULL; -+} -+ -+void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct *tsk) -+{ -+ struct task_token *token = (struct task_token *)(__phys_to_iee(__pa(tsk))); -+ token->valid = true; -+} -+ -+void __iee_code _iee_unset_token(unsigned long __unused, pte_t *token_ptep, pte_t *token_page_ptep, void *token, void *token_page, unsigned long order, int use_block_pmd) -+{ -+ token_ptep = (pte_t *)(__phys_to_iee(__pa(token_ptep))); -+ token_page_ptep = (pte_t *)(__phys_to_iee(__pa(token_page_ptep))); -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | ___D | __RW); -+ WRITE_ONCE(*pmdp, pmd); -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__iee_pa(token + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ token_ptep++; -+ } -+ } -+ else -+ { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | (__phys_to_pfn(__iee_pa(token + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte(pte_val(pte) | ___D | __RW); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_set_token(unsigned long __unused, pte_t *token_ptep, pte_t *token_page_ptep, void *token_page, unsigned long order, int use_block_pmd) -+{ -+ token_ptep = (pte_t *)(__phys_to_iee(__pa(token_ptep))); -+ token_page_ptep = (pte_t *)(__phys_to_iee(__pa(token_page_ptep))); -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) & ~__RW) & ~___D); -+ WRITE_ONCE(*pmdp, pmd); -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK)) | (__phys_to_pfn(__pa(token_page + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ token_ptep++; -+ } -+ } -+ else { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK)) | (__phys_to_pfn(__pa(token_page + i*PAGE_SIZE)) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte((pte_val(pte) & ~__RW) & ~___D); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+} -+ -+unsigned long __iee_code _iee_test_and_clear_bit(unsigned long __unused, long nr, volatile unsigned long *addr) -+{ -+ unsigned long *iee_addr = (unsigned long*)__phys_to_iee(__pa(addr)); -+ kcsan_mb(); -+ instrument_atomic_read_write(iee_addr + BIT_WORD(nr), sizeof(long)); -+ return arch_test_and_clear_bit(nr, iee_addr); -+} -+ -+void __iee_code _iee_set_sensitive_pte(unsigned long __unused, pte_t *lm_ptep, int order, int use_block_pmd) -+{ -+ int i; -+ lm_ptep = (pte_t *)(__phys_to_iee(__pa(lm_ptep))); -+ if (use_block_pmd) { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) & (~__RW) & (~___D))); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte((pte_val(pte) & (~__RW) & (~___D))); -+ WRITE_ONCE(*lm_ptep, pte); -+ lm_ptep++; -+ } -+ } -+} -+ -+void __iee_code _iee_unset_sensitive_pte(unsigned long __unused, pte_t *lm_ptep, int order, int use_block_pmd) -+{ -+ int i; -+ lm_ptep = (pte_t *)(__phys_to_iee(__pa(lm_ptep))); -+ if (use_block_pmd) -+ { -+ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); -+ pmd = __pmd((pmd_val(pmd) | __RW | ___D)); -+ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); -+ } -+ else -+ { -+ for(i = 0; i < (1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*lm_ptep); -+ pte = __pte(pte_val(pte) | __RW | ___D); -+ WRITE_ONCE(*lm_ptep, pte); -+ lm_ptep++; -+ } -+ } -+} -+ -+#ifdef CONFIG_CREDP -+static struct cred* iee_cred(unsigned long __unused, struct cred *cred) { -+ if(cred == &init_cred) -+ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred)); -+ else -+ cred = (struct cred *)(__phys_to_iee(__pa(cred))); -+ return cred; -+} -+ -+void __iee_code _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) -+{ -+ cred = iee_cred(__unused, cred); -+ *((struct rcu_head **)(&(cred->rcu.func))) = rcu; -+} -+ -+void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->security = security; -+} -+ -+unsigned long __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, struct cred *cred, int flag, int nr) -+{ -+ cred = iee_cred(__unused, cred); -+ switch (flag) -+ { -+ case AT_ADD: { -+ atomic_long_add(nr, &cred->usage); -+ return 0; -+ } -+ case AT_INC_NOT_ZERO: { -+ return atomic_long_inc_not_zero(&cred->usage); -+ } -+ case AT_SUB_AND_TEST: { -+ return atomic_long_sub_and_test(nr, &cred->usage); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) -+{ -+ cred = iee_cred(__unused, cred); -+ atomic_long_set(&cred->usage,i); -+} -+ -+void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->non_rcu = non_rcu; -+} -+ -+void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, struct key *session_keyring) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->session_keyring = session_keyring; -+} -+ -+void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, struct key *process_keyring) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->process_keyring = process_keyring; -+} -+ -+void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, struct key *thread_keyring) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->thread_keyring = thread_keyring; -+} -+ -+void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, struct key *request_key_auth) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->request_key_auth = request_key_auth; -+} -+ -+void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->jit_keyring = jit_keyring; -+} -+ -+void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, kernel_cap_t cap_inheritable) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->cap_inheritable = cap_inheritable; -+} -+ -+void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, kernel_cap_t cap_permitted) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->cap_permitted = cap_permitted; -+} -+ -+void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, kernel_cap_t cap_effective) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->cap_effective = cap_effective; -+} -+ -+void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->cap_bset = cap_bset; -+} -+ -+void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->cap_ambient = cap_ambient; -+} -+ -+void __iee_code _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, unsigned securebits) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->securebits = securebits; -+} -+ -+void __iee_code _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, struct group_info *group_info) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->group_info = group_info; -+} -+ -+void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->ucounts = ucounts; -+} -+ -+void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, struct user_namespace *user_ns) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->user_ns = user_ns; -+} -+ -+void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->user = user; -+} -+ -+void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->fsgid = fsgid; -+} -+ -+void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->fsuid = fsuid; -+} -+ -+void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->egid = egid; -+} -+ -+void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->euid = euid; -+} -+ -+void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->sgid = sgid; -+} -+ -+void __iee_code _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->suid = suid; -+} -+ -+void __iee_code _iee_copy_cred(unsigned long __unused, struct cred *old, struct cred *new) -+{ -+ if (new == &init_cred) { -+ panic("copy_cred for init_cred: %lx\n", (unsigned long)new); -+ } -+ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); -+ struct cred *_new = (struct cred *)__phys_to_iee(__pa(new)); -+ _iee_memcpy(__unused, new, old, sizeof(struct cred)); -+ *(struct rcu_head **)(&(_new->rcu.func)) = rcu; -+ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func); -+} -+ -+void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->gid = gid; -+} -+ -+void __iee_code _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) -+{ -+ cred = iee_cred(__unused, cred); -+ cred->uid = uid; -+} -+#endif -+ -+#ifdef CONFIG_KEYP -+unsigned long __iee_code _iee_set_key_flag_bit(unsigned long __unused, struct key *key, -+ long nr, int flag) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ switch (flag) { -+ case SET_BIT_OP: { -+ set_bit(nr, &key->flags); -+ break; -+ } -+ case TEST_AND_CLEAR_BIT: { -+ return test_and_clear_bit(nr, &key->flags); -+ } -+ case TEST_AND_SET_BIT: { -+ return test_and_set_bit(nr, &key->flags); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_key_restrict_link(unsigned long __unused, -+ struct key *key, -+ struct key_restriction *restrict_link) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->restrict_link = restrict_link; -+} -+ -+void __iee_code _iee_set_key_magic(unsigned long __unused, struct key *key, -+ unsigned magic) -+{ -+#ifdef KEY_DEBUGGING -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->magic = magic; -+#endif -+} -+ -+void __iee_code _iee_set_key_flags(unsigned long __unused, struct key *key, -+ unsigned long flags) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->flags = flags; -+} -+ -+void __iee_code _iee_set_key_index_key(unsigned long __unused, -+ struct key *key, -+ struct keyring_index_key* index_key) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->index_key = *index_key; -+} -+ -+void __iee_code _iee_set_key_hash(unsigned long __unused, struct key *key, -+ unsigned long hash) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->hash = hash; -+} -+ -+void __iee_code _iee_set_key_len_desc(unsigned long __unused, struct key *key, -+ unsigned long len_desc) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->len_desc = len_desc; -+} -+ -+void __iee_code _iee_set_key_type(unsigned long __unused, struct key *key, -+ struct key_type *type) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->type = type; -+} -+ -+void __iee_code _iee_set_key_domain_tag(unsigned long __unused, -+ struct key *key, -+ struct key_tag *domain_tag) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->domain_tag = domain_tag; -+} -+ -+void __iee_code _iee_set_key_description(unsigned long __unused, -+ struct key *key, char *description) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->description = description; -+} -+ -+void __iee_code _iee_set_key_uid(unsigned long __unused, struct key *key, -+ kuid_t uid) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->uid = uid; -+} -+ -+void __iee_code _iee_set_key_gid(unsigned long __unused, struct key *key, -+ kgid_t gid) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->gid = gid; -+} -+ -+void __iee_code _iee_set_key_perm(unsigned long __unused, struct key *key, -+ key_perm_t perm) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->perm = perm; -+} -+ -+void __iee_code _iee_set_key_quotalen(unsigned long __unused, struct key *key, -+ unsigned short quotalen) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->quotalen = quotalen; -+} -+ -+void __iee_code _iee_set_key_datalen(unsigned long __unused, struct key *key, -+ unsigned short datalen) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->datalen = datalen; -+} -+ -+void __iee_code _iee_set_key_state(unsigned long __unused, struct key *key, -+ short state) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ WRITE_ONCE(key->state, state); -+} -+ -+void __iee_code _iee_set_key_user(unsigned long __unused, struct key *key, -+ struct key_user *user) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->user = user; -+} -+ -+void __iee_code _iee_set_key_security(unsigned long __unused, struct key *key, -+ void *security) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->security = security; -+} -+ -+void __iee_code _iee_set_key_expiry(unsigned long __unused, struct key *key, -+ time64_t expiry) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->expiry = expiry; -+} -+ -+void __iee_code _iee_set_key_revoked_at(unsigned long __unused, -+ struct key *key, time64_t revoked_at) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->revoked_at = revoked_at; -+} -+ -+void __iee_code _iee_set_key_last_used_at(unsigned long __unused, -+ struct key *key, -+ time64_t last_used_at) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->last_used_at = last_used_at; -+} -+ -+unsigned long __iee_code _iee_set_key_usage(unsigned long __unused, struct key *key, -+ int n, int flag) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ switch (flag) { -+ case REFCOUNT_INC: { -+ refcount_inc(&key->usage); -+ break; -+ } -+ case REFCOUNT_SET: { -+ refcount_set(&key->usage, n); -+ break; -+ } -+ case REFCOUNT_DEC_AND_TEST: { -+ return refcount_dec_and_test(&key->usage); -+ } -+ case REFCOUNT_INC_NOT_ZERO: { -+ return refcount_inc_not_zero(&key->usage); -+ } -+ } -+ return 0; -+} -+ -+void __iee_code _iee_set_key_serial(unsigned long __unused, struct key *key, -+ key_serial_t serial) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->serial = serial; -+} -+ -+void __iee_code _iee_set_key_watchers(unsigned long __unused, struct key *key, struct watch_list *watchers) -+{ -+#ifdef CONFIG_KEY_NOTIFICATIONS -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->watchers = watchers; -+#endif -+} -+ -+void __iee_code _iee_set_key_union(unsigned long __unused, struct key *key, -+ struct key_union *key_union) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->graveyard_link.next = (struct list_head *)key_union; -+} -+ -+void __iee_code _iee_set_key_struct(unsigned long __unused, struct key *key, -+ struct key_struct *key_struct) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->name_link.prev = (struct list_head *)key_struct; -+} -+ -+void __iee_code _iee_set_key_payload(unsigned long __unused, struct key *key, -+ union key_payload *key_payload) -+{ -+ key = (struct key *)(__phys_to_iee(__pa(key))); -+ key->name_link.next = (struct list_head *)key_payload; -+} -+#endif -+ -+#ifdef CONFIG_IEE_SELINUX_P -+void __iee_code _iee_set_selinux_status_pg(unsigned long __unused, struct page* new_page) -+{ -+ struct page** iee_addr = (struct page**)__phys_to_iee(__pa_symbol(&(selinux_state.status_page))); -+ *iee_addr = new_page; -+} -+ -+void __iee_code _iee_set_selinux_enforcing(unsigned long __unused, bool value) -+{ -+ *(bool*)__phys_to_iee(__pa_symbol(&(selinux_state.enforcing))) = value; -+} -+ -+void __iee_code _iee_mark_selinux_initialized(unsigned long __unused) -+{ -+ smp_store_release(((bool*)__phys_to_iee(__pa_symbol(&(selinux_state.initialized)))), true); -+ printk("IEE: Mark selinux initialized."); -+} -+ -+void __iee_code _iee_set_sel_policy_cap(unsigned long __unused, unsigned int idx, int cap) -+{ -+ *(bool*)__phys_to_iee(__pa_symbol(&(selinux_state.policycap[idx]))) = cap; -+} -+ -+/* -+ * Please make sure param iee_new_policy is from policy_jar memcache. -+ * Need to free new_policy after calling this func as it's only used to -+ * trans data from kernel. -+ */ -+void __iee_code _iee_sel_rcu_assign_policy(unsigned long __unused, struct selinux_policy* new_policy, -+ struct selinux_policy* iee_new_policy) -+{ -+ /* TODO: Verify informations from incoming policy. */ -+ // /* Make sure iee_new_policy is from policy_jar memcache. */ -+ // struct slab* policy_pg = (struct slab*)pfn_to_page(__pa(iee_new_policy) >> PAGE_SHIFT); -+ // if (policy_pg->slab_cache != policy_jar) -+ // printk("IEE SELINUXP ERROR: new policy is not from iee memcache."); -+ /* Copy data from kernel to new allocated policy struct inside iee. */ -+ struct selinux_policy* iee_addr = (struct selinux_policy *)(__phys_to_iee(__pa(iee_new_policy))); -+ memcpy(iee_addr, new_policy, sizeof(struct selinux_policy)); -+ -+ rcu_assign_pointer(*((struct selinux_policy**)__phys_to_iee(__pa_symbol(&(selinux_state.policy)))), -+ iee_new_policy); -+ printk("IEE: assigned rcu pointer selinux_state.policy."); -+} -+#endif -+ -+/* iee si */ -+bool iee_pgt_jar_init __iee_si_data; -+bool iee_init_done __iee_si_data; -+unsigned long iee_base_swapper_pg_dir __iee_si_data; -+extern struct static_key_false cr_pinning; -+ -+u64 __iee_si_code notrace iee_si_handler(int flag, ...) -+{ -+ va_list pArgs; -+ u64 val; -+ -+ va_start(pArgs, flag); -+ switch (flag) { -+ case IEE_SI_TEST: -+ break; -+ case IEE_WRITE_CR0: { -+ val = va_arg(pArgs, u64); -+ unsigned long bits_missing = 0; -+ set_register_cr0: -+ asm volatile("mov %0,%%cr0" : "+r"(val) : : "memory"); -+ if (static_branch_likely(&cr_pinning)) { -+ if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { -+ bits_missing = X86_CR0_WP; -+ val |= bits_missing; -+ goto set_register_cr0; -+ } -+ /* Warn after we've set the missing bits. */ -+ WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); -+ } -+ break; -+ } -+ #ifdef CONFIG_KOI -+ case IEE_SWITCH_TO_KOI: -+ #endif -+ case IEE_WRITE_CR3: { -+ val = va_arg(pArgs, u64); -+ asm volatile("mov %0,%%cr3" : : "r"(val) : "memory"); -+ break; -+ } -+ #ifdef CONFIG_KOI -+ case IEE_SWITCH_TO_KERNEL: { -+ val = iee_base_swapper_pg_dir; -+ asm volatile("mov %0,%%cr3" : : "r"(val) : "memory"); -+ break; -+ } -+ #endif -+ case IEE_WRITE_CR4: { -+ break; -+ } -+ case IEE_LOAD_IDT: { -+ const struct desc_ptr *new_val = va_arg(pArgs, const struct desc_ptr*); -+ asm volatile("lidt %0"::"m" (*new_val)); -+ break; -+ } -+ } -+ va_end(pArgs); -+ return 0; -+} -\ No newline at end of file -diff --git a/arch/x86/kernel/iee/pgtable-slab.c b/arch/x86/kernel/iee/pgtable-slab.c -new file mode 100644 -index 000000000000..518bdfa5fc98 ---- /dev/null -+++ b/arch/x86/kernel/iee/pgtable-slab.c -@@ -0,0 +1,169 @@ -+#include -+#include -+#include -+#include -+#include -+ -+#define PGTABLE_INIT_ORDER 7 -+struct kmem_cache *pgtable_jar; -+struct kmem_cache *ptdesc_jar; -+struct kmem_cache *pgd_jar; -+unsigned long pgtable_jar_offset; -+unsigned long pgd_jar_offset; -+ -+extern void early_pgtable_jar_alloc(struct kmem_cache *pgtable_jar); -+ -+void iee_ptdesc_init(struct page *page) -+{ -+ struct ptdesc_t *tmp = kmem_cache_alloc(ptdesc_jar, GFP_KERNEL | __GFP_ZERO); -+ if (!tmp) { -+ panic("IEE: failed to alloc ptdesc_jar"); -+ } -+ spin_lock_init(&tmp->ptl); -+ page_to_iee_ptdesc(page) = tmp; -+ -+ page_to_iee_ptdesc(page)->page = page; -+} -+ -+void iee_ptdesc_free(struct page *page) -+{ -+ kmem_cache_free(ptdesc_jar, page_to_iee_ptdesc(page)); -+ page_to_iee_ptdesc(page) = NULL; -+} -+ -+void __init iee_pmd_pgtable_init(pud_t *pud) { -+ struct page *page; -+ pmd_t *orig_pmd = pud_pgtable(*pud); -+ pmd_t *pmd; -+ int i; -+ -+ for (i = 0; i < PTRS_PER_PMD; i++) { -+ pmd = orig_pmd + i; -+ if (pmd_none(*pmd) || pmd_bad(*pmd)) -+ continue; -+ page = pmd_page(*pmd); -+ iee_ptdesc_init(page); -+ } -+} -+ -+void __init iee_pud_pgtable_init(p4d_t *p4d) { -+ struct page *page; -+ pud_t *orig_pud = p4d_pgtable(*p4d); -+ pud_t *pud; -+ int i; -+ -+ for (i = 0; i < PTRS_PER_PUD; i++) { -+ pud = orig_pud + i; -+ if (pud_none(*pud) || pud_bad(*pud)) -+ continue; -+ iee_pmd_pgtable_init(pud); -+ page = pud_page(*pud); -+ iee_ptdesc_init(page); -+ } -+} -+ -+void __init iee_p4d_pgtable_init(pgd_t *pgd) { -+ struct page *page; -+ p4d_t *orig_p4d = (p4d_t *)pgd_page_vaddr(*pgd); -+ p4d_t *p4d; -+ int i; -+ -+ for (i = 0; i < PTRS_PER_P4D; i++) { -+ p4d = orig_p4d + i; -+ if (p4d_none(*p4d) || p4d_bad(*p4d)) -+ continue; -+ iee_pud_pgtable_init(p4d); -+ page = p4d_page(*p4d); -+ iee_ptdesc_init(page); -+ } -+} -+ -+void __init iee_pgtable_init(void) -+{ -+ int i; -+ pgd_t *pgd; -+ struct page* page; -+ -+ pr_err("IEE: iee_ptdesc_base=%lx, vmemmap_base=%lx", iee_ptdesc_base, vmemmap_base); -+ -+ ptdesc_jar = kmem_cache_create("ptdesc_jar", sizeof(struct ptdesc_t), 0, SLAB_PANIC|SLAB_RED_ZONE, NULL); -+ pgtable_jar = kmem_cache_create("pgtable_jar", PAGE_SIZE, PAGE_SIZE, -+ SLAB_PANIC, NULL); -+ pgd_jar = kmem_cache_create("pgd_jar", (PAGE_SIZE << PGD_ALLOCATION_ORDER), -+ (PAGE_SIZE << PGD_ALLOCATION_ORDER), SLAB_PANIC, NULL); -+ -+ for(i = 0; i < ((1 << (PGTABLE_INIT_ORDER))/nr_cpu_ids); i++) { -+ early_pgtable_jar_alloc(pgtable_jar); -+ // early_pgtable_jar_alloc(pgd_jar); -+ } -+ for (i = 0; i < PTRS_PER_PGD; i++) { -+ pgd = swapper_pg_dir + i; -+ if (pgtable_l5_enabled()) { -+ if (pgd_none_or_clear_bad(pgd)) -+ continue; -+ iee_p4d_pgtable_init(pgd); -+ } -+ else { -+ if (p4d_none_or_clear_bad((p4d_t *)pgd)) -+ continue; -+ iee_pud_pgtable_init((p4d_t *)pgd); -+ } -+ page = pgd_page(*pgd); -+ iee_ptdesc_init(page); -+ } -+ iee_pgt_jar_init = true; -+} -+ -+void *get_iee_pgtable_page(gfp_t gfpflags) -+{ -+ struct page *page; -+ void *res = kmem_cache_alloc(pgtable_jar, gfpflags); -+ if(!res) { -+ panic("IEE: failed to alloc pgtable_jar"); -+ } -+ iee_rw_gate(IEE_OP_SET_FREEPTR, (void **)((unsigned long)res + pgtable_jar_offset), NULL); -+ -+ page = virt_to_page(res); -+ iee_ptdesc_init(page); -+ return res; -+} -+ -+void free_iee_pgtable_page(void *obj) -+{ -+ struct page *page = virt_to_page(obj); -+ iee_ptdesc_free(page); -+ -+ kmem_cache_free(pgtable_jar, obj); -+} -+ -+void *get_iee_pgd_page(gfp_t gfpflags) -+{ -+ struct page *page; -+ void *res = kmem_cache_alloc(pgd_jar, gfpflags); -+ if(!res) { -+ panic("IEE: failed to alloc pgd_jar"); -+ } -+ iee_rw_gate(IEE_OP_SET_FREEPTR, (void **)((unsigned long)res + pgd_jar_offset), NULL); -+ -+ page = virt_to_page(res); -+ iee_ptdesc_init(page); -+ -+ #if PGD_ALLOCATION_ORDER == 1 -+ page = virt_to_page((unsigned long)res + PAGE_SIZE); -+ iee_ptdesc_init(page); -+ #endif -+ return res; -+} -+ -+void free_iee_pgd_page(void *obj) -+{ -+ struct page *page = virt_to_page(obj); -+ iee_ptdesc_free(page); -+ -+ #if PGD_ALLOCATION_ORDER == 1 -+ page = virt_to_page((unsigned long)obj + PAGE_SIZE); -+ iee_ptdesc_free(page); -+ #endif -+ -+ kmem_cache_free(pgd_jar, obj); -+} -\ No newline at end of file -diff --git a/arch/x86/kernel/iee/stack-slab.c b/arch/x86/kernel/iee/stack-slab.c -new file mode 100644 -index 000000000000..533cb7183d63 ---- /dev/null -+++ b/arch/x86/kernel/iee/stack-slab.c -@@ -0,0 +1,25 @@ -+#include -+#include -+ -+struct kmem_cache *iee_stack_jar; -+ -+void __init iee_stack_init(void) -+{ -+ iee_stack_jar = kmem_cache_create("iee_stack_jar", (PAGE_SIZE << 2), -+ (PAGE_SIZE << 2), SLAB_PANIC, NULL); -+} -+ -+void *get_iee_stack(void) -+{ -+ void *obj = kmem_cache_alloc(iee_stack_jar, GFP_KERNEL); -+ if (!obj) { -+ pr_err("IEE: failed to alloc iee_stack_jar"); -+ return NULL; -+ } -+ return __phys_to_iee(__pa(obj)); -+} -+ -+void free_iee_stack(void *obj) -+{ -+ kmem_cache_free(iee_stack_jar, __va(__iee_pa(obj))); -+} -diff --git a/arch/x86/kernel/koi/Makefile b/arch/x86/kernel/koi/Makefile -new file mode 100644 -index 000000000000..9be8710b714a ---- /dev/null -+++ b/arch/x86/kernel/koi/Makefile -@@ -0,0 +1 @@ -+obj-y += koi.o -\ No newline at end of file -diff --git a/arch/x86/kernel/koi/koi.c b/arch/x86/kernel/koi/koi.c -new file mode 100644 -index 000000000000..77500a7de3ef ---- /dev/null -+++ b/arch/x86/kernel/koi/koi.c -@@ -0,0 +1,1414 @@ -+#include "asm-generic/rwonce.h" -+#include "asm/current.h" -+#include "asm/desc_defs.h" -+#include "asm/page.h" -+#include "asm/page_64_types.h" -+#include "asm/page_types.h" -+#include "asm/percpu.h" -+#include "asm/pgtable.h" -+#include "asm/pgalloc.h" -+#include "asm/pgtable_64.h" -+#include "asm/pgtable_64_types.h" -+#include "asm/pgtable_types.h" -+#include "asm/processor-flags.h" -+#include "asm/special_insns.h" -+#include "linux/cpumask.h" -+#include "linux/export.h" -+#include "linux/mm_types.h" -+#include "linux/percpu-defs.h" -+#include "linux/pgtable.h" -+#include "linux/mm.h" -+#include "asm/pgalloc.h" -+#include "linux/printk.h" -+#include "linux/slab.h" -+#include "asm/koi.h" -+#include "linux/module.h" -+#include "linux/spinlock.h" -+#include "linux/vmalloc.h" -+#include "asm-generic/sections.h" -+#include "asm/desc.h" -+#ifdef CONFIG_IEE -+#include "asm/iee-si.h" -+#endif -+ -+// #define DEBUG -+ -+#ifdef DEBUG -+#define debug_printk(...) printk(KERN_ERR __VA_ARGS__) -+#else -+#define debug_printk(...) -+#endif -+ -+#define __koi_code __section(".koi.text") -+#define __koi_data __section(".data..koi") -+ -+extern unsigned long __koi_code_start[]; -+extern unsigned long __koi_code_end[]; -+extern unsigned long __koi_data_start[]; -+extern unsigned long __koi_data_end[]; -+#ifdef CONFIG_IEE -+extern unsigned long __iee_si_data_start[]; -+extern unsigned long __iee_si_text_start[]; -+extern unsigned long iee_offset; -+#endif -+ -+// extern unsigned long koi_idt_descr_addr; -+extern struct desc_ptr idt_descr; -+extern gate_desc idt_table[IDT_ENTRIES]; -+ -+#define IDT_TABLE_SIZE (IDT_ENTRIES * sizeof(gate_desc)) -+ -+extern void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp); -+ -+extern void error_entry(void); -+ -+extern void error_return(void); -+ -+DEFINE_PER_CPU_PAGE_ALIGNED(unsigned long, koi_kern_cr3); -+EXPORT_SYMBOL(koi_kern_cr3); -+ -+DEFINE_HASHTABLE(koi_mem_htbl, HASH_TABLE_BIT); -+ -+DEFINE_SPINLOCK(koi_mem_htbl_spin_lock); -+EXPORT_SYMBOL(koi_mem_htbl_spin_lock); -+ -+#ifndef CONFIG_IEE -+__koi_code void koi_error_entry_to_kernel(void) { -+ unsigned long cr3 = this_cpu_read(koi_kern_cr3); -+ asm volatile("mov %0,%%cr3": : "r" (cr3) : "memory"); -+} -+ -+__koi_code void koi_error_return_to_ko(unsigned long addr) { -+ struct koi_mem_hash_node *ko; -+ unsigned long cr3; -+ int bkt; -+ unsigned long flags; -+ struct task_token *token_addr = -+ (struct task_token *)((unsigned long)current + (unsigned long)KOI_OFFSET); -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ spin_lock_irqsave(&ko->mod_lock, flags); -+ if (!ko->is_valid) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ goto out; -+ } -+ if (ko->mod != NULL && ko->mod->mem[MOD_INIT_TEXT].base != NULL) { -+ if (addr >= (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base + ko->mod->mem[MOD_INIT_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (token_addr->current_ttbr1 == ko->ko_cr3) { -+ cr3 = __read_cr3(); -+ this_cpu_write(koi_kern_cr3, cr3); -+ cr3 = (cr3 & X86_CR3_PCID_MASK) | PTI_USER_PCID_MASK; -+ cr3 |= ko->ko_cr3; -+ rcu_read_unlock(); -+ native_write_cr3(cr3); -+ return; -+ } -+ goto out; -+ } -+ } -+ if (addr >= (unsigned long)ko->mod->mem[MOD_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_TEXT].base + ko->mod->mem[MOD_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ if (token_addr->current_ttbr1 == ko->ko_cr3) { -+ cr3 = __read_cr3(); -+ this_cpu_write(koi_kern_cr3, cr3); -+ cr3 = (cr3 & X86_CR3_PCID_MASK) | PTI_USER_PCID_MASK; -+ cr3 |= ko->ko_cr3; -+ rcu_read_unlock(); -+ native_write_cr3(cr3); -+ return; -+ } -+ goto out; -+ } -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ } -+out: -+ rcu_read_unlock(); -+ return; -+} -+#else -+__koi_code void koi_error_entry_to_kernel(void) { -+ unsigned long cr3 = this_cpu_read(koi_kern_cr3); -+ asm volatile("mov %0,%%cr3": : "r" (cr3) : "memory"); -+} -+ -+__koi_code void koi_error_return_to_ko(unsigned long addr) { -+ struct koi_mem_hash_node *ko; -+ unsigned long cr3; -+ int bkt; -+ unsigned long flags; -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ spin_lock_irqsave(&ko->mod_lock, flags); -+ if (!ko->is_valid) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ goto out; -+ } -+ if (ko->mod != NULL && ko->mod->mem[MOD_INIT_TEXT].base != NULL) { -+ if (addr >= (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_INIT_TEXT].base + ko->mod->mem[MOD_INIT_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ unsigned long current_ttbr1 = iee_rw_gate(IEE_READ_TOKEN_TTBR1, current); -+ if (current_ttbr1 == ko->ko_cr3) { -+ cr3 = __read_cr3(); -+ this_cpu_write(koi_kern_cr3, cr3); -+ cr3 = (cr3 & X86_CR3_PCID_MASK) | PTI_USER_PCID_MASK; -+ cr3 |= ko->ko_cr3; -+ rcu_read_unlock(); -+ native_write_cr3(cr3); -+ return; -+ } -+ goto out; -+ } -+ } -+ if (addr >= (unsigned long)ko->mod->mem[MOD_TEXT].base && -+ addr < (unsigned long)ko->mod->mem[MOD_TEXT].base + ko->mod->mem[MOD_TEXT].size) { -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ unsigned long current_ttbr1 = iee_rw_gate(IEE_READ_TOKEN_TTBR1, current); -+ if (current_ttbr1 == ko->ko_cr3) { -+ cr3 = __read_cr3(); -+ this_cpu_write(koi_kern_cr3, cr3); -+ cr3 = (cr3 & X86_CR3_PCID_MASK) | PTI_USER_PCID_MASK; -+ cr3 |= ko->ko_cr3; -+ rcu_read_unlock(); -+ native_write_cr3(cr3); -+ return; -+ } -+ goto out; -+ } -+ spin_unlock_irqrestore(&ko->mod_lock, flags); -+ } -+out: -+ rcu_read_unlock(); -+ return; -+} -+#endif -+ -+pte_t *find_pte(unsigned long addr, pgd_t *pgdir) -+{ -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); -+ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { -+ debug_printk("pgdp 0x%lx not available\n", (unsigned long) pgdp); -+ return NULL; -+ } -+ debug_printk( "pgd=0x%lx, pgd_val=0x%lx\n", (unsigned long) pgdp, pgd_val(*pgdp)); -+ -+ p4dp = p4d_offset(pgdp, addr); -+ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) { -+ debug_printk("p4dp 0x%lx not available\n", (unsigned long)p4dp); -+ return NULL; -+ } -+ debug_printk( "p4d=0x%lx, p4d_val=0x%lx\n", (unsigned long) p4dp, p4d_val(*p4dp)); -+ -+ pudp = pud_offset(p4dp, addr); -+ if (pud_none(*pudp) || pud_bad(*pudp)) { -+ debug_printk("pudp 0x%lx not available\n", (unsigned long)pudp); -+ return NULL; -+ } -+ debug_printk( "pud=0x%lx, pud_val=0x%lx\n", (unsigned long)pudp, pud_val(*pudp)); -+ pmdp = pmd_offset(pudp, addr); -+ debug_printk( "pmd=0x%lx, pmd_val=0x%lx\n",(unsigned long) pmdp, pmd_val(*pmdp)); -+ if (is_swap_pmd(*pmdp) || pmd_trans_huge(*pmdp) || -+ (pmd_devmap(*pmdp))) { -+ return (pte_t *)pmdp; -+ } -+ if (pmd_none(*pmdp)) { -+ debug_printk("pmdp 0x%lx not available none\n", (unsigned long)pmdp); -+ return NULL; -+ } -+ if (pmd_bad(*pmdp)) { -+ debug_printk("pmdp 0x%lx bad, pmd_val=0x%16lx\n",(unsigned long) pmdp, pmd_val(*pmdp)); -+ return NULL; -+ } -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ debug_printk( "pte=0x%lx, pte_val=0x%lx\n", (unsigned long)ptep, pte_val(*ptep)); -+ if (pte_none(*ptep)) { -+ debug_printk("ptep 0x%lx not available\n", (unsigned long)ptep); -+ return NULL; -+ } -+ return ptep; -+} -+ -+struct page *find_page(unsigned long addr, pgd_t *pgdir) -+{ -+ struct page *curr_page; -+ pte_t *ptep; -+ debug_printk("addr=0x%lx\n", addr); -+ // pgd_t *pgdir = swapper_pg_dir; -+ ptep = find_pte(addr, pgdir); -+ if (!ptep) { -+ return NULL; -+ } -+ curr_page = pte_page(*ptep); -+ debug_printk("page=0x%16lx\n", (unsigned long) curr_page); -+ return curr_page; -+} -+ -+__attribute__((unused)) static void check_pgd(pgd_t *new_pgd, pgd_t *old_pgd, -+ unsigned long addr) -+{ -+ pte_t *old_pte = find_pte(addr, old_pgd); -+ pte_t *new_pte = find_pte(addr, new_pgd); -+ if (old_pte == NULL || new_pte == NULL) { -+ debug_printk( "old_pte=0x%16lx, new_pte=0x%16lx\n", (unsigned long)old_pte, (unsigned long)new_pte); -+ return; -+ } -+ unsigned long old_pfn = pte_pfn(*old_pte); -+ unsigned long new_pfn = pte_pfn(*new_pte); -+ struct page *old_page, *new_page; -+ unsigned long *new_address, *old_address; -+ if (old_pfn != new_pfn) { -+ debug_printk( -+ "pfn different! old_pfn=0x%16lx, new_pfn=0x%16lx\n", -+ old_pfn, new_pfn); -+ } -+ old_page = find_page(addr, old_pgd); -+ debug_printk("swapper_pg_dir walk finished\n"); -+ new_page = find_page(addr, new_pgd); -+ debug_printk("new pgd walk finished\n"); -+ -+ if (new_page != old_page) -+ debug_printk("Wrong page walk\n"); -+ if (new_page == NULL) { -+ debug_printk("new page not available\n"); -+ } -+ if (old_page == NULL) { -+ debug_printk("old page not available\n"); -+ } -+ if (old_page == NULL || new_page == NULL) -+ return; -+ new_address = (unsigned long *)page_address(new_page); -+ old_address = (unsigned long *)page_address(old_page); -+ -+ debug_printk("addr=0x%lx, new=0x%lx, old=0x%lx\n", addr, (unsigned long)new_address, -+ (unsigned long)old_address); -+ if (new_address != old_address) { -+ debug_printk("Different page address!\n"); -+ } else { -+ debug_printk("check pgd pass successfully!\n"); -+ } -+} -+ -+// int __koi_pte_alloc(struct mm_struct *mm, pmd_t *pmd) -+// { -+// spinlock_t *ptl; -+// // pgtable_t new = alloc_page(GFP_PGTABLE_KERNEL); -+// pte_t *new = pte_alloc_one_kernel(mm); -+// // debug_printk("alloc new=0x%16lx\n", (unsigned long)new); -+// if (!new) -+// return -ENOMEM; -+ -+// /* -+// * Ensure all pte setup (eg. pte page lock and page clearing) are -+// * visible before the pte is made visible to other CPUs by being -+// * put into page tables. -+// * -+// * The other side of the story is the pointer chasing in the page -+// * table walking code (when walking the page table without locking; -+// * ie. most of the time). Fortunately, these data accesses consist -+// * of a chain of data-dependent loads, meaning most CPUs (alpha -+// * being the notable exception) will already guarantee loads are -+// * seen in-order. See the alpha page table accessors for the -+// * smp_rmb() barriers in page table walking code. -+// */ -+// smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ -+ -+// ptl = pmd_lockptr(mm, pmd); -+// debug_printk("pte_alloc ptl=0x%16lx\n", (unsigned long)ptl); -+// spin_lock(ptl); -+// if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ -+// mm_inc_nr_ptes(mm); -+// pmd_populate_kernel(mm, pmd, new); -+// new = NULL; -+// } -+// spin_unlock(ptl); -+// debug_printk("unlock pte_alloc ptl=0x%16lx\n", (unsigned long)ptl); -+// if (new) { -+// pte_free_kernel(mm, new); -+// } -+// return 0; -+// } -+/* -+#define koi_pte_alloc(mm, pmd) \ -+ (unlikely(pmd_none(*(pmd))) && __koi_pte_alloc(mm, pmd)) -+ -+#define koi_pte_offset_map_lock(mm, pmd, address, ptlp) \ -+ ({ \ -+ spinlock_t *__ptl = pte_lockptr(mm, pmd); \ -+ pte_t *__pte = pte_offset_kernel(pmd, address); \ -+ *(ptlp) = __ptl; \ -+ debug_printk("pre lock ptlp=0x%16lx\n", (unsigned long)__ptl); \ -+ __pte; \ -+ }) -+ -+#define koi_pte_alloc_map_lock(mm, pmd, address, ptlp) \ -+ (koi_pte_alloc(mm, pmd) ? \ -+ NULL : \ -+ koi_pte_offset_map_lock(mm, pmd, address, ptlp)) -+*/ -+// static inline pud_t *koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { -+// return (unlikely(p4d_none(*p4d)) && __koi_pud_alloc(mm, p4d, address)) ? -+// NULL : pud_offset(p4d, address); -+// } -+ -+// static inline pmd_t *koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) -+// { -+// return (unlikely(pud_none(*pud)) && __koi_pmd_alloc(mm, pud, address))? -+// NULL: pmd_offset(pud, address); -+// } -+ -+/** -+ * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page -+ * is required to copy this pte. -+*/ -+static inline int koi_copy_present_pte(struct mm_struct *ko_mm, pte_t *dst_pte, -+ pte_t *src_pte, unsigned long addr) -+{ -+ pte_t pte = *src_pte; -+ -+ set_pte(dst_pte, pte); -+ return 0; -+} -+ -+/** -+*koi_copy_pte_range - copy pte from kernel space to driver space -+*/ -+static int koi_copy_pte_range(struct mm_struct *ko_mm, pmd_t *dst_pmd, -+ pmd_t *src_pmd, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pte_t *src_pte, *dst_pte; -+ pte_t *orig_src_pte, *orig_dst_pte; -+ spinlock_t *dst_ptl; -+ int ret = 0; -+ -+ dst_pte = pte_alloc_map_lock(ko_mm, dst_pmd, addr, &dst_ptl); -+ if (!dst_pte) { -+ ret = -ENOMEM; -+ goto unlock; -+ } -+ debug_printk("lock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ src_pte = pte_offset_kernel(src_pmd, addr); -+ if (!src_pte) { -+ pte_unmap_unlock(dst_pte, dst_ptl); -+ debug_printk("unlock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ goto unlock; -+ } -+ // spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); -+ orig_src_pte = src_pte; -+ orig_dst_pte = dst_pte; -+ -+ do { -+ if (pte_none(*src_pte)) -+ continue; -+ if (unlikely(!pte_present(*src_pte))) { -+ debug_printk( -+ "not present pte found: addr=0x%16lx, end=0x%16lx, pte_val=0x%16lx\n", -+ addr, end, pte_val(*src_pte)); -+ continue; -+ } -+ if (pte_valid(*dst_pte)) { -+ continue; -+ } -+ /* koi_copy_present_pte() will clear `*prealloc` if consumed */ -+ ret = koi_copy_present_pte(ko_mm, dst_pte, src_pte, addr); -+ debug_printk( -+ "dst_pte=0x%16lx, dst_pte_val=0x%16lx, src_pte=0x%16lx, src_pte_val=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)dst_pte, pte_val(*dst_pte), (unsigned long)src_pte, pte_val(*src_pte), -+ addr); -+ debug_printk( "pte=0x%16lx, pte_val=0x%16lx\n", (unsigned long)dst_pte, (unsigned long)pte_val(*dst_pte)); -+ } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); -+ pte_unmap_unlock(orig_dst_pte, dst_ptl); -+ debug_printk("unlock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ -+unlock: -+ // spin_unlock_irq(dst_ptl); -+ debug_printk( "unlock1 dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ return ret; -+} -+ -+/** -+* copy huge pmd from kernel space to driver space. -+*/ -+static int koi_copy_huge_pmd(struct mm_struct *ko_mm, pmd_t *dst_pmd, -+ pmd_t *src_pmd, unsigned long addr, pteval_t prot) -+{ -+ spinlock_t *dst_ptl; -+ int ret = -ENOMEM; -+ debug_printk( -+ "hugepmd: src_pmd=0x%16lx, dst_pmd=0x%16lx, src_pmd_val=0x%16lx, dst_pmd_val=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)src_pmd, (unsigned long)dst_pmd, pmd_val(*src_pmd), pmd_val(*dst_pmd), addr); -+ dst_ptl = pmd_lockptr(ko_mm, dst_pmd); -+ debug_printk("pmd lock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ spin_lock(dst_ptl); -+ -+ set_pmd(dst_pmd, __pmd(pmd_val(*src_pmd) | prot)); -+ ret = 0; -+ spin_unlock(dst_ptl); -+ debug_printk("pmd unlock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ debug_printk( -+ "hugepmd: src_pmd=0x%16lx, dst_pmd=0x%16lx, src_pmd_val=0x%16lx, dst_pmd_val=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)src_pmd, (unsigned long)dst_pmd, pmd_val(*src_pmd), pmd_val(*dst_pmd), addr); -+ return ret; -+} -+ -+/** -+*kio_copy_pmd_range - copy pmd from kernel to driver space -+*/ -+static inline int koi_copy_pmd_range(struct mm_struct *ko_mm, pud_t *dst_pud, -+ pud_t *src_pud, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pmd_t *src_pmd, *dst_pmd; -+ unsigned long next, flag; -+ int err; -+ debug_printk( -+ "koi_copy_pmd_range src_pud=0x%16lx, dst_pud=0x%16lx, addr=0x%16lx, end=0x%16lx\n", -+ (unsigned long)src_pud, (unsigned long)dst_pud, addr, end); -+ dst_pmd = pmd_alloc(ko_mm, dst_pud, addr); -+ if (!dst_pmd) { -+ return -ENOMEM; -+ } -+ src_pmd = pmd_offset(src_pud, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ flag = pmd_val(*src_pmd) & PTE_FLAGS_MASK; -+ // debug_printk("src_pmd=0x%16lx, dst_pmd=0x%16lx, addr=0x%16lx\n", src_pmd, dst_pmd, next); -+ // CONFIG_TRANSPARENT_HUGEPAGE is enabled, so we must add copy_huge_pmd -+ if (!pmd_present(*src_pmd)) -+ continue; -+ if (pmd_leaf(*src_pmd)) { -+ // if src_pmd is huge page -+ debug_printk( -+ "koi_copy_pmd_range src_pmd=0x%16lx, dst_pmd=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)src_pmd, (unsigned long)dst_pmd, addr); -+ err = koi_copy_huge_pmd(ko_mm, dst_pmd, src_pmd, addr, prot); -+ if (err == -ENOMEM) -+ return -ENOMEM; -+ continue; -+ } -+ if (koi_copy_pte_range(ko_mm, dst_pmd, src_pmd, addr, next, prot)) -+ return -ENOMEM; -+ set_pmd(dst_pmd, __pmd((pmd_val(*dst_pmd) & (~PTE_FLAGS_MASK)) | flag)); -+ debug_printk( -+ "koi_copy_pmd_range src_pmd_val=0x%16lx, dst_pmd_val=0x%16lx, addr=0x%16lx\n", -+ pmd_val(*src_pmd), pmd_val(*dst_pmd), addr); -+ debug_printk("pmd=0x%16lx, pmd_val=0x%16lx\n", (unsigned long)dst_pmd, (unsigned long)pmd_val(*dst_pmd)); -+ } while (dst_pmd++, src_pmd++, addr = next, addr != end); -+ return 0; -+} -+ -+ -+static int koi_copy_huge_pud(struct mm_struct *ko_mm, pud_t *dst_pud, -+ pud_t *src_pud, unsigned long addr) -+{ -+ spinlock_t *dst_ptl; -+ // pmd_t pmd; -+ int ret = -ENOMEM; -+ debug_printk("huge src_pud=0x%16lx, dst_pud=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)src_pud, (unsigned long)dst_pud, addr); -+ dst_ptl = pud_lockptr(ko_mm, dst_pud); -+ -+ debug_printk("pud lock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ spin_lock(dst_ptl); -+ set_pte((pte_t *)dst_pud, __pte(pud_val(*src_pud))); -+ spin_unlock(dst_ptl); -+ debug_printk("pud unlock dst_ptl=0x%16lx\n", (unsigned long)dst_ptl); -+ ret = 0; -+ return ret; -+} -+ -+/** -+*koi_copy_pud_range - copy pud from kernel to driver -+*/ -+static inline int koi_copy_pud_range(struct mm_struct *ko_mm, p4d_t *dst_p4d, -+ p4d_t *src_p4d, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ pud_t *src_pud, *dst_pud; -+ unsigned long next, flag; -+ dst_pud = pud_alloc(ko_mm, dst_p4d, addr); -+ if (!dst_pud) -+ return -ENOMEM; -+ src_pud = pud_offset(src_p4d, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ flag = pud_val(*src_pud) & PTE_FLAGS_MASK; -+ debug_printk( -+ "koi_copy_pud_range src_pud=0x%16lx, dst_pud=0x%16lx, addr=0x%16lx, next=0x%16lx\n", -+ (unsigned long)src_pud, (unsigned long)dst_pud, addr, next); -+ if (!pud_present(*src_pud)) { -+ continue; -+ } -+ if (pud_leaf(*src_pud)) { -+ // indicates that the src_pud maps to a huge page -+ koi_copy_huge_pud(ko_mm, dst_pud, src_pud, addr); -+ continue; -+ } -+ // if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) { -+ // debug_printk("pud_trans_huge=%d, pud_devmap=%d, src_pud=0x%16lx\n", pud_trans_huge(*src_pud) , pud_devmap(*src_pud), src_pud); -+ // continue; -+ // /* fall through */ -+ // } -+ if (koi_copy_pmd_range(ko_mm, dst_pud, src_pud, addr, next, prot)) -+ return -ENOMEM; -+ debug_printk("koi_copy_pud_range dst_p4d=0x%16lx, dst_p4d_val=0x%16lx, pud=0x%16lx, pud_val=0x%16lx, addr=0x%16lx, next=0x%16lx\n", (unsigned long)dst_p4d, (unsigned long)p4d_val(*dst_p4d), (unsigned long)dst_pud,(unsigned long) pud_val(*dst_pud), addr, next); -+ set_pud(dst_pud, __pud((pud_val(*dst_pud) & (~PTE_FLAGS_MASK)) | flag)); -+ debug_printk("koi_copy_pud_range pud=0x%16lx, pud_val=0x%16lx\n", (unsigned long)dst_pud,(unsigned long) pud_val(*dst_pud)); -+ } while (dst_pud++, src_pud++, addr = next, addr != end); -+ return 0; -+} -+ -+/** -+* koi_copy_p4d_range - map the kernel pagetable to the driver space level by level -+* @ko_mm: the mm_struct of driver module -+* @dst_pgd: destination pgd -+* @src_pgd: source pgd -+* @addr: the start of address -+* @end: the end of address -+*/ -+static inline int koi_copy_p4d_range(struct mm_struct *ko_mm, pgd_t *dst_pgd, -+ pgd_t *src_pgd, unsigned long addr, -+ unsigned long end, pteval_t prot) -+{ -+ p4d_t *src_p4d, *dst_p4d; -+ unsigned long next; -+ dst_p4d = p4d_alloc(ko_mm, dst_pgd, addr); -+ if (!dst_p4d) -+ return -ENOMEM; -+ src_p4d = p4d_offset(src_pgd, addr); -+ do { -+ next = p4d_addr_end(addr, end); -+ if (p4d_none(*src_p4d) || p4d_bad(*src_p4d)) -+ continue; -+ unsigned long flag = p4d_val(*src_p4d) & PTE_FLAGS_MASK; -+ debug_printk( -+ "koi_copy_p4d_range dst_p4d=0x%16lx, dst_p4d_val=0x%16lx, src_p4d=0x%16lx, src_p4d_val=0x%16lx, addr=0x%16lx\n", -+ (unsigned long)dst_p4d, p4d_val(*dst_p4d), (unsigned long)src_p4d, p4d_val(*src_p4d), addr); -+ if (koi_copy_pud_range(ko_mm, dst_p4d, src_p4d, addr, next, prot)) { -+ return -ENOMEM; -+ } -+ set_p4d(dst_p4d, __p4d((p4d_val(*dst_p4d) & (~PTE_FLAGS_MASK)) | flag)); -+ debug_printk( "p4d=0x%16lx, p4d_val=0x%16lx\n", (unsigned long)dst_p4d, (unsigned long)p4d_val(*dst_p4d)); -+ } while (dst_p4d++, src_p4d++, addr = next, addr != end); -+ return 0; -+} -+ -+/** -+*int koi_copy_pagetable - map the address range from "addr" to "end" to the driver pagetable -+*@ko_mm: the mm_struct of the driver module -+*@koi_pg_dir: koi_pg_dir, related to the driver module, the entry for driver pagetable -+*@addr: the starting address of mapping zone -+*@end: the end address of mapping zone -+*/ -+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, -+ unsigned long addr, unsigned long end, pteval_t prot) -+{ -+ int ret = 0; -+ unsigned long next; -+ -+ pgd_t *src_pgd, *dst_pgd; -+ unsigned long flag; -+ if (addr == 0 || end <= addr || (addr & (PAGE_SIZE - 1)) != 0) { -+ printk(KERN_INFO "Wrong Arguments! addr=0x%16lx, end=0x%16lx, %ld\n", addr, end, addr & (PAGE_SIZE - 1)); -+ return 0; -+ } -+ src_pgd = pgd_offset_pgd(swapper_pg_dir, addr); -+ dst_pgd = pgd_offset_pgd(koi_pg_dir, addr); -+ do { -+ flag = pgd_val(*src_pgd) & PTE_FLAGS_MASK; -+ next = pgd_addr_end(addr, end); -+ if (pgd_none(*src_pgd) || pgd_bad(*src_pgd)) -+ continue; -+ debug_printk("koi_copy_pagetable pgd=0x%16lx, pgd_val=0x%16lx, addr=0x%16lx, next=0x%16lx\n", (unsigned long)dst_pgd, (unsigned long)pgd_val(*dst_pgd), addr, next); -+ if (unlikely(koi_copy_p4d_range(ko_mm, dst_pgd, src_pgd, addr, -+ next, prot))) { -+ ret = -ENOMEM; -+ break; -+ } -+ set_pgd(dst_pgd, __pgd((pgd_val(*dst_pgd) & (~PTE_FLAGS_MASK)) | flag)); -+ debug_printk("koi_copy_pagetable set pgd=0x%16lx, pgd_val=0x%16lx, addr=0x%16lx, next=0x%16lx\n", (unsigned long)dst_pgd, (unsigned long)pgd_val(*dst_pgd), addr, next); -+ } while (dst_pgd++, src_pgd++, addr = next, addr != end); -+ return ret; -+} -+ -+void koi_create_pagetable(struct module *mod) -+{ -+ int ret = 0; -+ int cpu; -+ int i; -+ unsigned long addr; -+ struct desc_ptr desc; -+ struct koi_mem_hash_node *new_node = kzalloc(sizeof(struct koi_mem_hash_node), GFP_KERNEL); -+ if (!new_node) { -+ debug_printk( "[NO MEM] KOI ALLOC new node failed\n"); -+ return; -+ } -+ // if (koi_kern_cr3 == 0) { -+ // // CONFIG_ADDRESS_MASKING is not set in opoeneuler_defconfig, so we don't take lam into consideration. -+ // koi_kern_cr3 = __sme_pa(swapper_pg_dir) | CR3_NOFLUSH; -+ // } -+ new_node->pgdp = koi_pgd_alloc(); -+ new_node->ko_mm = kzalloc(sizeof(struct mm_struct) + sizeof(unsigned long) * BITS_TO_LONGS(NR_CPUS),GFP_KERNEL); -+ new_node->mod = mod; -+ init_ko_mm(new_node->ko_mm, new_node->pgdp); -+ new_node->ko_cr3 = __sme_pa(new_node->pgdp) /*| CR3_NOFLUSH */; -+ printk(KERN_ERR "ko_cr3=0x%16lx, pgdp=0x%16lx\n", new_node->ko_cr3, (unsigned long)new_node->pgdp); -+ // map module layout into module pagetable. -+ for_each_mod_mem_type(type) { -+ printk(KERN_ERR "\033[33mcopying mem range, start=0x%16lx, end=0x%16lx\033[0m\n", -+ (unsigned long)mod->mem[type].base, -+ (unsigned long)mod->mem[type].base + mod->mem[type].size); -+ if (!mod->mem[type].base || !mod->mem[type].size) { -+ continue; -+ } -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)mod->mem[type].base, (unsigned long)mod->mem[type].base + mod->mem[type].size, 0); -+ if (ret != 0) -+ printk(KERN_ERR -+ "\033[33mError occured when copying range from 0x%lx to 0x%lx, Eno:%d\033[0m\n", -+ (unsigned long)mod->mem[type].base, -+ (unsigned long)mod->mem[type].base + mod->mem[type].size, -+ ret); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ addr = (unsigned long)per_cpu_ptr(&koi_kern_cr3, cpu); -+ -+ debug_printk( "cpu=%d, addr=0x%16lx, this_cpu_off=0x%16lx, this_cpu_off_addr=0x%16lx\n", cpu, addr, this_cpu_read(this_cpu_off), (unsigned long)per_cpu_ptr(&this_cpu_off, cpu)); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, addr + PAGE_SIZE, 0); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)per_cpu_ptr(&this_cpu_off, cpu) & PAGE_MASK, ((unsigned long)per_cpu_ptr(&this_cpu_off, cpu) & PAGE_MASK) + PAGE_SIZE, 0); -+ } -+ printk(KERN_ERR "mapping koi_data\n"); -+ // map koi_data into module pagetable. -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, -+ (unsigned long)__koi_data_start, -+ (unsigned long)__koi_data_end, 0); -+ printk(KERN_ERR "mapping koi_text\n"); -+ // map koi_text into module pagetable. -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)__koi_code_start, (unsigned long)__koi_code_end, 0); -+ -+ // map exception entry into module pagetable. -+ printk(KERN_ERR "idt_desct=0x%16lx, size =0x%16lx, end=0x%16lx\n", (unsigned long)&idt_descr & PAGE_MASK, sizeof(idt_descr), ((unsigned long)&idt_descr + sizeof(idt_descr) + PAGE_SIZE) & PAGE_MASK); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)&idt_descr & PAGE_MASK, ((unsigned long)&idt_descr + sizeof(idt_descr) + PAGE_SIZE) & PAGE_MASK, 0); -+ printk(KERN_ERR "idt_base=0x%16lx, end=0x%16lx, size=0x%16lx\n", idt_descr.address, (unsigned long)idt_descr.address + IDT_TABLE_SIZE, IDT_TABLE_SIZE - 1); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, idt_descr.address, (unsigned long)idt_descr.address + IDT_TABLE_SIZE, 0); -+ printk(KERN_ERR "__entry_text_start=0x%16lx, __entry_text_end=0x%16lx, error_entry=0x%16lx\n", (unsigned long)__entry_text_start & PAGE_MASK, (unsigned long)(__entry_text_end + PAGE_SIZE) & PAGE_MASK, (unsigned long)error_entry); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)__entry_text_start & PAGE_MASK, (unsigned long)(__entry_text_end + PAGE_SIZE) & PAGE_MASK, 0); -+ // struct desc_ptr dtr; -+ // asm volatile("sidt %0":"=m" (dtr)); -+ // printk(KERN_ERR "dtr.size=%d, address=0x%16lx\n", dtr.size, dtr.address); -+ for (i = 0; i < IDT_ENTRIES; i++) { -+ gate_desc desc = idt_table[i]; -+ addr = desc.offset_low | ((unsigned long)desc.offset_middle << 16) | ((unsigned long)desc.offset_high << 32); -+ printk(KERN_ERR "idt handler addr=0x%16lx, segment=%x, ist=%d, zero=%d, type=%d, dpl=%d, p=%d\n", -+ addr, desc.segment, desc.bits.ist, desc.bits.zero, desc.bits.type, desc.bits.dpl, desc.bits.p); -+ if (addr > (unsigned long)__entry_text_start && addr < (unsigned long)__entry_text_end) -+ continue; -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr & PAGE_MASK, (addr + PAGE_SIZE) & PAGE_MASK, 0); -+ } -+ -+ // mapping gdt into module pagetable. -+ for_each_possible_cpu(cpu) { -+ native_store_gdt(&desc); -+ printk(KERN_ERR "address=0x%16lx, size=%d\n", desc.address, desc.size); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, desc.address, desc.address + PAGE_SIZE, 0); -+ -+ addr = (long)get_cpu_gdt_rw(cpu); // this address is logical address. -+ printk(KERN_ERR "gdt rw=0x%16lx, addr=0x%16lx\n", (unsigned long)per_cpu_ptr(gdt_page.gdt, cpu), addr); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, addr + PAGE_SIZE, 0); -+ -+ addr = (long)get_cpu_gdt_ro(cpu); // this address is same to base stored in GDTR. -+ printk(KERN_ERR "gdtp=0x%16lx, addr=0x%16lx\n", (unsigned long)per_cpu_ptr(gdt_page.gdt, cpu), addr); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, addr + PAGE_SIZE, 0); -+ // struct desc_struct desc = per_cpu(gdt_page.gdt[GDT_ENTRY_KERNEL_CS], cpu); -+ // debug_printk("base0=0x%x, base1=0x%x, base2=0x%x, type=0x%x, s=%d, dpl=%d, p=%d, avl=%d, l=%d, d=%d, g=%d\n", -+ // desc.base0, desc.base1, desc.base2, desc.type, desc.s, desc.dpl, desc.p, desc.avl, desc.l, desc.d, desc.g); -+ } -+ -+ // map irq stack into module pagetable. -+ for_each_possible_cpu(cpu) { -+ // unsigned long top_of_stack = per_cpu(pcpu_hot.top_of_stack, cpu); -+ unsigned long hard_irq_stack = (unsigned long)per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) + 8 - IRQ_STACK_SIZE; -+ debug_printk("top_of_stack=0x%16lx, hard_irq_stack=0x%16lx, current_stack=0x%16lx\n", top_of_stack, hard_irq_stack, (unsigned long)current->stack); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, hard_irq_stack, hard_irq_stack + IRQ_STACK_SIZE, 0); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ // map TSS pointed by TSS descriptor in GDT -+ // We can use segment selector in tr to get the descriptor, and the corresponding base address and limit stored in the GDT entry. -+ addr = (unsigned long) &get_cpu_entry_area(cpu)->tss.x86_tss; -+ debug_printk( "addr=0x%16lx\n",(unsigned long) &get_cpu_entry_area(cpu)->tss.x86_tss); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, (addr + sizeof(struct tss_struct) + PAGE_SIZE) & PAGE_MASK, 0); -+ // map ist stack -+ struct tss_struct *tss = (struct tss_struct *)addr; -+ // printk(KERN_ERR "tss=0x%16lx, tss->ist=0x%16lx, DF=0x%16lx, NMI=0x%16lx, DB=0x%16lx, MCE=0x%16lx\n", -+ // (unsigned long)tss, (unsigned long)tss->x86_tss.ist, (unsigned long)tss->x86_tss.ist[IST_INDEX_DF], (unsigned long)tss->x86_tss.ist[IST_INDEX_NMI], (unsigned long)tss->x86_tss.ist[IST_INDEX_DB],(unsigned long) tss->x86_tss.ist[IST_INDEX_MCE] -+ // ); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)tss->x86_tss.ist[IST_INDEX_DF], (unsigned long)tss->x86_tss.ist[IST_INDEX_DF] + PAGE_SIZE, 0); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)tss->x86_tss.ist[IST_INDEX_NMI], (unsigned long)tss->x86_tss.ist[IST_INDEX_NMI] + PAGE_SIZE, 0); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)tss->x86_tss.ist[IST_INDEX_DB], (unsigned long)tss->x86_tss.ist[IST_INDEX_DB] + PAGE_SIZE, 0); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)tss->x86_tss.ist[IST_INDEX_MCE], (unsigned long)tss->x86_tss.ist[IST_INDEX_MCE] + PAGE_SIZE, 0); -+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, (unsigned long)tss->x86_tss.ist[IST_INDEX_VC], (unsigned long)tss->x86_tss.ist[IST_INDEX_VC] + PAGE_SIZE, 0); -+ } -+ spin_lock_init(&new_node->mod_lock); -+ new_node->is_valid = true; -+ -+ spin_lock(&koi_mem_htbl_spin_lock); -+ hash_add_rcu(koi_mem_htbl, &new_node->node, (unsigned long)new_node->mod); -+ spin_unlock(&koi_mem_htbl_spin_lock); -+ printk(KERN_ERR "create pagetable finished\n"); -+ unsigned long gs = 0; -+ asm volatile( -+ "mov %%gs, %0" -+ : "=r"(gs) -+ : -+ ); -+ printk(KERN_ERR "gs=0x%16lx\n", gs); -+} -+ -+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size) -+{ -+ struct koi_mem_hash_node *target = NULL; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ debug_printk("mapping addr=0x%16lx, end=0x%16lx\n", addr & PAGE_MASK, (addr + size + PAGE_SIZE) & PAGE_MASK); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK, 0); -+ // flush_tlb_one_kernel(addr & PAGE_MASK); -+} -+EXPORT_SYMBOL(koi_map_mem); -+ -+void koi_unmap_pte_table(struct mm_struct *ko_mm, pmd_t *pmd, -+ unsigned long addr, unsigned long end) -+{ -+ pte_t *pte; -+ if (!pmd) -+ return; -+ debug_printk("pmd=0x%16lx, addr=0x%16lx, end=0x%16lx\n", (unsigned long)pmd, addr, end); -+ -+ // struct page *page = pte_page(*pte); -+ // printk(KERN_ERR "pte=0x%16llx, pte_val=0x%16llx\n", pte, pte_val(*pte)); -+ // debug_printk("free pte table 0x%16llx, pmd=0x%16llx, page=0x%16llx, pmd points to page=0x%16llx\n", pte, pmd, page, pte_page(pmd_pte(*pmd))); -+ // printk(KERN_ERR "pmd_pfn=0x%16llx, pte_pfn=0x%16llx\n", pmd_pfn(*pmd), pte_pfn(*pte)); -+ // pte_free(ko_mm, pte_page(pmd_pte(*pmd))); -+ do { -+ pte = pte_offset_kernel(pmd, addr); -+ debug_printk("pte=0x%16lx, pte_val=0x%16lx\n", (unsigned long)pte, pte_val(*pte)); -+ set_pte(pte, __pte(0)); -+ } while (addr += PAGE_SIZE, addr != end); -+} -+ -+void koi_unmap_pmd_range(struct mm_struct *ko_mm, pud_t *pud, -+ unsigned long addr, unsigned long end) -+{ -+ pmd_t *pmd, *orig_pmd; -+ unsigned long next; -+ if (!pud) -+ return; -+ orig_pmd = pmd_offset(pud, addr); -+ pmd = orig_pmd; -+ debug_printk("pud=0x%16lx, addr=0x%16lx, end=0x%16lx\n", (unsigned long) pud, addr, end); -+ // printk(KERN_ERR "pud_pfn=0x%16llx, pmd_pfn=0x%16llx\n", pud_pfn(*pud), pmd_pfn(*pmd)); -+ do { -+ debug_printk( "pmd=0x%16lx, pmd_val=0x%16lx\n", (unsigned long)pmd, pmd_val(*pmd)); -+ next = pmd_addr_end(addr, end); -+ if (pmd_none(*pmd)) -+ continue; -+ if (pmd_bad(*pmd)) { -+ set_pmd(pmd, __pmd(0)); -+ continue; -+ } -+ koi_unmap_pte_table(ko_mm, pmd, addr, next); -+ } while (pmd++, addr = next, addr != end); -+} -+ -+void koi_unmap_pud_range(struct mm_struct *ko_mm, p4d_t *p4d, -+ unsigned long addr, unsigned long end) -+{ -+ pud_t *pud, *orig_pud; -+ unsigned long next; -+ if (!p4d) -+ return; -+ orig_pud = pud_offset(p4d, addr); -+ pud = orig_pud; -+ debug_printk("p4d=0x%16lx, addr=0x%16lx, end=0x%16lx\n", (unsigned long)p4d, addr, end); -+ do { -+ debug_printk( "pud=0x%16llx, pud_val=0x%16llx\n", pud, pud_val(*pud)); -+ next = pud_addr_end(addr, end); -+ if (pud_none(*pud)) -+ continue; -+ if (pud_bad(*pud)) { -+ set_pud(pud, __pud(0)); -+ continue; -+ } -+ koi_unmap_pmd_range(ko_mm, pud, addr, next); -+ } while (pud++, addr = next, addr != end); -+ debug_printk("free pud 0x%16lx, p4d=0x%16lx, orig_pud=0x%16lx\n", (unsigned long) pud, (unsigned long) p4d, (unsigned long) orig_pud); -+} -+ -+void koi_unmap_p4d_range(struct mm_struct *ko_mm, pgd_t *pgd, -+ unsigned long addr, unsigned long end) -+{ -+ p4d_t *p4d, *orig_p4d; -+ unsigned long next; -+ if (!pgd) -+ return; -+ debug_printk("pgd=0x%16lx, addr=0x%16lx, end=0x%16lx\n", (unsigned long)pgd, addr, end); -+ orig_p4d = p4d_offset(pgd, addr); -+ p4d = orig_p4d; -+ do { -+ next = p4d_addr_end(addr, end); -+ debug_printk( "p4d=0x%16lx, p4d_val=0x%16lx, p4d_none=%d\n", (unsigned long)p4d, p4d_val(*p4d), p4d_none(*p4d)); -+ if (p4d_none_or_clear_bad(p4d)) { -+ continue; -+ } -+ koi_unmap_pud_range(ko_mm, p4d, addr, next); -+ } while (p4d++, addr = next, addr != end); -+} -+ -+void koi_unmap_pagetable(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, -+ unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ pgd_t *pgd = pgd_offset_pgd(ko_pg_dir, addr); -+ debug_printk("freepagetable addr=0x%16lx, end=0x%16lx\n", addr, end); -+ do { -+ next = pgd_addr_end(addr, end); -+ debug_printk( "pgd=0x%16llx, pgd_val=0x%16llx\n", pgd, pgd_val(*pgd)); -+ if (pgd_none_or_clear_bad(pgd)) { -+ continue; -+ } -+ koi_unmap_p4d_range(ko_mm, pgd, addr, next); -+ } while (pgd++, addr = next, addr != end); -+} -+ -+void koi_unmap_mem(struct module *mod, unsigned long addr, unsigned long size) -+{ -+ debug_printk( "koi_unmap_mem addr=0x%16lx, size=0x%16lx\n", addr, size); -+ struct koi_mem_hash_node *target = NULL; -+ if (!addr || ! size) { -+ return; -+ } -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (target == NULL) { -+ printk(KERN_ERR "[KOI UNMAP] mem node for module: %s not found\n", -+ mod->name); -+ return; -+ } -+ koi_unmap_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK); -+ flush_tlb_kernel_range(addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK); -+} -+EXPORT_SYMBOL(koi_unmap_mem); -+ -+void koi_remove_pte_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, pmd_t *pmd) -+{ -+ pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); -+ debug_printk( -+ "pte=0x%16lx, page=0x%16lx, pmd=0x%16lx, pmd_val=0x%16lx\n", -+ (unsigned long)pte, (unsigned long)pmd_page(*pmd), (unsigned long)pmd, (unsigned long)pmd_val(*pmd)); -+ debug_printk( "free orig_pte=0x%16lx\n", (unsigned long)pte); -+ pte_free_kernel(ko_mm, pte); -+} -+ -+void koi_remove_pmd_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, pud_t *pud) -+{ -+ pmd_t *orig_pmd = pud_pgtable(*pud); -+ pmd_t *pmd; -+ int i; -+ for (i = 0; i < PTRS_PER_PMD; i++) { -+ pmd = orig_pmd + i; -+ if (!pmd_present(*pmd) || pmd_bad(*pmd)) -+ continue; -+ debug_printk("pmd=0x%16lx, pmd_val=0x%16lx\n", (unsigned long)pmd, pmd_val(*pmd)); -+ koi_remove_pte_range(ko_mm, ko_pg_dir, pmd); -+ } -+ debug_printk( "free pmd=0x%16lx, page=0x%16lx\n",(unsigned long) orig_pmd, (unsigned long) pud_page(*pud)); -+ -+ pmd_free(ko_mm, orig_pmd); -+} -+ -+void koi_remove_pud_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, p4d_t *p4d) -+{ -+ pud_t *orig_pud = p4d_pgtable(*p4d); -+ pud_t *pud; -+ int i; -+ for (i = 0; i < PTRS_PER_PUD; i++) { -+ pud = orig_pud + i; -+ debug_printk( "p4d=0x%16lx, pud=0x%16lx, orig_pud=0x%16lx\n", (unsigned long)p4d, (unsigned long)pud, (unsigned long)orig_pud); -+ if (!pud_present(*pud) || pud_bad(*pud)) -+ continue; -+ debug_printk("pud=0x%16lx, pud_val=0x%16lx\n", (unsigned long)pud, pud_val(*pud)); -+ koi_remove_pmd_range(ko_mm, ko_pg_dir, pud); -+ } -+ debug_printk( "free pud=0x%16lx, page=0x%16lx\n", (unsigned long)orig_pud, (unsigned long) p4d_pgtable(*p4d)); -+ // free pud page dir -+ pud_free(ko_mm, orig_pud); -+} -+ -+void koi_remove_p4d_range(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, pgd_t *pgd) -+{ -+ p4d_t *orig_p4d = __va(pgd_val(*pgd) & PTE_PFN_MASK); -+ p4d_t *p4d; -+ int i; -+ for (i = 0; i < PTRS_PER_PGD; i++) { -+ p4d = orig_p4d + i; -+ if (!p4d_present(*p4d) || p4d_bad(*p4d)) -+ continue; -+ debug_printk( "p4d=0x%16lx, p4d_val=0x%16lx\n", (unsigned long)p4d, (unsigned long)p4d_val(*p4d)); -+ koi_remove_pud_range(ko_mm, ko_pg_dir, p4d); -+ } -+ debug_printk( "free orig_p4d=0x%16lx\n", (unsigned long) orig_p4d); -+ -+ // free p4d page dir. -+ p4d_free(ko_mm, orig_p4d); -+} -+ -+void koi_remove_pagetable(struct mm_struct *ko_mm, pgd_t *ko_pg_dir) -+{ -+ pgd_t *pgd; -+ int i; -+ for (i = 0; i < PTRS_PER_PGD; i++) { -+ pgd = ko_pg_dir + i; -+ if (pgd_none(*pgd) || pgd_bad(*pgd)) -+ continue; -+ debug_printk("pgd=0x%16lx, pgd_val=0x%16lx\n", (unsigned long)pgd, pgd_val(*pgd)); -+ koi_remove_p4d_range(ko_mm, ko_pg_dir, pgd); -+ } -+ debug_printk( "free pgd=0x%16lx\n", (unsigned long) ko_pg_dir); -+ // free pgd page dir -+ free_pages((unsigned long)ko_pg_dir, 0); -+} -+ -+void koi_destroy_pagetable(struct module *mod) -+{ -+ // int cpu; -+ // unsigned long *ptr; -+ struct koi_mem_hash_node *target = NULL; -+ unsigned long flags; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ printk(KERN_ERR "mem node for module: %s not found, maybe destroyed before?\n", -+ mod->name); -+ return; -+ } -+ spin_lock_irqsave(&target->mod_lock, flags); -+ target->is_valid = false; -+ spin_unlock_irqrestore(&target->mod_lock, flags);; -+ -+ koi_remove_pagetable(target->ko_mm, target->ko_mm->pgd); -+ kfree(target->ko_mm); -+} -+ -+/** -+* koi_cr3_ctor - return ttbr1 for the given driver module -+*/ -+unsigned long koi_cr3_ctor(struct module *mod) -+{ -+ struct koi_mem_hash_node *ko; -+ struct mm_struct *ko_mm; -+ unsigned long cr3; -+ -+ int bkt; -+ rcu_read_lock(); -+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { -+ if (ko->mod == mod) { -+ ko_mm = ko->ko_mm; -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (!ko || !ko_mm) { -+ panic("cannot found module %s in koi_mem_htbl, mod=0x%16lx", -+ mod->name, (unsigned long)mod); -+ return 0; -+ } -+ asm volatile("mov %%cr3,%0\n\t" : "=r" (cr3) : __FORCE_ORDER); -+ this_cpu_write(koi_kern_cr3, cr3); -+ cr3 = (cr3 & X86_CR3_PCID_MASK) | PTI_USER_PCID_MASK | ko->ko_cr3; -+ -+ return cr3; -+} -+EXPORT_SYMBOL(koi_cr3_ctor); -+ -+int koi_share_kstack(struct module *mod) -+{ -+ unsigned long kstack_start, cur_sp; -+ struct koi_mem_hash_node *target = NULL; -+ asm volatile( -+ "mov %%rsp, %0\n" -+ : "=r"(cur_sp) -+ : -+ ); -+ kstack_start = (unsigned long)current->stack; -+ debug_printk("cur_sp=0x%16lx, kstack_start=0x%16lx\n", cur_sp, kstack_start); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ -+ return koi_copy_pagetable(target->ko_mm, target->pgdp, kstack_start, -+ kstack_start + THREAD_SIZE, 0); -+} -+EXPORT_SYMBOL(koi_share_kstack); -+ -+// map the driver stack to kernel -+void koi_map_kostack(struct module *mod) -+{ -+} -+EXPORT_SYMBOL(koi_map_kostack); -+ -+__koi_code void koi_set_upage(struct module *mod, unsigned long addr, unsigned long size) { -+ struct koi_mem_hash_node *target = NULL; -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ -+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, -+ (addr + size + PAGE_SIZE) & PAGE_MASK, _PAGE_USER); -+} -+EXPORT_SYMBOL(koi_set_upage); -+ -+//kzalloc function in driver space -+__koi_code void *koi_kzalloc_wrapper(struct module *mod, size_t size, gfp_t flags) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ panic("mem node for module: %s not found\n", mod->name); -+ } -+ printk(KERN_ERR "kzalloc wrapper\n"); -+ addr = kzalloc(cnt * PAGE_SIZE, flags); -+ printk(KERN_ERR "kzalloc mapping addr=0x%16lx, end=0x%16lx\n", (unsigned long)addr, (unsigned long)addr + PAGE_SIZE * cnt); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ printk(KERN_ERR "kzalloc wrapper return\n"); -+ -+ __koi_switch_to_ko(mod); -+ return addr; -+} -+EXPORT_SYMBOL(koi_kzalloc_wrapper); -+ -+__koi_code void *koi_kzalloc_node_wrapper(struct module *mod, size_t size, gfp_t flags, int node) { -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr = NULL; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu(koi_mem_htbl, target, node, (unsigned long)mod) { -+ if (target->mod == mod) -+ break; -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ panic(KERN_ERR "mem node for module: %s not found\n", mod->name); -+ } -+ addr = kzalloc_node(cnt * PAGE_SIZE, flags, node); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ __koi_switch_to_ko(mod); -+ return (void *)addr; -+} -+EXPORT_SYMBOL(koi_kzalloc_node_wrapper); -+ -+//kmalloc function in driver space -+__koi_code void * -+koi_kmalloc_wrapper(struct module *mod, size_t size, gfp_t flags) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr = NULL; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ panic(KERN_ERR"mem node for module: %s not found\n", mod->name); -+ } -+ -+ addr = kmalloc(cnt * PAGE_SIZE, flags); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ __koi_switch_to_ko(mod); -+ return (void *)addr; -+} -+EXPORT_SYMBOL(koi_kmalloc_wrapper); -+ -+//vmalloc function in driver space -+__koi_code void *koi_vmalloc_wrapper(struct module *mod, -+ unsigned long size) -+{ -+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ panic("mem node for module: %s not found\n", mod->name); -+ } -+ addr = vmalloc(cnt * PAGE_SIZE); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * cnt, 0); -+ __koi_switch_to_ko(mod); -+ return addr; -+} -+EXPORT_SYMBOL(koi_vmalloc_wrapper); -+ -+//kmalloc_array function in driver space -+__koi_code void *koi_kmalloc_array_wrapper(struct module *mod, -+ size_t n, size_t size, -+ gfp_t flags) -+{ -+ int kpage; -+ void *addr; -+ struct koi_mem_hash_node *target = NULL; -+ koi_switch_to_kernel(); -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ if (target == NULL) { -+ panic("mem node for module: %s not found\n", mod->name); -+ } -+ kpage = (n * size + PAGE_SIZE - 1) / PAGE_SIZE; -+ n = (kpage * PAGE_SIZE) / size; -+ addr = kmalloc_array(n, size, flags); -+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, -+ (unsigned long)addr + PAGE_SIZE * kpage, 0); -+ __koi_switch_to_ko(mod); -+ return addr; -+} -+EXPORT_SYMBOL(koi_kmalloc_array_wrapper); -+ -+__koi_code void *koi_kcalloc_wrapper(struct module *mod, size_t n, size_t size, gfp_t flags) { -+ return koi_kmalloc_array_wrapper(mod, n, size, flags | __GFP_ZERO); -+} -+EXPORT_SYMBOL(koi_kcalloc_wrapper); -+ -+#ifndef CONFIG_IEE -+void koi_init_token(struct task_struct *tsk) -+{ -+ struct task_token *token_addr = -+ (struct task_token *)((unsigned long)tsk + KOI_OFFSET); -+ -+ token_addr->koi_kernel_stack = NULL; -+ // token_addr->koi_stack = NULL; -+ // token_addr->koi_stack_base = NULL; -+ token_addr->current_ttbr1 = 0; -+} -+ -+#endif -+ -+ -+#if defined(CONFIG_KOI) && !defined(CONFIG_IEE) -+ -+ -+// mapping the new page to token, whose address is new+KOI_OFFSET. -+ -+static void do_split_huge_pmd(pmd_t* pmdp) -+{ -+ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); -+ int i; -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)((unsigned long)pgtable); -+ for (i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = pmd_pgprot(*pmdp); -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); -+ if (pmd_leaf(READ_ONCE(*pmdp))) { -+ smp_wmb(); -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ pgtable = NULL; -+ } -+ spin_unlock(ptl); -+ if(pgtable) -+ { -+ pte_free_kernel(&init_mm, pgtable); -+ } -+} -+void koi_add_page_mapping(void *token, void *token_page, unsigned int order) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *token_pmdp = pmd_offset(pudp, (unsigned long)token); -+ pte_t *token_ptep; -+ -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_page); -+ p4dp = p4d_offset(pgdp, (unsigned long)token_page); -+ pudp = pud_offset(p4dp, (unsigned long)token_page); -+ pmd_t *token_page_pmdp = pmd_offset(pudp, (unsigned long)token_page); -+ pte_t *token_page_ptep; -+ -+ int use_block_pmd = 0; -+ if (pmd_leaf(*token_pmdp) && order < 9) { -+ do_split_huge_pmd(token_pmdp); -+ do_split_huge_pmd(token_page_pmdp); -+ } else if (pmd_leaf(*token_pmdp)) { -+ use_block_pmd = 1; -+ } -+ -+ if (use_block_pmd) { -+ token_ptep = (pte_t *)token_pmdp; -+ token_page_ptep = (pte_t *)token_page_pmdp; -+ } else { -+ token_ptep = pte_offset_kernel(token_pmdp, (unsigned long)token); -+ token_page_ptep = pte_offset_kernel(token_page_pmdp, (unsigned long)token_page); -+ } -+ -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) & ~__RW) & ~___D & ~__PP); -+ WRITE_ONCE(*pmdp, pmd); -+ pmdp = (pmd_t *)token_ptep; -+ pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(((pmd_val(pmd) & ~PTE_PFN_MASK) | __PP) | (__phys_to_pfn(__pa(token_page)) << PAGE_SHIFT)); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK) | __PP) | (__phys_to_pfn(__pa(token_page) + i * PAGE_SIZE) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte((pte_val(pte) & ~__RW) & ~___D & ~__PP); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)token_page, (unsigned long)(token_page + (PAGE_SIZE * (1 << order)))); -+} -+ -+void koi_remove_page_mapping(unsigned long token, void *__unused, unsigned long order) { -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *token_pmdp = pmd_offset(pudp, (unsigned long)token); -+ pte_t *token_ptep; -+ void *token_page; -+ int use_block_pmd = 0; -+ if (pmd_leaf(*token_pmdp)) { -+ use_block_pmd = 1; -+ token_ptep = (pte_t *)token_pmdp; -+ token_page = page_address(pmd_page(*token_pmdp)); -+ } else { -+ token_ptep = pte_offset_kernel(token_pmdp, (unsigned long)token); -+ token_page = page_address(pte_page(*token_ptep)); -+ } -+ pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_page); -+ p4dp = p4d_offset(pgdp, (unsigned long)token_page); -+ pudp = pud_offset(p4dp, (unsigned long)token_page); -+ pmd_t *token_page_pmdp = pmd_offset(pudp, (unsigned long)token_page); -+ pte_t *token_page_ptep; -+ if (use_block_pmd) { -+ token_page_ptep = (pte_t *)token_page_pmdp; -+ } else { -+ token_page_ptep = pte_offset_kernel(token_page_pmdp, (unsigned long)token_page); -+ } -+ if (use_block_pmd) -+ { -+ pmd_t *pmdp = (pmd_t *)token_page_ptep; -+ pmd_t pmd = READ_ONCE(*pmdp); -+ pmd = __pmd(pmd_val(pmd) | ___D | __RW | __PP); -+ WRITE_ONCE(*pmdp, pmd); -+ pmdp = (pmd_t *)token_ptep; -+ pmd = READ_ONCE(*pmdp); -+ pmd = __pmd((pmd_val(pmd) & ~PTE_PFN_MASK & ~__PP) | (__phys_to_pfn(__pa(token - KOI_OFFSET)) << PAGE_SHIFT)); -+ WRITE_ONCE(*pmdp, pmd); -+ } -+ else -+ { -+ for(int i = 0; i < (0x1 << order); i++) -+ { -+ pte_t pte = READ_ONCE(*token_ptep); -+ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK & ~__PP) | (__phys_to_pfn(__pa(token - KOI_OFFSET) + i * PAGE_SIZE) << PAGE_SHIFT)); -+ WRITE_ONCE(*token_ptep, pte); -+ pte = READ_ONCE(*token_page_ptep); -+ pte = __pte(pte_val(pte) | ___D | __RW | __PP); -+ WRITE_ONCE(*token_page_ptep, pte); -+ token_ptep++; -+ token_page_ptep++; -+ } -+ } -+ free_pages((unsigned long)token_page, order); -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token + (PAGE_SIZE * (1 << order)))); -+ flush_tlb_kernel_range((unsigned long)token_page, (unsigned long)(token_page + (PAGE_SIZE * (1 << order)))); -+} -+ -+#endif -\ No newline at end of file -diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c -index adc67f98819a..5eda6c3a5fac 100644 ---- a/arch/x86/kernel/ldt.c -+++ b/arch/x86/kernel/ldt.c -@@ -408,7 +408,11 @@ static void free_ldt_pgtables(struct mm_struct *mm) - */ - tlb_gather_mmu_fullmm(&tlb, mm); - free_pgd_range(&tlb, start, end, start, end); -+ #ifdef CONFIG_PTP -+ iee_tlb_finish_mmu(&tlb); -+ #else - tlb_finish_mmu(&tlb); -+ #endif - #endif - } - -diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c -index bb2af7bcc47d..a5cf20bbbc5f 100644 ---- a/arch/x86/kernel/machine_kexec_64.c -+++ b/arch/x86/kernel/machine_kexec_64.c -@@ -30,6 +30,10 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif -+ - #ifdef CONFIG_ACPI - /* - * Used while adding mapping for ACPI tables. -@@ -139,12 +143,27 @@ map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p) + #endif /* __ASM_PGTABLE_H */ +diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h +index 2c29239d05c3..1c3489ff2277 100644 +--- a/arch/arm64/include/asm/tlb.h ++++ b/arch/arm64/include/asm/tlb.h +@@ -13,7 +13,11 @@ - static void free_transition_pgtable(struct kimage *image) + static inline void __tlb_remove_table(void *_table) { + #ifdef CONFIG_PTP -+ unset_iee_page((unsigned long)image->arch.p4d, 0); -+ #endif - free_page((unsigned long)image->arch.p4d); - image->arch.p4d = NULL; -+ -+ #ifdef CONFIG_PTP -+ unset_iee_page((unsigned long)image->arch.pud, 0); -+ #endif - free_page((unsigned long)image->arch.pud); - image->arch.pud = NULL; -+ -+ #ifdef CONFIG_PTP -+ unset_iee_page((unsigned long)image->arch.pmd, 0); -+ #endif - free_page((unsigned long)image->arch.pmd); - image->arch.pmd = NULL; -+ -+ #ifdef CONFIG_PTP -+ unset_iee_page((unsigned long)image->arch.pte, 0); ++ ptp_pg_free(&pg_cache, page_to_virt((struct page *)_table)); ++ #else + free_page_and_swap_cache((struct page *)_table); + #endif - free_page((unsigned long)image->arch.pte); - image->arch.pte = NULL; } -@@ -166,6 +185,9 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) - p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); - if (!p4d) - goto err; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)p4d, 0); -+ #endif - image->arch.p4d = p4d; - set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE)); - } -@@ -174,6 +196,9 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) - pud = (pud_t *)get_zeroed_page(GFP_KERNEL); - if (!pud) - goto err; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pud, 0); -+ #endif - image->arch.pud = pud; - set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); - } -@@ -182,6 +207,9 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) - pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL); - if (!pmd) - goto err; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pmd, 0); -+ #endif - image->arch.pmd = pmd; - set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE)); - } -@@ -190,6 +218,9 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) - pte = (pte_t *)get_zeroed_page(GFP_KERNEL); - if (!pte) - goto err; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pte, 0); -+ #endif - image->arch.pte = pte; - set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE)); - } -@@ -216,6 +247,10 @@ static void *alloc_pgt_page(void *data) - clear_page(p); - } -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)page_to_virt(page), 0); -+ #endif -+ - return p; - } + #define tlb_flush tlb_flush +diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile +index d67126570cb2..8f430e98622d 100644 +--- a/arch/arm64/kernel/Makefile ++++ b/arch/arm64/kernel/Makefile +@@ -36,6 +36,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ + syscall.o proton-pack.o idreg-override.o idle.o \ + patching.o rsi.o -@@ -234,7 +269,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) ++obj-$(CONFIG_IEE) += haoc/ + obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \ + sys_compat.o + obj-$(CONFIG_AARCH32_EL0) += sigreturn32.o +diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c +index 87ac0b9c0b4f..f543581fdcd1 100644 +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -17,6 +17,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif - level4p = (pgd_t *)__va(start_pgtable); - clear_page(level4p); -- -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)level4p, 0); -+ #endif -+ - if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { - info.page_flag |= _PAGE_ENC; - info.kernpg_flag |= _PAGE_ENC; -diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c -index 8d51c86caa41..c719f4dc4cdc 100644 ---- a/arch/x86/kernel/paravirt.c -+++ b/arch/x86/kernel/paravirt.c -@@ -81,7 +81,11 @@ void __init native_pv_lock_init(void) + #define CREATE_TRACE_POINTS + #include "trace-events-emulation.h" +@@ -306,11 +309,19 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) - static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) + static int cp15_barrier_set_hw_mode(bool enable) { -+ #ifdef CONFIG_PTP -+ iee_tlb_remove_page(tlb, table); -+ #else - tlb_remove_page(tlb, table); -+ #endif ++ #ifdef CONFIG_IEE_SIP ++ if (enable) ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_EL1_CP15BEN); ++ else ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_CP15BEN, 0); ++ return 0; ++#else + if (enable) + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN); + else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0); + return 0; ++#endif } - unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, -diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c -index bd33c4f7c125..4241d0e712c2 100644 ---- a/arch/x86/kernel/setup.c -+++ b/arch/x86/kernel/setup.c -@@ -57,6 +57,13 @@ - #include - #include + static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn) +@@ -341,11 +352,19 @@ static int setend_set_hw_mode(bool enable) + if (!cpu_supports_mixed_endian_el0()) + return -EINVAL; -+#ifdef CONFIG_IEE -+#include -+#endif -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+void *init_token_page_vaddr; ++#ifdef CONFIG_IEE_SIP ++ if (enable) ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_SED, 0); ++ else ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_EL1_SED); ++ return 0; ++#else + if (enable) + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0); + else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED); + return 0; +#endif -+ - /* - * max_low_pfn_mapped: highest directly mapped pfn < 4 GB - * max_pfn_mapped: highest directly mapped pfn > 4 GB -@@ -837,6 +844,101 @@ static void __init x86_report_nx(void) - } } -+#ifdef CONFIG_KOI -+unsigned long KOI_OFFSET = 0x200000000000; // IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; -+unsigned long koi_offset = 0x200000000000; -+#endif -+ -+#ifdef CONFIG_IEE -+#include -+unsigned long IEE_OFFSET = 0x200000000000; // IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; -+unsigned long iee_offset = 0x200000000000; -+EXPORT_SYMBOL(IEE_OFFSET); -+#ifdef CONFIG_X86_5LEVEL -+void init_iee_offset(void) { -+ if(pgtable_l5_enabled()) { -+ IEE_OFFSET = 0x40000000000000; -+ iee_offset = IEE_OFFSET; -+ } -+} -+#endif /* CONFIG_X86_5LEVEL */ -+ -+void __init iee_set_token_page_valid_pre_init(void *token, void *token_page) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); -+ if (pmd_leaf(*pmdp)) { -+ extern void *alloc_low_pages(unsigned int num); -+ pte_t* pgtable = alloc_low_pages(1); -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)((unsigned long)pgtable); -+ for (int i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = pmd_pgprot(*pmdp); -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+ #ifdef CONFIG_PTP -+ iee_pmd_populate_kernel_pre_init(&init_mm, pmdp, pgtable); -+ #else -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ #endif -+ } -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK) | __PP) | (__phys_to_pfn(__pa(token_page)) << PAGE_SHIFT)); -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(ptep, pte); -+ #else -+ set_pte(ptep, pte); -+ #endif -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); -+} + static int __a32_setend_handler(struct pt_regs *regs, u32 big_endian) +diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c +index 37b093bf20c0..6aa2a0d234cc 100644 +--- a/arch/arm64/kernel/cpu_errata.c ++++ b/arch/arm64/kernel/cpu_errata.c +@@ -18,6 +18,9 @@ + #include + #include + #endif ++#ifdef CONFIG_IEE_SIP ++#include +#endif -+#ifdef CONFIG_KOI -+#include -+EXPORT_SYMBOL(KOI_OFFSET); -+#ifdef CONFIG_X86_5LEVEL -+void init_koi_offset(void) { -+ if(pgtable_l5_enabled()) { -+ KOI_OFFSET = 0x40000000000000; -+ koi_offset = KOI_OFFSET; -+ } -+} -+#endif /* CONFIG_X86_5LEVEL */ -+ -+ -+void __init koi_set_token_page_valid_pre_init(void *token, void *token_page) -+{ -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); -+ if (pmd_leaf(*pmdp)) { -+ extern void *alloc_low_pages(unsigned int num); -+ pte_t* pgtable = alloc_low_pages(1); -+ struct page *page = pmd_page(*pmdp); -+ pte_t *ptep = (pte_t *)((unsigned long)pgtable); -+ for (int i = 0; i < PMD_SIZE / PAGE_SIZE; i++, ptep++) { -+ pte_t entry; -+ pgprot_t pgprot = pmd_pgprot(*pmdp); -+ entry = mk_pte(page + i, pgprot); -+ WRITE_ONCE(*ptep, entry); -+ } -+ pmd_populate_kernel(&init_mm, pmdp, pgtable); -+ } -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); -+ pte_t pte = READ_ONCE(*ptep); -+ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK) | __PP) | (__phys_to_pfn(__pa(token_page)) << PAGE_SHIFT)); -+ set_pte(ptep, pte); -+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); -+} -+#endif /* CONFIG_KOI */ -+ - /* - * Determine if we were loaded by an EFI loader. If so, then we have also been - * passed the efi memmap, systab, etc., so we should use these data structures -@@ -1078,6 +1180,13 @@ void __init setup_arch(char **cmdline_p) - * Define random base addresses for memory sections after max_pfn is - * defined and before each memory section base is used. - */ -+ #if defined(CONFIG_IEE) && defined(CONFIG_X86_5LEVEL) -+ init_iee_offset(); -+ #else -+ #if defined(CONFIG_KOI) && defined(CONFIG_X86_5LEVEL) -+ init_koi_offset(); -+ #endif -+ #endif - kernel_randomize_memory(); - - #ifdef CONFIG_X86_32 -@@ -1163,6 +1272,70 @@ void __init setup_arch(char **cmdline_p) - init_mem_mapping(); - -+ #ifdef CONFIG_IEE -+ printk(KERN_ERR "init_iee_mapping begin ...\n"); -+ init_iee_mapping(); -+ printk(KERN_ERR "init_iee_mapping done ...\n"); -+ -+ printk(KERN_ERR "init token begin ...\n"); -+ // Change init_task image va to Logival VA -+ unsigned long init_task_la = (unsigned long)__va(__pa_symbol(&init_task)); -+ raw_cpu_write(pcpu_hot.current_task, (struct task_struct *)init_task_la); -+ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask); -+ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev))); -+ init_task.children.next = (__va(__pa_symbol(init_task.children.next))); -+ -+ void *new; -+ void *init_token; -+ struct task_token *token; -+ -+ // Alloc a page for init_token. -+ extern void *alloc_low_pages(unsigned int num); -+ new = alloc_low_pages(1); -+ init_token_page_vaddr = new; -+ init_token = (void *)__phys_to_iee(__pa_symbol(&init_task)); -+ // Use lm to write token before IEE initialized. -+ token = (struct task_token *)((unsigned long)new + (((unsigned long)&init_task) & ~PAGE_MASK)); -+ token->pgd = NULL; -+ token->iee_stack = (void *)__phys_to_iee(__pa_symbol(init_iee_stack_end)); -+ token->valid = true; -+ iee_set_token_page_valid_pre_init(init_token, new); -+ printk(KERN_ERR "init token end ...\n"); -+ #else -+ #ifdef CONFIG_KOI -+ printk(KERN_ERR "init_iee_mapping begin ...\n"); -+ init_koi_mapping(); -+ printk(KERN_ERR "init_iee_mapping done ...\n"); -+ -+ // Change init_task image va to Logival VA -+ unsigned long init_task_la = (unsigned long)__va(__pa_symbol(&init_task)); -+ raw_cpu_write(pcpu_hot.current_task, (struct task_struct *)init_task_la); -+ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask); -+ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev))); -+ init_task.children.next = (__va(__pa_symbol(init_task.children.next))); -+ -+ void *new; -+ void *init_token; -+ struct task_token *token; -+ -+ // Alloc a page for init_token. -+ extern void *alloc_low_pages(unsigned int num); -+ new = alloc_low_pages(1); -+ init_token_page_vaddr = new; -+ init_token = (void *)__phys_to_koi(__pa_symbol(&init_task)); -+ // Use lm to write token before IEE initialized. -+ token = (struct task_token *)((unsigned long)new + (((unsigned long)&init_task) & ~PAGE_MASK)); -+ koi_set_token_page_valid_pre_init(init_token, new); -+ printk(KERN_ERR "init token end ...\n"); -+ #endif -+ #endif /* CONFIG_IEE*/ -+ -+ #ifdef CONFIG_PTP -+ printk(KERN_ERR "mapping page table into iee begin ...\n"); -+ init_iee(); -+ printk(KERN_ERR "mapping page table into iee done ...\n"); -+ #endif -+ - idt_setup_early_pf(); - - /* -@@ -1355,3 +1528,4 @@ static int __init register_kernel_offset_dumper(void) - return 0; + static u64 target_impl_cpu_num; + static struct target_impl_cpu *target_impl_cpus; +@@ -143,7 +146,11 @@ hisilicon_1980005_enable(const struct arm64_cpu_capabilities *__unused) + __set_bit(ARM64_HAS_CACHE_IDC, system_cpucaps); + arm64_ftr_reg_ctrel0.sys_val |= BIT(CTR_EL0_IDC_SHIFT); + arm64_ftr_reg_ctrel0.strict_mask &= ~BIT(CTR_EL0_IDC_SHIFT); ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_UCT, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); ++#endif } - __initcall(register_kernel_offset_dumper); -+ -diff --git a/arch/x86/kernel/sfi_bpf_arch.c b/arch/x86/kernel/sfi_bpf_arch.c -new file mode 100644 -index 000000000000..0021c3ed36f6 ---- /dev/null -+++ b/arch/x86/kernel/sfi_bpf_arch.c -@@ -0,0 +1,85 @@ -+#include -+#include -+#include -+ -+pte_t *bpf_sfi_get_ptep(u64 addr) -+{ -+ pgd_t *pgdp; -+ p4d_t *p4dp; -+ pud_t *pudp; -+ pmd_t *pmdp; -+ pte_t *ptep; -+ -+ /* DEBUG check pgd */ -+ // u64 ttbr1_phy, ttbr1_vir; -+ // __asm__ volatile( -+ // "mrs %0, ttbr1_el1\n\t" -+ // : "=r" (ttbr1_phy) -+ // :: -+ // ); -+ // pr_debug("phy = 0x%llx, after mask = 0x%llx\n", ttbr1_phy, (u64)(ttbr1_phy << 16) >> 16); -+ // ttbr1_vir = (u64)__phys_to_kimg((u64)(ttbr1_phy << 16) >> 16); -+ // pr_info("1, ttbr1_vir = 0x%llx, \n", ttbr1_vir); -+ // pr_info("2, init_mm.pgd = 0x%llx\n", (u64)init_mm.pgd); -+ // pr_info("3, swapper_pg_dir = 0x%llx\n", (u64)swapper_pg_dir); -+ -+ pgdp = pgd_offset(&init_mm, addr); -+ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { -+ pr_err("get pgdp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ p4dp = p4d_offset(pgdp, addr); -+ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) { -+ pr_err("get p4dp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ /* IMPORTANT judge huge page first, then judge table */ -+ pudp = pud_offset(p4dp, addr); -+ if (pud_huge(*pudp)) { -+ // pud is huge page -+ pr_warn("pud of 0x%llx is huge page", addr); -+ // return (pte_t *)pudp; -+ return ERR_PTR(-ENOTSUPP); -+ } -+ if (pud_none(*pudp) || pud_bad(*pudp)) { -+ pr_err("get pudp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ pmdp = pmd_offset(pudp, addr); -+ if (pmd_huge(*pmdp)) { -+ // pmd is huge page -+ pr_warn("pmd of 0x%llx is huge page", addr); -+ // return (pte_t *)pmdp; -+ return ERR_PTR(-ENOTSUPP); -+ } -+ if (pmd_none(*pmdp) || pmd_bad(*pmdp)) { -+ pr_err("get pmdp of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ ptep = pte_offset_kernel(pmdp, addr); -+ if (!ptep) { -+ pr_err("get ptep of 0x%llx failed\n", addr); -+ return ERR_PTR(-ENOENT); -+ } -+ -+ return ptep; -+} -+ -+// int bpf_sfi_hook_kernel_fault(u64 addr) -+// { -+// pte_t *ptep; -+ -+// ptep = bpf_sfi_get_ptep(addr); -+// if (IS_ERR(ptep)) -+// return PTR_ERR(ptep); -+ -+// if (pte_val(*ptep) & PTE_BPF_SFI_GP) { -+// return true; -+// } -+// else -+// return false; -+// } -\ No newline at end of file -diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S -index fbd34ee394a7..3a60fb1bd88a 100644 ---- a/arch/x86/kernel/vmlinux.lds.S -+++ b/arch/x86/kernel/vmlinux.lds.S -@@ -99,6 +99,60 @@ jiffies = jiffies_64; - #endif -+#ifdef CONFIG_IEE -+#define IEE_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_code_start = .; \ -+ *(.iee.text.header) \ -+ *(.iee.text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_code_end = .; +@@ -195,7 +202,11 @@ cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) + enable_uct_trap = true; + + if (enable_uct_trap) ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_UCT, 0); +#else -+#define IEE_TEXT -+#endif -+#ifdef CONFIG_IEE -+#define IEE_SI_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_text_start = .; \ -+ *(.iee.si_text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_trampoline_si_text_start = .; \ -+ *(.iee.trampoline.si_text) \ -+ __iee_trampoline_si_text_end = .; \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_text_end = .; -+#else -+#define IEE_SI_TEXT + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); +#endif -+#ifdef CONFIG_IEE -+#define IEE_SI_DATA \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_data_start = .; \ -+ *(.iee.si_data) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __iee_si_data_end = .; + } + + #ifdef CONFIG_ARM64_ERRATUM_1463225 +@@ -210,7 +221,11 @@ has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry, + static void __maybe_unused + cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) + { ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_UCI, 0); +#else -+#define IEE_SI_DATA + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); +#endif -+ -+#ifdef CONFIG_CREDP -+ #define CRED_DATA \ -+ . = ALIGN(PAGE_SIZE); \ -+ *(.iee.cred) \ -+ . = ALIGN(PAGE_SIZE); -+#else -+ #define CRED_DATA + } + + #ifdef CONFIG_HISILICON_ERRATUM_HIP08_RU_PREFETCH +diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c +index 414b9d28ecd1..b4c98007dcc9 100644 +--- a/arch/arm64/kernel/cpufeature.c ++++ b/arch/arm64/kernel/cpufeature.c +@@ -95,6 +95,9 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include +#endif -+ -+#ifdef CONFIG_IEE_SELINUX_P -+ #define IEE_SELINUX_DATA \ -+ . = ALIGN(PAGE_SIZE); \ -+ *(.iee.selinux) \ -+ . = ALIGN(PAGE_SIZE); + + /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ + static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly; +@@ -1621,7 +1624,11 @@ static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unu + * value. + */ + if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT))) ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_UCT, 0); +#else -+ #define IEE_SELINUX_DATA + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); +#endif -+ - PHDRS { - text PT_LOAD FLAGS(5); /* R_E */ - data PT_LOAD FLAGS(6); /* RW_ */ -@@ -111,6 +165,17 @@ PHDRS { - note PT_NOTE FLAGS(0); /* ___ */ } -+#ifdef CONFIG_KOI -+#define KOI_TEXT \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_code_start = .; \ -+ *(.koi.text) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_code_end = .; -+#else -+#define KOI_TEXT -+#endif -+ - SECTIONS + static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, +@@ -1822,7 +1829,11 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) + remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); + + if (!cpu) { ++ #ifdef CONFIG_PTP ++ alloc = __va(early_iee_pgtable_alloc(0)); ++ #else + alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); ++ #endif + kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); + kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd); + +@@ -1881,8 +1892,12 @@ static inline void __cpu_enable_hw_dbm(void) { - #ifdef CONFIG_X86_32 -@@ -127,10 +192,12 @@ SECTIONS - _stext = .; - /* bootstrapping code */ - HEAD_TEXT -+ IEE_TEXT - TEXT_TEXT - SCHED_TEXT - LOCK_TEXT - KPROBES_TEXT -+ IEE_SI_TEXT - SOFTIRQENTRY_TEXT - #ifdef CONFIG_RETPOLINE - *(.text..__x86.indirect_thunk) -@@ -151,6 +218,7 @@ SECTIONS - *(.text..__x86.rethunk_safe) - #endif - ALIGN_ENTRY_TEXT_END -+ KOI_TEXT - *(.gnu.warning) + u64 tcr = read_sysreg(tcr_el1) | TCR_HD; - } :text = 0xcccccccc -@@ -181,6 +249,9 @@ SECTIONS - CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES) ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_SET_TCR_EL1, tcr); ++#else + write_sysreg(tcr, tcr_el1); + isb(); ++#endif + local_flush_tlb_all(); + } - DATA_DATA -+ IEE_SI_DATA -+ CRED_DATA -+ IEE_SELINUX_DATA - CONSTRUCTORS +@@ -2064,7 +2079,11 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) + */ + WARN_ON_ONCE(in_interrupt()); - /* rarely changed data like cpu maps */ -@@ -410,6 +481,13 @@ SECTIONS - __bss_stop = .; - } ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_SPAN, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); ++#endif + set_pstate_pan(1); + } + #endif /* CONFIG_ARM64_PAN */ +@@ -2129,7 +2148,11 @@ static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, + static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap) + { + if (this_cpu_has_cap(ARM64_HAS_E0PD)) ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(TCR_EL1, 0, TCR_E0PD1); ++#else + sysreg_clear_set(tcr_el1, 0, TCR_E0PD1); ++#endif + } + #endif /* CONFIG_ARM64_E0PD */ -+#ifdef CONFIG_IEE -+ . = ALIGN(PAGE_SIZE*4); -+ init_iee_stack_begin = .; -+ . += PAGE_SIZE*4; -+ init_iee_stack_end = .; +@@ -2258,8 +2281,12 @@ static void nmi_enable(const struct arm64_cpu_capabilities *__unused) + * avoid leaving things masked. + */ + _allint_clear(); ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); + isb(); +#endif -+ - /* - * The memory occupied from _text to here, __end_of_kernel_reserve, is - * automatically reserved in setup_arch(). Anything after here must be -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index e568b64a2b6b..3320f4a9fe5b 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -272,6 +272,9 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) - - spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - spinlock_t *pgt_lock; + } + #endif - /* the pgt_lock only for Xen */ -diff --git a/arch/x86/mm/ident_map_for_iee.c b/arch/x86/mm/ident_map_for_iee.c -new file mode 100644 -index 000000000000..8b1f1ea52ec4 ---- /dev/null -+++ b/arch/x86/mm/ident_map_for_iee.c -@@ -0,0 +1,197 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Helper routines for building identity mapping page tables. This is -+ * included by both the compressed kernel and the regular kernel. -+ */ -+ -+#ifdef CONFIG_PTP -+static void ident_pmd_init_for_iee(struct x86_mapping_info *info, pmd_t *pmd_page, -+ unsigned long addr, unsigned long end) -+{ -+ addr &= PMD_MASK; -+ for (; addr < end; addr += PMD_SIZE) { -+ pmd_t *pmd = pmd_page + pmd_index(addr); -+ -+ if (pmd_present(*pmd)) -+ continue; -+ -+ #ifdef CONFIG_PTP -+ iee_set_pmd_pre_init(pmd, __pmd((addr - info->offset) | info->page_flag)); -+ #else -+ set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); -+ #endif -+ } -+} -+ -+static int ident_pud_init_for_iee(struct x86_mapping_info *info, pud_t *pud_page, -+ unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ -+ for (; addr < end; addr = next) { -+ pud_t *pud = pud_page + pud_index(addr); -+ pmd_t *pmd; -+ -+ next = (addr & PUD_MASK) + PUD_SIZE; -+ if (next > end) -+ next = end; -+ -+ if (info->direct_gbpages) { -+ pud_t pudval; -+ -+ if (pud_present(*pud)) -+ continue; -+ -+ addr &= PUD_MASK; -+ pudval = __pud((addr - info->offset) | info->page_flag); -+ #ifdef CONFIG_PTP -+ iee_set_pud_pre_init(pud, pudval); -+ #else -+ set_pud(pud, pudval); -+ #endif -+ continue; -+ } -+ -+ if (pud_present(*pud)) { -+ pmd = pmd_offset(pud, 0); -+ #ifdef CONFIG_PTP -+ ident_pmd_init_for_iee(info, pmd, addr, next); -+ #else -+ ident_pmd_init(info, pmd, addr, next); -+ #endif -+ continue; -+ } -+ pmd = (pmd_t *)info->alloc_pgt_page(info->context); -+ if (!pmd) -+ return -ENOMEM; -+ #ifdef CONFIG_PTP -+ ident_pmd_init_for_iee(info, pmd, addr, next); -+ #else -+ ident_pmd_init(info, pmd, addr, next); -+ #endif -+ #ifdef CONFIG_PTP -+ iee_set_pud_pre_init(pud, __pud(__pa(pmd) | info->kernpg_flag)); -+ #else -+ set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); -+ #endif -+ } -+ -+ return 0; -+} -+ -+static int ident_p4d_init_for_iee(struct x86_mapping_info *info, p4d_t *p4d_page, -+ unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ int result; -+ -+ for (; addr < end; addr = next) { -+ p4d_t *p4d = p4d_page + p4d_index(addr); -+ pud_t *pud; -+ -+ next = (addr & P4D_MASK) + P4D_SIZE; -+ if (next > end) -+ next = end; -+ -+ if (p4d_present(*p4d)) { -+ pud = pud_offset(p4d, 0); -+ #ifdef CONFIG_PTP -+ result = ident_pud_init_for_iee(info, pud, addr, next); -+ #else -+ result = ident_pud_init(info, pud, addr, next); -+ #endif -+ if (result) -+ return result; -+ -+ continue; -+ } -+ pud = (pud_t *)info->alloc_pgt_page(info->context); -+ if (!pud) -+ return -ENOMEM; -+ -+ #ifdef CONFIG_PTP -+ result = ident_pud_init_for_iee(info, pud, addr, next); -+ #else -+ result = ident_pud_init(info, pud, addr, next); -+ #endif -+ if (result) -+ return result; -+ -+ #ifdef CONFIG_PTP -+ iee_set_p4d_pre_init(p4d, __p4d(__pa(pud) | info->kernpg_flag)); -+ #else -+ set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); -+ #endif -+ } -+ -+ return 0; -+} -+ -+int kernel_ident_mapping_init_for_iee(struct x86_mapping_info *info, pgd_t *pgd_page, -+ unsigned long pstart, unsigned long pend) -+{ -+ unsigned long addr = pstart + info->offset; -+ unsigned long end = pend + info->offset; -+ unsigned long next; -+ int result; -+ -+ /* Set the default pagetable flags if not supplied */ -+ if (!info->kernpg_flag) -+ info->kernpg_flag = _KERNPG_TABLE; -+ -+ /* Filter out unsupported __PAGE_KERNEL_* bits: */ -+ info->kernpg_flag &= __default_kernel_pte_mask; -+ -+ for (; addr < end; addr = next) { -+ pgd_t *pgd = pgd_page + pgd_index(addr); -+ p4d_t *p4d; -+ -+ next = (addr & PGDIR_MASK) + PGDIR_SIZE; -+ if (next > end) -+ next = end; -+ -+ if (pgd_present(*pgd)) { -+ p4d = p4d_offset(pgd, 0); -+ #ifdef CONFIG_PTP -+ result = ident_p4d_init_for_iee(info, p4d, addr, next); -+ #else -+ result = ident_p4d_init(info, p4d, addr, next); -+ #endif -+ if (result) -+ return result; -+ continue; -+ } -+ -+ p4d = (p4d_t *)info->alloc_pgt_page(info->context); -+ if (!p4d) -+ return -ENOMEM; -+ #ifdef CONFIG_PTP -+ result = ident_p4d_init_for_iee(info, p4d, addr, next); -+ #else -+ result = ident_p4d_init(info, p4d, addr, next); -+ #endif -+ if (result) -+ return result; -+ if (pgtable_l5_enabled()) { -+ #ifdef CONFIG_PTP -+ iee_set_pgd_pre_init(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); -+ #else -+ set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); -+ #endif -+ } else { -+ /* -+ * With p4d folded, pgd is equal to p4d. -+ * The pgd entry has to point to the pud page table in this case. -+ */ -+ pud_t *pud = pud_offset(p4d, 0); -+ #ifdef CONFIG_PTP -+ iee_set_pgd_pre_init(pgd, __pgd(__pa(pud) | info->kernpg_flag)); -+ #else -+ set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag)); -+ #endif -+ } -+ } -+ -+ return 0; -+} +@@ -2273,15 +2300,23 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) + * So, be strict and forbid other BRs using other registers to + * jump onto a PACIxSP instruction: + */ ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); ++#else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); + isb(); +#endif -\ No newline at end of file -diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c -index 6215dfa23578..a13a5c41e44c 100644 ---- a/arch/x86/mm/init.c -+++ b/arch/x86/mm/init.c -@@ -28,6 +28,10 @@ - #include - #include + } + #endif /* CONFIG_ARM64_BTI */ -+#ifdef CONFIG_PTP -+#include + #ifdef CONFIG_ARM64_MTE + static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) + { ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); ++#else + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); +#endif -+ - /* - * We need to define the tracepoints somewhere, and tlb.c - * is only compiled when SMP=y. -@@ -252,6 +256,8 @@ static void __init probe_page_size_mask(void) - if (cpu_feature_enabled(X86_FEATURE_PTI)) - __default_kernel_pte_mask &= ~_PAGE_GLOBAL; -+ #ifndef CONFIG_IEE -+ #ifndef CONFIG_KOI - /* Enable 1 GB linear kernel mappings if available: */ - if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { - printk(KERN_INFO "Using GB pages for direct mapping\n"); -@@ -259,6 +265,8 @@ static void __init probe_page_size_mask(void) - } else { - direct_gbpages = 0; - } -+ #endif -+ #endif + mte_cpu_setup(); + +@@ -2326,7 +2361,11 @@ static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, in + + static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) + { ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_EL1_TIDCP); ++#else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP); ++#endif } - /* -@@ -445,6 +453,8 @@ static int __meminit split_mem_range(struct map_range *mr, int nr_range, - } + static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) +@@ -2336,7 +2375,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) - #ifdef CONFIG_X86_64 -+#ifndef CONFIG_IEE -+#ifndef CONFIG_KOI - /* big page (1G) range */ - start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); - end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); -@@ -463,6 +473,8 @@ static int __meminit split_mem_range(struct map_range *mr, int nr_range, - page_size_mask & (1<lockdep_hardirqs = lockdep_hardirqs_enabled(); + +diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S +index cceb4526745f..4dc46251d2fd 100644 +--- a/arch/arm64/kernel/entry.S ++++ b/arch/arm64/kernel/entry.S +@@ -28,6 +28,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE ++#include +#endif + + .macro clear_gp_regs + .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29 +@@ -709,6 +712,9 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) + .endif #endif + kernel_entry \el, \regsize ++#ifdef CONFIG_IEE ++ iee_elr_check ++#endif + mov x0, sp + bl el\el\ht\()_\regsize\()_\label\()_handler + .if \el == 0 +diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c +index 998906b75075..87377fffd740 100644 +--- a/arch/arm64/kernel/fpsimd.c ++++ b/arch/arm64/kernel/fpsimd.c +@@ -44,6 +44,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif + + #define FPEXC_IOF (1 << 0) + #define FPEXC_DZF (1 << 1) +@@ -1309,8 +1312,12 @@ void sme_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) + isb(); - /* tail is not big page (2M) alignment */ -@@ -804,6 +816,653 @@ void __init init_mem_mapping(void) - early_memtest(0, max_pfn_mapped << PAGE_SHIFT); + /* Allow EL0 to access TPIDR2 */ ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_WRITE_sctlr_el1, read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2); ++#else + write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1); + isb(); ++#endif } -+#ifdef CONFIG_IEE -+extern unsigned long IEE_OFFSET; -+static unsigned long __init init_range_memory_mapping_for_iee( -+ unsigned long r_start, -+ unsigned long r_end) + /* +diff --git a/arch/arm64/kernel/haoc/Kconfig b/arch/arm64/kernel/haoc/Kconfig +new file mode 100644 +index 000000000000..2440947d3d80 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/Kconfig +@@ -0,0 +1,49 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# Hardware assisted os compartmentalization(Haoc) configuration ++# ++menu "Hardware Assisted OS Compartmentalization(HAOC)" ++ ++config IEE ++ bool "Isolated Execution Environment Framework(IEE)" ++ help ++ Support for Isolated Execution Environment Framework. Foundation of HAOC. ++ depends on ARM64_4K_PAGES ++ depends on ARM64_VA_BITS_48 ++ def_bool y ++ ++config IEE_SIP ++ bool "Sensitive Instruction Protection for IEE(IEE_SIP)" ++ help ++ Protects critical instructions that may break the isolation of IEE, ++ such as writing system control registers. These instructions would ++ be executated inside IEE. ++ depends on IEE ++ def_bool y ++ ++config IEE_PTRP ++ bool "Pointer Protection for IEE(IEE_PTRP)" ++ help ++ Provide IEE matadata for each process called task_token to allow ++ verifing pointers inside task_struct, like struct cred* that determines ++ the capabilities of a process. ++ Could be a enhancement of other sub-module of HAOC. ++ depends on IEE ++ def_bool y ++ ++config PTP ++ bool "Page Table Protection(PTP)" ++ help ++ Protects page tables by IEE, requring each page table modification call ++ IEE Gate for secure mapping. ++ depends on IEE ++ def_bool y ++ ++config PTP_RESERVE_ORDER ++ depends on PTP ++ int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" ++ range 9 15 ++ default 12 ++ ++ ++endmenu # HAOC +diff --git a/arch/arm64/kernel/haoc/Makefile b/arch/arm64/kernel/haoc/Makefile +new file mode 100644 +index 000000000000..3b38c062f869 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/Makefile +@@ -0,0 +1,13 @@ ++# SPDX-License-Identifier: GPL-2.0 ++obj-y += haoc.o haoc-bitmap.o ++obj-y += iee/ ++obj-$(CONFIG_CREDP) += credp/ ++obj-$(CONFIG_PTP) += ptp/ ++obj-$(CONFIG_KEYP) += keyp/ ++ ++ccflags-$(CONFIG_IEE_SELINUX_P) += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include ++obj-$(CONFIG_IEE_SELINUX_P) += selinuxp/ ++ifdef CONFIG_IEE_SELINUX_P ++FLASK_HDR := $(srctree)/security/selinux/flask.h ++$(obj)/iee-selinuxp.o: $(FLASK_HDR) ++endif +\ No newline at end of file +diff --git a/arch/arm64/kernel/haoc/credp/Makefile b/arch/arm64/kernel/haoc/credp/Makefile +new file mode 100644 +index 000000000000..cc494acaa7a1 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/credp/Makefile +@@ -0,0 +1,3 @@ ++obj-y += credp.o ++ ++ccflags-y += -I$(srctree)/mm +\ No newline at end of file +diff --git a/arch/arm64/kernel/haoc/credp/credp.c b/arch/arm64/kernel/haoc/credp/credp.c +new file mode 100644 +index 000000000000..b46706d044d5 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/credp/credp.c +@@ -0,0 +1,405 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++ ++static inline void iee_verify_cred_type(const struct cred *cred) +{ -+ unsigned long start_pfn, end_pfn; -+ unsigned long mapped_ram_size = 0; -+ int i; -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { -+ u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); -+ u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); -+ if (start >= end) -+ continue; -+ /* -+ * if it is overlapping with brk pgt, we need to -+ * alloc pgt buf from memblock instead. -+ */ -+ can_use_brk_pgt = max(start, (u64)pgt_buf_end<= -+ min(end, (u64)pgt_buf_top<> PAGE_SHIFT; -+ last_start = real_end; -+ /* -+ * We start from the top (end of memory) and go to the bottom. -+ * The memblock_find_in_range() gets us a block of RAM from the -+ * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages -+ * for page table. -+ */ -+ while (last_start > map_start) { -+ unsigned long start; -+ if (last_start > step_size) { -+ start = round_down(last_start - 1, step_size); -+ if (start < map_start) -+ start = map_start; -+ } else -+ start = map_start; -+ mapped_ram_size += init_range_memory_mapping_for_iee(start, -+ last_start); -+ last_start = start; -+ min_pfn_mapped = last_start >> PAGE_SHIFT; -+ if (mapped_ram_size >= step_size) -+ step_size = get_new_step_size(step_size); -+ } -+ if (real_end < map_end) -+ init_range_memory_mapping_for_iee(real_end, map_end); ++ iee_verify_cred_type(cred); ++ iee_verify_cred(cred); ++ cred = __ptr_to_iee(cred); ++ *((struct rcu_head **)(&(cred->rcu.func))) = rcu; +} -+/** -+ * memory_map_bottom_up - Map [map_start, map_end) bottom up -+ * @map_start: start address of the target memory range -+ * @map_end: end address of the target memory range -+ * -+ * This function will setup direct mapping for memory range -+ * [map_start, map_end) in bottom-up. Since we have limited the -+ * bottom-up allocation above the kernel, the page tables will -+ * be allocated just above the kernel and we map the memory -+ * in [map_start, map_end) in bottom-up. -+ */ -+static void __init memory_map_bottom_up_for_iee(unsigned long map_start, -+ unsigned long map_end) -+{ -+ unsigned long next, start; -+ unsigned long mapped_ram_size = 0; -+ /* step_size need to be small so pgt_buf from BRK could cover it */ -+ unsigned long step_size = PMD_SIZE; -+ start = map_start; -+ min_pfn_mapped = start >> PAGE_SHIFT; -+ /* -+ * We start from the bottom (@map_start) and go to the top (@map_end). -+ * The memblock_find_in_range() gets us a block of RAM from the -+ * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages -+ * for page table. -+ */ -+ while (start < map_end) { -+ if (step_size && map_end - start > step_size) { -+ next = round_up(start + 1, step_size); -+ if (next > map_end) -+ next = map_end; -+ } else { -+ next = map_end; -+ } -+ mapped_ram_size += init_range_memory_mapping_for_iee(start, next); -+ start = next; -+ if (mapped_ram_size >= step_size) -+ step_size = get_new_step_size(step_size); -+ } ++ ++void __iee_code _iee_set_cred_security(unsigned long __unused, struct cred *cred, ++ void *security) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->security = security; +} -+unsigned long __ref init_memory_mapping_for_iee(unsigned long start, -+ unsigned long end, pgprot_t prot) -+{ -+ struct map_range mr[NR_RANGE_MR]; -+ unsigned long ret = 0; -+ int nr_range, i; -+ pr_debug("init_memory_mapping_for_iee: [mem %#010lx-%#010lx]\n", -+ start, end - 1); -+ memset(mr, 0, sizeof(mr)); -+ nr_range = split_mem_range(mr, 0, start, end); -+ for (i = 0; i < nr_range; i++) -+ ret = kernel_physical_mapping_init_for_iee(mr[i].start, mr[i].end, -+ 0, -+ prot); -+ -+ add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); -+ return ret >> PAGE_SHIFT; -+} -+void __init init_iee_mapping(void) -+{ -+ unsigned long end; -+#ifdef CONFIG_X86_64 -+ end = max_pfn << PAGE_SHIFT; -+#else -+ end = max_low_pfn << PAGE_SHIFT; -+#endif -+ /* the ISA range is always mapped regardless of memory holes */ -+ init_memory_mapping_for_iee(0, ISA_END_ADDRESS, SET_NG(SET_UPAGE(PAGE_KERNEL))); -+ if(__pa_symbol(_end) > IEE_OFFSET) { -+ panic("Image on too high phys mem.\n"); ++ ++bool __iee_code _iee_set_cred_atomic_op_usage(unsigned long __unused, ++ struct cred *cred, int flag, int nr) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_cred(cred); ++ cred = __ptr_to_iee(cred); ++ switch (flag) { ++ case AT_ADD: { ++ atomic_long_add(nr, &cred->usage); ++ return 0; + } -+ /* -+ * If the allocation is in bottom-up direction, we setup direct mapping -+ * in bottom-up, otherwise we setup direct mapping in top-down. -+ */ -+ if (memblock_bottom_up()) { -+ unsigned long kernel_end = __pa_symbol(_end); -+ -+ /* -+ * we need two separate calls here. This is because we want to -+ * allocate page tables above the kernel. So we first map -+ * [kernel_end, end) to make memory above the kernel be mapped -+ * as soon as possible. And then use page tables allocated above -+ * the kernel to map [ISA_END_ADDRESS, kernel_end). -+ */ -+ -+ printk("memory_map_bottom_up_for_iee...\n"); -+ memory_map_bottom_up_for_iee(kernel_end, end); -+ memory_map_bottom_up_for_iee(ISA_END_ADDRESS, kernel_end); -+ } else { -+ printk("memory_map_top_down_for_iee...\n"); -+ memory_map_top_down_for_iee(ISA_END_ADDRESS, end); ++ case AT_INC_NOT_ZERO: { ++ return atomic_long_inc_not_zero(&cred->usage); + } -+#ifdef CONFIG_X86_64 -+ if (max_pfn > max_low_pfn) { -+ /* can we preserve max_low_pfn ?*/ -+ max_low_pfn = max_pfn; ++ case AT_SUB_AND_TEST: { ++ return atomic_long_sub_and_test(nr, &cred->usage); + } -+#else -+ early_ioremap_page_table_range_init(); -+#endif -+ early_memtest(0, max_pfn_mapped << PAGE_SHIFT); ++ } ++ return 0; +} -+#else -+#ifdef CONFIG_KOI -+static unsigned long __init init_range_memory_mapping_for_koi( -+ unsigned long r_start, -+ unsigned long r_end) ++ ++void __iee_code _iee_set_cred_atomic_set_usage(unsigned long __unused, ++ struct cred *cred, int i) +{ -+ unsigned long start_pfn, end_pfn; -+ unsigned long mapped_ram_size = 0; -+ int i; -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { -+ u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); -+ u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); -+ if (start >= end) -+ continue; -+ /* -+ * if it is overlapping with brk pgt, we need to -+ * alloc pgt buf from memblock instead. -+ */ -+ can_use_brk_pgt = max(start, (u64)pgt_buf_end<= -+ min(end, (u64)pgt_buf_top<usage, i); +} -+static void __init memory_map_top_down_for_koi(unsigned long map_start, -+ unsigned long map_end) ++ ++void __iee_code _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, ++ int non_rcu) +{ -+ unsigned long real_end, last_start; -+ unsigned long step_size; -+ unsigned long addr; -+ unsigned long mapped_ram_size = 0; -+ /* -+ * Systems that have many reserved areas near top of the memory, -+ * e.g. QEMU with less than 1G RAM and EFI enabled, or Xen, will -+ * require lots of 4K mappings which may exhaust pgt_buf. -+ * Start with top-most PMD_SIZE range aligned at PMD_SIZE to ensure -+ * there is enough mapped memory that can be allocated from -+ * memblock. -+ */ -+ addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start, -+ map_end); -+ memblock_phys_free(addr, PMD_SIZE); -+ real_end = addr + PMD_SIZE; -+ /* step_size need to be small so pgt_buf from BRK could cover it */ -+ step_size = PMD_SIZE; -+ // max_pfn_mapped = 0; /* will get exact value next */ -+ min_pfn_mapped = real_end >> PAGE_SHIFT; -+ last_start = real_end; -+ /* -+ * We start from the top (end of memory) and go to the bottom. -+ * The memblock_find_in_range() gets us a block of RAM from the -+ * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages -+ * for page table. -+ */ -+ while (last_start > map_start) { -+ unsigned long start; -+ if (last_start > step_size) { -+ start = round_down(last_start - 1, step_size); -+ if (start < map_start) -+ start = map_start; -+ } else -+ start = map_start; -+ mapped_ram_size += init_range_memory_mapping_for_koi(start, -+ last_start); -+ last_start = start; -+ min_pfn_mapped = last_start >> PAGE_SHIFT; -+ if (mapped_ram_size >= step_size) -+ step_size = get_new_step_size(step_size); -+ } -+ if (real_end < map_end) -+ init_range_memory_mapping_for_koi(real_end, map_end); ++ iee_verify_cred_type(cred); ++ iee_verify_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->non_rcu = non_rcu; +} -+/** -+ * memory_map_bottom_up - Map [map_start, map_end) bottom up -+ * @map_start: start address of the target memory range -+ * @map_end: end address of the target memory range -+ * -+ * This function will setup direct mapping for memory range -+ * [map_start, map_end) in bottom-up. Since we have limited the -+ * bottom-up allocation above the kernel, the page tables will -+ * be allocated just above the kernel and we map the memory -+ * in [map_start, map_end) in bottom-up. -+ */ -+static void __init memory_map_bottom_up_for_koi(unsigned long map_start, -+ unsigned long map_end) -+{ -+ unsigned long next, start; -+ unsigned long mapped_ram_size = 0; -+ /* step_size need to be small so pgt_buf from BRK could cover it */ -+ unsigned long step_size = PMD_SIZE; -+ start = map_start; -+ min_pfn_mapped = start >> PAGE_SHIFT; -+ /* -+ * We start from the bottom (@map_start) and go to the top (@map_end). -+ * The memblock_find_in_range() gets us a block of RAM from the -+ * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages -+ * for page table. -+ */ -+ while (start < map_end) { -+ if (step_size && map_end - start > step_size) { -+ next = round_up(start + 1, step_size); -+ if (next > map_end) -+ next = map_end; -+ } else { -+ next = map_end; -+ } -+ mapped_ram_size += init_range_memory_mapping_for_koi(start, next); -+ start = next; -+ if (mapped_ram_size >= step_size) -+ step_size = get_new_step_size(step_size); -+ } ++ ++void __iee_code _iee_set_cred_session_keyring(unsigned long __unused, ++ struct cred *cred, struct key *session_keyring) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->session_keyring = session_keyring; +} -+unsigned long __ref init_memory_mapping_for_koi(unsigned long start, -+ unsigned long end, pgprot_t prot) -+{ -+ struct map_range mr[NR_RANGE_MR]; -+ unsigned long ret = 0; -+ int nr_range, i; -+ pr_debug("init_memory_mapping_for_koi: [mem %#010lx-%#010lx]\n", -+ start, end - 1); -+ memset(mr, 0, sizeof(mr)); -+ nr_range = split_mem_range(mr, 0, start, end); -+ for (i = 0; i < nr_range; i++) -+ ret = kernel_physical_mapping_init_for_koi(mr[i].start, mr[i].end, -+ mr[i].page_size_mask, -+ prot); -+ -+ add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); -+ return ret >> PAGE_SHIFT; -+} -+void __init init_koi_mapping(void) -+{ -+ unsigned long end; -+ end = max_pfn << PAGE_SHIFT; -+ /* the ISA range is always mapped regardless of memory holes */ -+ init_memory_mapping_for_koi(0, ISA_END_ADDRESS, SET_NG(PAGE_KERNEL)); -+ if(__pa_symbol(_end) > KOI_OFFSET) { -+ panic("Image on too high phys mem.\n"); -+ } -+ /* -+ * If the allocation is in bottom-up direction, we setup direct mapping -+ * in bottom-up, otherwise we setup direct mapping in top-down. -+ */ -+ if (memblock_bottom_up()) { -+ unsigned long kernel_end = __pa_symbol(_end); -+ -+ /* -+ * we need two separate calls here. This is because we want to -+ * allocate page tables above the kernel. So we first map -+ * [kernel_end, end) to make memory above the kernel be mapped -+ * as soon as possible. And then use page tables allocated above -+ * the kernel to map [ISA_END_ADDRESS, kernel_end). -+ */ -+ -+ printk("memory_map_bottom_up_for_iee...\n"); -+ memory_map_bottom_up_for_koi(kernel_end, end); -+ memory_map_bottom_up_for_koi(ISA_END_ADDRESS, kernel_end); -+ } else { -+ printk("memory_map_top_down_for_iee...\n"); -+ memory_map_top_down_for_koi(ISA_END_ADDRESS, end); -+ } -+ if (max_pfn > max_low_pfn) { -+ /* can we preserve max_low_pfn ?*/ -+ max_low_pfn = max_pfn; -+ } -+ early_memtest(0, max_pfn_mapped << PAGE_SHIFT); ++ ++void __iee_code _iee_set_cred_process_keyring(unsigned long __unused, ++ struct cred *cred, struct key *process_keyring) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->process_keyring = process_keyring; +} + -+#endif /* CONFIG_KOI */ -+#endif /* CONFIG_IEE */ ++void __iee_code _iee_set_cred_thread_keyring(unsigned long __unused, ++ struct cred *cred, struct key *thread_keyring) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->thread_keyring = thread_keyring; ++} + -+#ifdef CONFIG_PTP -+extern int DIRECT_MAP_SHIFT; -+void __init set_iee_valid_pre_init(unsigned long addr) {} ++void __iee_code _iee_set_cred_request_key_auth(unsigned long __unused, ++ struct cred *cred, struct key *request_key_auth) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->request_key_auth = request_key_auth; ++} + -+static void __init move_pte_table_into_iee(pmd_t *pmdp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_jit_keyring(unsigned long __unused, ++ struct cred *cred, unsigned char jit_keyring) +{ -+ pmd_t pmd = READ_ONCE(*pmdp); -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__pmd_to_phys(pmd)); -+ // printk("move_pte_table_into_iee:\n"); -+ set_iee_valid_pre_init(iee_addr); ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->jit_keyring = jit_keyring; +} + -+static void __init move_pmd_table_into_iee(pud_t *pudp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_cap_inheritable(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_inheritable) +{ -+ unsigned long next; -+ pud_t pud = READ_ONCE(*pudp); -+ pmd_t *pmdp; -+ pmd_t pmd; ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->cap_inheritable = cap_inheritable; ++} + -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__pud_to_phys(pud)); -+ // printk("move_pmd_table_into_iee:\n"); -+ set_iee_valid_pre_init(iee_addr); ++void __iee_code _iee_set_cred_cap_permitted(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_permitted) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->cap_permitted = cap_permitted; ++} + -+ pmdp = pmd_offset(pudp, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ pmd = READ_ONCE(*pmdp); -+ if(pmd_val(pmd) & _PSE) { -+ continue; -+ } else { -+ move_pte_table_into_iee(pmdp, addr, next); -+ } -+ } while (pmdp++, addr = next, addr != end); ++void __iee_code _iee_set_cred_cap_effective(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_effective) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->cap_effective = cap_effective; +} + -+static void __init move_pud_table_into_iee(p4d_t *p4dp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_cap_bset(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_bset) +{ -+ unsigned long next; -+ p4d_t p4d = READ_ONCE(*p4dp); -+ pud_t *pudp; -+ pud_t pud; ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->cap_bset = cap_bset; ++} + -+ // printk("p4d_phys: 0x%16lx\n", __p4d_to_phys(p4d)); -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__p4d_to_phys(p4d)); -+ // printk("move_pud_table_into_iee:\n"); -+ set_iee_valid_pre_init(iee_addr); ++void __iee_code _iee_set_cred_cap_ambient(unsigned long __unused, ++ struct cred *cred, kernel_cap_t cap_ambient) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->cap_ambient = cap_ambient; ++} + -+ pudp = pud_offset(p4dp, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ pud = READ_ONCE(*pudp); -+ if(pud_val(pud) & _PSE) { -+ // _PSE = 1 means a page, not a table -+ continue; -+ } else { -+ move_pmd_table_into_iee(pudp, addr, next); -+ } -+ } while (pudp++, addr = next, addr != end); ++void __iee_code _iee_set_cred_securebits(unsigned long __unused, ++ struct cred *cred, unsigned int securebits) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->securebits = securebits; ++} ++ ++void __iee_code _iee_set_cred_group_info(unsigned long __unused, ++ struct cred *cred, struct group_info *group_info) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->group_info = group_info; ++} ++ ++void __iee_code _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, ++ struct ucounts *ucounts) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->ucounts = ucounts; ++} ++ ++void __iee_code _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, ++ struct user_namespace *user_ns) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->user_ns = user_ns; +} + -+static void __init move_p4d_table_into_iee(pgd_t *pgdp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_user(unsigned long __unused, struct cred *cred, ++ struct user_struct *user) +{ -+ unsigned long next; -+ pgd_t pgd = READ_ONCE(*pgdp); -+ p4d_t *p4dp; -+ p4d_t p4d; ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->user = user; ++} + -+ // printk("pgdp: 0x%16lx\n", pgd.pgd); -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__pgd_to_phys(pgd)); -+ // printk("move_p4d_table_into_iee:\n"); -+ set_iee_valid_pre_init(iee_addr); ++void __iee_code _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, ++ kgid_t fsgid) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->fsgid = fsgid; ++} + -+ p4dp = p4d_offset(pgdp, addr); -+ do { -+ next = p4d_addr_end(addr, end); -+ p4d = READ_ONCE(*p4dp); -+ /* No 512 GiB huge pages yet */ -+ move_pud_table_into_iee(p4dp, addr, next); -+ } while (p4dp++, addr = next, addr != end); ++void __iee_code _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, ++ kuid_t fsuid) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->fsuid = fsuid; +} + -+static void __init init_iee_for_one_region(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) ++void __iee_code _iee_set_cred_egid(unsigned long __unused, struct cred *cred, ++ kgid_t egid) +{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->egid = egid; ++} + -+ addr = va_start & PAGE_MASK; -+ end = PAGE_ALIGN(va_end); ++void __iee_code _iee_set_cred_euid(unsigned long __unused, struct cred *cred, ++ kuid_t euid) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->euid = euid; ++} + -+ do { -+ // printk("region start va: 0x%16lx\n", addr); -+ next = pgd_addr_end(addr, end); -+ move_p4d_table_into_iee(pgdp, addr, next); -+ } while (pgdp++, addr = next, addr != end); ++void __iee_code _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, ++ kgid_t sgid) ++{ ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->sgid = sgid; +} + -+void __init init_iee(void) ++void __iee_code _iee_set_cred_suid(unsigned long __unused, struct cred *cred, ++ kuid_t suid) +{ -+ unsigned long iee_addr; -+ pgd_t *pgdp; -+ phys_addr_t start, end; -+ u64 i; ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->suid = suid; ++} + -+ // handling 1-level page table swapper_pg_dir -+ pgdp = swapper_pg_dir; -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(swapper_pg_dir)); -+ set_iee_valid_pre_init(iee_addr); -+ #if PGD_ALLOCATION_ORDER == 1 -+ set_iee_valid_pre_init(iee_addr + PAGE_SIZE); -+ #endif -+ -+ #ifdef CONFIG_X86_5LEVEL -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(level4_kernel_pgt)); -+ set_iee_valid_pre_init(iee_addr); -+ #endif ++extern void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); + -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(level3_kernel_pgt)); -+ set_iee_valid_pre_init(iee_addr); ++void __iee_code _iee_copy_cred(unsigned long __unused, struct cred *new) ++{ ++ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); ++ struct cred *_new = __ptr_to_iee(new); ++ /* Get old cred inside IEE is safer. */ ++ const struct cred *old = current_cred(); + -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(level2_kernel_pgt)); -+ set_iee_valid_pre_init(iee_addr); ++ iee_verify_cred_type(new); + -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(level2_fixmap_pgt)); -+ set_iee_valid_pre_init(iee_addr); ++ #ifdef CONFIG_IEE_PTRP ++ iee_verify_cred(); ++ /* Would verify this field in commit_cred. */ ++ struct task_token *token = (struct task_token *)__addr_to_iee(current); ++ token->new_cred = new; ++ #endif + -+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(level1_fixmap_pgt)); -+ for (i = 0; i < FIXMAP_PMD_NUM; i++) { -+ set_iee_valid_pre_init(iee_addr + PAGE_SIZE * i); -+ } ++ _iee_memcpy(0, new, old, sizeof(struct cred)); ++ _new->non_rcu = 0; ++ atomic_long_set(&_new->usage, 1); ++ *(struct rcu_head **)(&(_new->rcu.func)) = rcu; ++ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func); ++} + -+ // handling 2/3/4-level page table for kernel -+ init_iee_for_one_region(pgdp, (unsigned long)_text, (unsigned long)_etext); -+ init_iee_for_one_region(pgdp, (unsigned long)__start_rodata, (unsigned long)__end_rodata); -+ init_iee_for_one_region(pgdp, (unsigned long)_sdata, (unsigned long)_edata); -+ init_iee_for_one_region(pgdp, (unsigned long)__bss_start, (unsigned long)__bss_stop); ++/* Only used to copy privilege creds like init_cred. */ ++void __iee_code _iee_copy_kernel_cred(unsigned long __unused, const struct cred *old, ++ struct cred *new) ++{ ++ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); ++ struct cred *_new = __ptr_to_iee(new); ++ struct task_struct *task = current; + -+ // handling page table for fixmap i.e. FIXADDR_START ~ FIXADDR_TOP -+ // printk("fixmap into iee:\n"); -+ init_iee_for_one_region(pgdp, FIXADDR_START, FIXADDR_TOP); ++ iee_verify_cred_type(new); + -+ // handling page table for %esp fixup stacks -+ // espfix_pud_page in espfix_64.c ++ #ifdef CONFIG_IEE_PTRP ++ iee_verify_cred(); ++ /* Would verify this field in commit_cred. */ ++ struct task_token *token = (struct task_token *)__addr_to_iee(task); ++ token->new_cred = new; ++ #endif + -+ // handling 2/3/4-level page table for logical mem and iee -+ for_each_mem_range(i, &start, &end) { -+ if(start >= end) { -+ break; -+ } -+ init_iee_for_one_region(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); -+ init_iee_for_one_region(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); -+ } ++ _iee_memcpy(0, new, old, sizeof(struct cred)); ++ _new->non_rcu = 0; ++ atomic_long_set(&_new->usage, 1); ++ *(struct rcu_head **)(&(_new->rcu.func)) = rcu; ++ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func); +} + -+static void __init iee_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) ++/* Used only inside copy_creds. */ ++void __iee_code _iee_init_copied_cred(unsigned long __unused, ++ struct task_struct *new_task, struct cred *new) +{ -+ pmd_t pmd = READ_ONCE(*pmdp); -+ unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); -+ iee_set_logical_mem_ro(logical_addr); -+} ++ #ifdef CONFIG_PTRP ++ struct task_token *old_task_token = (struct task_token *)__addr_to_iee(current); ++ struct task_token *new_task_token = (struct task_token *)__addr_to_iee(new_task); + -+static void __init iee_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) -+{ -+ unsigned long next; -+ pud_t pud = READ_ONCE(*pudp); -+ pmd_t *pmdp; -+ pmd_t pmd; -+ unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); -+ iee_set_logical_mem_ro(logical_addr); ++ iee_verify_cred(); ++ if (!old_task_token->valid || !new_task_token->valid) ++ panic("IEE: (%s) Invalid Token.", __func__); + -+ pmdp = pmd_offset(pudp, addr); -+ do { -+ next = pmd_addr_end(addr, end); -+ pmd = READ_ONCE(*pmdp); -+ if (pmd_val(pmd) & _PSE) { -+ continue; -+ } else { -+ iee_set_pte_table_ro(pmdp, addr, next); -+ } -+ } while (pmdp++, addr = next, addr != end); ++ if (old_task_token->new_cred != new) ++ pr_err("IEE: (%s) token error. token new cred 0x%llx, new 0%llx", __func__, ++ (u64)old_task_token->new_cred, (u64)new); ++ /* Update token info of new task by current task token. */ ++ new_task_token->curr_cred = new; ++ old_task_token->new_cred = NULL; ++ #endif ++ iee_verify_cred_type(new); ++ new_task->cred = new_task->real_cred = new; +} + -+static void __init iee_set_pud_table_ro(p4d_t *p4dp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_gid(unsigned long __unused, struct cred *cred, ++ kgid_t gid) +{ -+ unsigned long next; -+ p4d_t p4d = READ_ONCE(*p4dp); -+ pud_t *pudp; -+ pud_t pud; -+ unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ pudp = pud_offset(p4dp, addr); -+ do { -+ next = pud_addr_end(addr, end); -+ pud = READ_ONCE(*pudp); -+ if (pud_val(pud) & _PSE) { -+ // _PSE = 1 means a page, not a table -+ continue; -+ } else { -+ iee_set_pmd_table_ro(pudp, addr, next); -+ } -+ } while (pudp++, addr = next, addr != end); ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->gid = gid; +} + -+static void __init iee_set_p4d_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) ++void __iee_code _iee_set_cred_uid(unsigned long __unused, struct cred *cred, ++ kuid_t uid) +{ -+ unsigned long next; -+ pgd_t pgd = READ_ONCE(*pgdp); -+ p4d_t *p4dp; -+ p4d_t p4d; -+ unsigned long logical_addr = (unsigned long)__va(__pgd_to_phys(pgd)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ p4dp = p4d_offset(pgdp, addr); -+ do { -+ next = p4d_addr_end(addr, end); -+ p4d = READ_ONCE(*p4dp); -+ /* No 512 GiB huge pages yet */ -+ iee_set_pud_table_ro(p4dp, addr, next); -+ } while (p4dp++, addr = next, addr != end); ++ iee_verify_cred_type(cred); ++ iee_verify_update_cred(cred); ++ cred = __ptr_to_iee(cred); ++ cred->uid = uid; +} + -+static void __init iee_mark_pgtable_for_one_region_ro(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) ++void __iee_code _iee_abort_cred(unsigned long __unused, const struct cred *cred) +{ -+ unsigned long addr, end, next; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); -+ -+ addr = va_start & PAGE_MASK; -+ end = PAGE_ALIGN(va_end); ++ iee_verify_cred_type(cred); + -+ do { -+ next = pgd_addr_end(addr, end); -+ iee_set_p4d_table_ro(pgdp, addr, next); -+ } while (pgdp++, addr = next, addr != end); ++ #ifdef CONFIT_IEE_PTRP ++ iee_verify_cred(); ++ struct task_token *token = (struct task_token *)__addr_to_iee(current); ++ token->new_cred = NULL; ++ #endif +} + -+// Mark pgtable outside as RO. -+void __init iee_mark_all_lm_pgtable_ro(void) ++void __iee_code _iee_commit_creds(unsigned long __unused, const struct cred *new) +{ -+ unsigned long logical_addr; -+ phys_addr_t start, end; -+ u64 i; -+ pgd_t *pgdp; -+ -+ // handling 1-level page table swapper_pg_dir -+ pgdp = swapper_pg_dir; -+ iee_set_logical_mem_ro((unsigned long)swapper_pg_dir); -+ logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); -+ iee_set_logical_mem_ro(logical_addr); ++ struct task_struct *task = current; + -+ // handling 2/3/4/5-level page table for kernel -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_text, (unsigned long)_etext); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__start_rodata, (unsigned long)__end_rodata); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_sdata, (unsigned long)_edata); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__bss_start, (unsigned long)__bss_stop); ++ iee_verify_cred_type(new); + -+ // handling 2/3/4/5-level statically allocated page table -+ #ifdef CONFIG_X86_5LEVEL -+ iee_set_logical_mem_ro((unsigned long)level4_kernel_pgt); -+ logical_addr = (unsigned long)__va(__pa_symbol(level4_kernel_pgt)); -+ iee_set_logical_mem_ro(logical_addr); ++ #ifdef CONFIG_IEE_PTRP ++ iee_verify_cred(); ++ struct task_token *token = (struct task_token *)__addr_to_iee(task); ++ if (token->new_cred != new) ++ pr_err("IEE: (%s) Invalid cred 0x%llx.", __func__, (u64)new); ++ /* task->cred shall be updated once. */ ++ token->new_cred = NULL; ++ token->curr_cred = new; + #endif + -+ iee_set_logical_mem_ro((unsigned long)level3_kernel_pgt); -+ logical_addr = (unsigned long)__va(__pa_symbol(level3_kernel_pgt)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ iee_set_logical_mem_ro((unsigned long)level2_kernel_pgt); -+ logical_addr = (unsigned long)__va(__pa_symbol(level2_kernel_pgt)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ iee_set_logical_mem_ro((unsigned long)level2_fixmap_pgt); -+ logical_addr = (unsigned long)__va(__pa_symbol(level2_fixmap_pgt)); -+ iee_set_logical_mem_ro(logical_addr); -+ -+ iee_set_logical_mem_ro((unsigned long)level1_fixmap_pgt); -+ logical_addr = (unsigned long)__va(__pa_symbol(level1_fixmap_pgt)); -+ for (i = 0; i < FIXMAP_PMD_NUM; i++) { -+ iee_set_logical_mem_ro(logical_addr + PAGE_SIZE * i); -+ } -+ -+ // handling 2/3/4-level page table for logical mem and iee -+ for_each_mem_range(i, &start, &end) { -+ if (start >= end) -+ break; -+ /* -+ * The linear map must allow allocation tags reading/writing -+ * if MTE is present. Otherwise, it has the same attributes as -+ * PAGE_KERNEL. -+ */ -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); -+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); -+ } ++ rcu_assign_pointer(task->real_cred, new); ++ rcu_assign_pointer(task->cred, new); +} -+#endif /* CONFIG_PTP */ + - /* - * Initialize an mm_struct to be used during poking and a pointer to be used - * during patching. -diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c -index aa69353da49f..2d9b860fd0a4 100644 ---- a/arch/x86/mm/init_64.c -+++ b/arch/x86/mm/init_64.c -@@ -60,6 +60,10 @@ - - #include "ident_map.c" - -+#ifdef CONFIG_PTP -+#include -+#endif ++void __iee_code _iee_override_creds(unsigned long __unused, struct cred *new) ++{ ++ iee_verify_cred_type(new); + - #define DEFINE_POPULATE(fname, type1, type2, init) \ - static inline void fname##_init(struct mm_struct *mm, \ - type1##_t *arg1, type2##_t *arg2, bool init) \ -@@ -90,6 +94,38 @@ DEFINE_ENTRY(pud, pud, init) - DEFINE_ENTRY(pmd, pmd, init) - DEFINE_ENTRY(pte, pte, init) - -+#ifdef CONFIG_PTP -+#define DEFINE_IEE_POPULATE(fname, type1, type2, init) \ -+static inline void iee_##fname##_init(struct mm_struct *mm, \ -+ type1##_t *arg1, type2##_t *arg2, bool init) \ -+{ \ -+ if (init) \ -+ iee_##fname##_safe_pre_init(mm, arg1, arg2); \ -+ else \ -+ iee_##fname##_pre_init(mm, arg1, arg2); \ -+} -+ -+DEFINE_IEE_POPULATE(p4d_populate, p4d, pud, init) -+DEFINE_IEE_POPULATE(pgd_populate, pgd, p4d, init) -+DEFINE_IEE_POPULATE(pud_populate, pud, pmd, init) -+DEFINE_IEE_POPULATE(pmd_populate_kernel, pmd, pte, init) -+ -+#define DEFINE_IEE_ENTRY(type1, type2, init) \ -+static inline void iee_set_##type1##_init(type1##_t *arg1, \ -+ type2##_t arg2, bool init) \ -+{ \ -+ if (init) \ -+ iee_set_##type1##_safe_pre_init(arg1, arg2); \ -+ else \ -+ iee_set_##type1##_pre_init(arg1, arg2); \ -+} -+ -+DEFINE_IEE_ENTRY(p4d, p4d, init) -+DEFINE_IEE_ENTRY(pud, pud, init) -+DEFINE_IEE_ENTRY(pmd, pmd, init) -+DEFINE_IEE_ENTRY(pte, pte, init) -+#endif -+ - static inline pgprot_t prot_sethuge(pgprot_t prot) - { - WARN_ON_ONCE(pgprot_val(prot) & _PAGE_PAT); -@@ -147,6 +183,9 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) - - spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - pgd_t *pgd; - spinlock_t *pgt_lock; - -@@ -188,6 +227,9 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) - - spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - pgd_t *pgd; - p4d_t *p4d; - spinlock_t *pgt_lock; -@@ -243,6 +285,11 @@ static __ref void *spp_getpage(void) - - pr_debug("spp_getpage %p\n", ptr); - -+ #ifdef CONFIG_PTP -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__pa(ptr)); -+ set_iee_page_valid(iee_addr); ++ #ifdef CONFIG_IEE_PTRP ++ struct task_token *token = (struct task_token *)__addr_to_iee(current); ++ iee_verify_cred(); ++ /* TO DO: Verify input cred.*/ ++ token->curr_cred = new; + #endif + - return ptr; - } - -@@ -442,7 +489,11 @@ void __init cleanup_highmap(void) - if (pmd_none(*pmd)) - continue; - if (vaddr < (unsigned long) _text || vaddr > end) -+ #ifdef CONFIG_PTP -+ iee_set_pmd_pre_init(pmd, __pmd(0)); -+ #else - set_pmd(pmd, __pmd(0)); -+ #endif - } - } - -@@ -470,7 +521,11 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end, - E820_TYPE_RAM) && - !e820__mapped_any(paddr & PAGE_MASK, paddr_next, - E820_TYPE_RESERVED_KERN)) -+ #ifdef CONFIG_PTP -+ iee_set_pte_init(pte, __pte(0), init); -+ #else - set_pte_init(pte, __pte(0), init); -+ #endif - continue; - } - -@@ -490,7 +545,11 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end, - pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr, - pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte); - pages++; -+ #ifdef CONFIG_PTP -+ iee_set_pte_init(pte, pfn_pte(paddr >> PAGE_SHIFT, prot), init); -+ #else - set_pte_init(pte, pfn_pte(paddr >> PAGE_SHIFT, prot), init); -+ #endif - paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE; - } - -@@ -525,7 +584,11 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, - E820_TYPE_RAM) && - !e820__mapped_any(paddr & PMD_MASK, paddr_next, - E820_TYPE_RESERVED_KERN)) -+ #ifdef CONFIG_PTP -+ iee_set_pmd_init(pmd, __pmd(0), init); -+ #else - set_pmd_init(pmd, __pmd(0), init); -+ #endif - continue; - } - -@@ -563,9 +626,15 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, - if (page_size_mask & (1<> PAGE_SHIFT, prot_sethuge(prot)), -+ init); -+ #else - set_pmd_init(pmd, - pfn_pmd(paddr >> PAGE_SHIFT, prot_sethuge(prot)), - init); -+ #endif - spin_unlock(&init_mm.page_table_lock); - paddr_last = paddr_next; - continue; -@@ -575,7 +644,11 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, - paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot, init); - - spin_lock(&init_mm.page_table_lock); -+ #ifdef CONFIG_PTP -+ iee_pmd_populate_kernel_init(&init_mm, pmd, pte, init); -+ #else - pmd_populate_kernel_init(&init_mm, pmd, pte, init); -+ #endif - spin_unlock(&init_mm.page_table_lock); - } - update_page_count(PG_LEVEL_2M, pages); -@@ -612,7 +685,11 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, - E820_TYPE_RAM) && - !e820__mapped_any(paddr & PUD_MASK, paddr_next, - E820_TYPE_RESERVED_KERN)) -+ #ifdef CONFIG_PTP -+ iee_set_pud_init(pud, __pud(0), init); -+ #else - set_pud_init(pud, __pud(0), init); -+ #endif - continue; - } - -@@ -649,9 +726,15 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, - if (page_size_mask & (1<> PAGE_SHIFT, prot_sethuge(prot)), -+ init); -+ #else - set_pud_init(pud, - pfn_pud(paddr >> PAGE_SHIFT, prot_sethuge(prot)), - init); -+ #endif - spin_unlock(&init_mm.page_table_lock); - paddr_last = paddr_next; - continue; -@@ -662,7 +745,11 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, - page_size_mask, prot, init); - - spin_lock(&init_mm.page_table_lock); -+ #ifdef CONFIG_PTP -+ iee_pud_populate_init(&init_mm, pud, pmd, init); -+ #else - pud_populate_init(&init_mm, pud, pmd, init); -+ #endif - spin_unlock(&init_mm.page_table_lock); - } - -@@ -715,7 +802,11 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end, - page_size_mask, prot, init); - - spin_lock(&init_mm.page_table_lock); -+ #ifdef CONFIG_PTP -+ iee_p4d_populate_init(&init_mm, p4d, pud, init); -+ #else - p4d_populate_init(&init_mm, p4d, pud, init); -+ #endif - spin_unlock(&init_mm.page_table_lock); - } - -@@ -757,10 +848,19 @@ __kernel_physical_mapping_init(unsigned long paddr_start, - - spin_lock(&init_mm.page_table_lock); - if (pgtable_l5_enabled()) -+ #ifdef CONFIG_PTP -+ iee_pgd_populate_init(&init_mm, pgd, p4d, init); -+ #else - pgd_populate_init(&init_mm, pgd, p4d, init); -+ #endif - else -+ #ifdef CONFIG_PTP -+ iee_p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), -+ (pud_t *) p4d, init); -+ #else - p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), - (pud_t *) p4d, init); -+ #endif - - spin_unlock(&init_mm.page_table_lock); - pgd_changed = true; -@@ -788,6 +888,118 @@ kernel_physical_mapping_init(unsigned long paddr_start, - page_size_mask, prot, true); - } - -+#ifdef CONFIG_IEE -+static unsigned long __meminit -+__kernel_physical_mapping_init_for_iee(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, -+ pgprot_t prot, bool init) -+{ -+ bool pgd_changed = false; -+ unsigned long vaddr, vaddr_start, vaddr_end, vaddr_next, paddr_last; -+ paddr_last = paddr_end; -+ vaddr = (unsigned long)__phys_to_iee(paddr_start); -+ vaddr_end = (unsigned long)__phys_to_iee(paddr_end); -+ vaddr_start = vaddr; -+ for (; vaddr < vaddr_end; vaddr = vaddr_next) { -+ pgd_t *pgd = pgd_offset_k(vaddr); -+ p4d_t *p4d; -+ vaddr_next = (vaddr & PGDIR_MASK) + PGDIR_SIZE; -+ if (pgd_val(*pgd)) { -+ p4d = (p4d_t *)pgd_page_vaddr(*pgd); -+ paddr_last = phys_p4d_init(p4d, __iee_pa(vaddr), -+ __iee_pa(vaddr_end), -+ page_size_mask, prot, init); -+ continue; -+ } -+ p4d = alloc_low_page(); -+ paddr_last = phys_p4d_init(p4d, __iee_pa(vaddr), -+ __iee_pa(vaddr_end), -+ page_size_mask, prot, init); -+ spin_lock(&init_mm.page_table_lock); -+ if (pgtable_l5_enabled()) -+ #ifdef CONFIG_PTP -+ iee_pgd_populate_init(&init_mm, pgd, p4d, init); -+ #else -+ pgd_populate_init(&init_mm, pgd, p4d, init); -+ #endif -+ else -+ #ifdef CONFIG_PTP -+ iee_p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), -+ (pud_t *) p4d, init); -+ #else -+ p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), -+ (pud_t *) p4d, init); -+ #endif -+ spin_unlock(&init_mm.page_table_lock); -+ pgd_changed = true; -+ } -+ if (pgd_changed) -+ sync_global_pgds(vaddr_start, vaddr_end - 1); -+ return paddr_last; -+} -+unsigned long __meminit -+kernel_physical_mapping_init_for_iee(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, pgprot_t prot) -+{ -+ return __kernel_physical_mapping_init_for_iee(paddr_start, paddr_end, -+ page_size_mask, prot, true); -+} -+#else -+#ifdef CONFIG_KOI -+static unsigned long __meminit -+__kernel_physical_mapping_init_for_koi(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, -+ pgprot_t prot, bool init) -+{ -+ bool pgd_changed = false; -+ unsigned long vaddr, vaddr_start, vaddr_end, vaddr_next, paddr_last; -+ paddr_last = paddr_end; -+ vaddr = (unsigned long)__phys_to_koi(paddr_start); -+ vaddr_end = (unsigned long)__phys_to_koi(paddr_end); -+ vaddr_start = vaddr; -+ for (; vaddr < vaddr_end; vaddr = vaddr_next) { -+ pgd_t *pgd = pgd_offset_k(vaddr); -+ p4d_t *p4d; -+ vaddr_next = (vaddr & PGDIR_MASK) + PGDIR_SIZE; -+ if (pgd_val(*pgd)) { -+ p4d = (p4d_t *)pgd_page_vaddr(*pgd); -+ paddr_last = phys_p4d_init(p4d, __koi_pa(vaddr), -+ __koi_pa(vaddr_end), -+ page_size_mask, prot, init); -+ continue; -+ } -+ p4d = alloc_low_page(); -+ paddr_last = phys_p4d_init(p4d, __koi_pa(vaddr), -+ __koi_pa(vaddr_end), -+ page_size_mask, prot, init); -+ spin_lock(&init_mm.page_table_lock); -+ if (pgtable_l5_enabled()) -+ pgd_populate_init(&init_mm, pgd, p4d, init); -+ else -+ p4d_populate_init(&init_mm, p4d_offset(pgd, vaddr), -+ (pud_t *) p4d, init); -+ spin_unlock(&init_mm.page_table_lock); -+ pgd_changed = true; -+ } -+ if (pgd_changed) -+ sync_global_pgds(vaddr_start, vaddr_end - 1); -+ return paddr_last; ++ rcu_assign_pointer(current->cred, new); +} -+unsigned long __meminit -+kernel_physical_mapping_init_for_koi(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, pgprot_t prot) ++ ++void __iee_code _iee_revert_creds(unsigned long __unused, const struct cred *old) +{ -+ return __kernel_physical_mapping_init_for_koi(paddr_start, paddr_end, -+ page_size_mask, prot, true); ++ iee_verify_cred_type(old); ++ ++ #ifdef CONFIG_IEE_PTRP ++ struct task_token *token = (struct task_token *)__addr_to_iee(current); ++ iee_verify_cred(); ++ /* TO DO: Verify input cred.*/ ++ token->curr_cred = old; ++ #endif ++ ++ rcu_assign_pointer(current->cred, old); +} +diff --git a/arch/arm64/kernel/haoc/haoc-bitmap.c b/arch/arm64/kernel/haoc/haoc-bitmap.c +new file mode 100644 +index 000000000000..f2f83bea6505 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/haoc-bitmap.c +@@ -0,0 +1,213 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+#endif -+#endif /* CONFIG_IEE*/ ++bool haoc_bitmap_ready; + - /* - * This function is similar to kernel_physical_mapping_init() above with the - * exception that it uses set_{pud,pmd}() instead of the set_{pud,pte}_safe() -@@ -811,6 +1023,131 @@ void __init initmem_init(void) - } - #endif - -+#ifdef CONFIG_PTP -+static void * __init iee_ptdesc_alloc_block_zero(unsigned long size, int node, bool is_pgtable) ++static void * __init haoc_bitmap_alloc_block_zero(unsigned long size) +{ -+ // void *p = memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS), -+ // MEMBLOCK_ALLOC_ACCESSIBLE, node); -+ void *p = alloc_low_page(); -+ ++ #ifdef CONFIG_PTP ++ void *p = __va(early_iee_pgtable_alloc(0)); ++ #else ++ void *p = memblock_alloc(size, size); ++ #endif + if (!p) + return NULL; -+ memset(p, 0, size); -+ -+ if (is_pgtable) -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(p))); -+ + return p; +} + -+static pte_t * __init iee_ptdesc_pte_populate(pmd_t *pmd, unsigned long addr, int node) ++static pte_t * __init haoc_bitmap_pte_populate(pmd_t *pmd, unsigned long addr) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); -+ if (pte_none(*pte)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, false); ++ if (pte_none(ptep_get(pte))) { ++ pte_t entry; ++ void *p = __va(early_iee_data_alloc(0)); + if (!p) + return NULL; -+ -+ pte_t entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); ++ ++ entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); ++ #ifdef CONFIG_PTP ++ write_sysreg(read_sysreg(TCR_EL1) | TCR_HPD1 | TCR_A1, tcr_el1); ++ isb(); ++ WRITE_ONCE(*__ptr_to_iee(pte), entry); ++ write_sysreg(read_sysreg(TCR_EL1) & ~(TCR_HPD1 | TCR_A1), tcr_el1); ++ isb(); ++ #else + set_pte_at(&init_mm, addr, pte, entry); ++ #endif + } + return pte; +} + -+static pmd_t * __init iee_ptdesc_pmd_populate(pud_t *pud, unsigned long addr, int node) ++static pmd_t * __init haoc_bitmap_pmd_populate(pud_t *pud, unsigned long addr) +{ + pmd_t *pmd = pmd_offset(pud, addr); ++ void *p; ++ + if (pmd_none(*pmd)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); ++ p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); @@ -19901,35 +4382,43 @@ index aa69353da49f..2d9b860fd0a4 100644 + return pmd; +} + -+static pud_t * __init iee_ptdesc_pud_populate(p4d_t *p4d, unsigned long addr, int node) ++static pud_t * __init haoc_bitmap_pud_populate(p4d_t *p4d, unsigned long addr) +{ + pud_t *pud = pud_offset(p4d, addr); ++ void *p; ++ + if (pud_none(*pud)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); ++ p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; ++ pmd_init(p); + pud_populate(&init_mm, pud, p); + } + return pud; +} + -+static p4d_t * __init iee_ptdesc_p4d_populate(pgd_t *pgd, unsigned long addr, int node) ++static p4d_t * __init haoc_bitmap_p4d_populate(pgd_t *pgd, unsigned long addr) +{ + p4d_t *p4d = p4d_offset(pgd, addr); ++ void *p; ++ + if (p4d_none(*p4d)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); ++ p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; ++ pud_init(p); + p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + -+static pgd_t * __init iee_ptdesc_pgd_populate(unsigned long addr, int node) ++static pgd_t * __init haoc_bitmap_pgd_populate(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); ++ void *p; ++ + if (pgd_none(*pgd)) { -+ void *p = iee_ptdesc_alloc_block_zero(PAGE_SIZE, node, true); ++ p = haoc_bitmap_alloc_block_zero(PAGE_SIZE); + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); @@ -19937,13651 +4426,8313 @@ index aa69353da49f..2d9b860fd0a4 100644 + return pgd; +} + -+static pte_t * __init iee_ptdesc_populate(unsigned long addr) ++/* Create mappings if that address is not mapped. */ ++static pte_t * __init haoc_bitmap_populate_address(unsigned long addr) +{ + pgd_t *pgd; -+ p4d_t *p4d; -+ pud_t *pud; -+ pmd_t *pmd; -+ pte_t *pte; -+ int node = NUMA_NO_NODE; ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ pte_t *pte; + -+ pgd = iee_ptdesc_pgd_populate(addr, node); ++ pgd = haoc_bitmap_pgd_populate(addr); + if (!pgd) + return NULL; -+ p4d = iee_ptdesc_p4d_populate(pgd, addr, node); ++ p4d = haoc_bitmap_p4d_populate(pgd, addr); + if (!p4d) + return NULL; -+ pud = iee_ptdesc_pud_populate(p4d, addr, node); ++ pud = haoc_bitmap_pud_populate(p4d, addr); + if (!pud) + return NULL; -+ pmd = iee_ptdesc_pmd_populate(pud, addr, node); ++ pmd = haoc_bitmap_pmd_populate(pud, addr); + if (!pmd) + return NULL; -+ pte = iee_ptdesc_pte_populate(pmd, addr, node); ++ pte = haoc_bitmap_pte_populate(pmd, addr); ++ if (!pte) ++ return NULL; ++ + return pte; +} + -+int __init iee_ptdesc_sparse_init(void) ++/* Map haoc bitmap array after vmemmap region. */ ++int __init haoc_bitmap_sparse_init(void) +{ + unsigned long start_pfn, end_pfn; + int i, nid; ++ /* Iterate through available memory blocks. */ + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { -+ // pr_err("IEE: start_pfn, end_pfn = %lx, %lx", start_pfn, end_pfn); -+ u64 ptdesc_start = ALIGN_DOWN((u64)(__pfn_to_ptdesc(start_pfn)), PAGE_SIZE); -+ u64 end = ALIGN_DOWN((u64)(__pfn_to_ptdesc(end_pfn)), PAGE_SIZE); -+ unsigned long addr = ptdesc_start; ++ u64 bitmap_start = ALIGN_DOWN(((u64)__pfn_to_haoc_bitmap(start_pfn)), ++ PAGE_SIZE); ++ u64 bitmap_end = ALIGN(((u64)__pfn_to_haoc_bitmap(end_pfn)), ++ PAGE_SIZE); ++ u64 addr = bitmap_start; + pte_t *pte; -+ for (; addr < end; addr += PAGE_SIZE) { -+ // pr_err("IEE: addr = %lx", addr); -+ pte = iee_ptdesc_populate(addr); ++ ++ for (; addr < bitmap_end; addr += PAGE_SIZE) { ++ pte = haoc_bitmap_populate_address(addr); + if (!pte) -+ return -ENOMEM; ++ pr_err("HAOC: failed on bitmap init."); + } + } ++ haoc_bitmap_ready = true; + return 0; +} -+#endif + - void __init paging_init(void) - { - sparse_init(); -@@ -824,6 +1161,10 @@ void __init paging_init(void) - node_clear_state(0, N_MEMORY); - node_clear_state(0, N_NORMAL_MEMORY); - ++void __init haoc_bitmap_setup(void) ++{ ++ /* Setup bitmap types of global data. */ + #ifdef CONFIG_PTP -+ iee_ptdesc_sparse_init(); ++ { ++ unsigned long start, end, num_pages; ++ ++ start = (unsigned long)idmap_pg_dir; ++ end = (unsigned long)__iee_ptp_data_end; ++ num_pages = (end - start) / PAGE_SIZE; ++ iee_set_bitmap_type(start, num_pages, IEE_PGTABLE); ++ } + #endif ++ #ifdef CONFIG_CREDP ++ { ++ unsigned long start, end, num_pages; + - zone_sizes_init(); - } - -diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c -index aa7d279321ea..dc0f4f13a0c6 100644 ---- a/arch/x86/mm/ioremap.c -+++ b/arch/x86/mm/ioremap.c -@@ -888,7 +888,11 @@ void __init early_ioremap_init(void) - - pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); - memset(bm_pte, 0, sizeof(bm_pte)); -+ #ifdef CONFIG_PTP -+ iee_pmd_populate_kernel_pre_init(&init_mm, pmd, bm_pte); -+ #else - pmd_populate_kernel(&init_mm, pmd, bm_pte); ++ start = (unsigned long)__iee_cred_data_start; ++ end = (unsigned long)__iee_cred_data_end; ++ num_pages = (end - start) / PAGE_SIZE; ++ iee_set_bitmap_type(start, num_pages, IEE_CRED); ++ } + #endif - - /* - * The boot-ioremap range spans multiple pmds, for which -@@ -929,8 +933,41 @@ void __init __early_set_fixmap(enum fixed_addresses idx, - pgprot_val(flags) &= __supported_pte_mask; - - if (pgprot_val(flags)) -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ #else - set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ #endif - else -+ #ifdef CONFIG_PTP -+ iee_set_pte_pre_init(pte, __pte(0)); -+ #else - pte_clear(&init_mm, addr, pte); -+ #endif - flush_tlb_one_kernel(addr); - } ++ #ifdef CONFIG_IEE_SELINUX_P ++ { ++ unsigned long start, end, num_pages; ++ ++ start = (unsigned long)__iee_selinux_data_start; ++ end = (unsigned long)__iee_selinux_data_end; ++ num_pages = (end - start) / PAGE_SIZE; ++ iee_set_bitmap_type(start, num_pages, IEE_SELINUX); ++ } ++ #endif ++ ++ /* Mark bitmaps of early allocated iee data. */ ++ setup_iee_early_data_bitmap(); ++} ++ ++#pragma GCC push_options ++#pragma GCC optimize("O0") ++static void __iee_code _iee_bitmap_memset(void *ptr, int data, size_t n) ++{ ++ char *_ptr; ++ ++ _ptr = (char *)ptr; ++ ++ while (n--) ++ *_ptr++ = data; ++} ++#pragma GCC pop_options ++ ++void __iee_code _iee_set_bitmap_type(unsigned long __unused, ++ u64 va, enum HAOC_BITMAP_TYPE type, int num_pages) ++{ ++ // _iee_bitmap_memset(__va_to_haoc_bitmap(va), type, num_pages); ++} +diff --git a/arch/arm64/kernel/haoc/haoc.c b/arch/arm64/kernel/haoc/haoc.c +new file mode 100644 +index 000000000000..a97b9d1ea385 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/haoc.c +@@ -0,0 +1,106 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include + ++typedef void (*iee_func)(void); ++iee_func iee_funcs[] = { ++ (iee_func)_iee_memcpy, ++ (iee_func)_iee_memset, ++ (iee_func)_iee_set_freeptr, ++#ifdef CONFIG_IEE_PTRP ++ (iee_func)_iee_set_token_pgd, ++ (iee_func)_iee_init_token, ++ (iee_func)_iee_invalidate_token, ++ (iee_func)_iee_validate_token, ++#endif ++ (iee_func)_iee_set_bitmap_type, ++#ifdef CONFIG_CREDP ++ (iee_func)_iee_copy_cred, ++ (iee_func)_iee_copy_kernel_cred, ++ (iee_func)_iee_init_copied_cred, ++ (iee_func)_iee_commit_creds, ++ (iee_func)_iee_abort_cred, ++ (iee_func)_iee_override_creds, ++ (iee_func)_iee_revert_creds, ++ (iee_func)_iee_set_cred_uid, ++ (iee_func)_iee_set_cred_gid, ++ (iee_func)_iee_set_cred_suid, ++ (iee_func)_iee_set_cred_sgid, ++ (iee_func)_iee_set_cred_euid, ++ (iee_func)_iee_set_cred_egid, ++ (iee_func)_iee_set_cred_fsuid, ++ (iee_func)_iee_set_cred_fsgid, ++ (iee_func)_iee_set_cred_user, ++ (iee_func)_iee_set_cred_user_ns, ++ (iee_func)_iee_set_cred_group_info, ++ (iee_func)_iee_set_cred_securebits, ++ (iee_func)_iee_set_cred_cap_inheritable, ++ (iee_func)_iee_set_cred_cap_permitted, ++ (iee_func)_iee_set_cred_cap_effective, ++ (iee_func)_iee_set_cred_cap_bset, ++ (iee_func)_iee_set_cred_cap_ambient, ++ (iee_func)_iee_set_cred_jit_keyring, ++ (iee_func)_iee_set_cred_session_keyring, ++ (iee_func)_iee_set_cred_process_keyring, ++ (iee_func)_iee_set_cred_thread_keyring, ++ (iee_func)_iee_set_cred_request_key_auth, ++ (iee_func)_iee_set_cred_non_rcu, ++ (iee_func)_iee_set_cred_atomic_set_usage, ++ (iee_func)_iee_set_cred_atomic_op_usage, ++ (iee_func)_iee_set_cred_security, ++ (iee_func)_iee_set_cred_rcu, ++ (iee_func)_iee_set_cred_ucounts, ++#endif +#ifdef CONFIG_PTP -+void __init __iee_set_fixmap_pre_init(enum fixed_addresses idx, -+ phys_addr_t phys, pgprot_t flags) ++ (iee_func)_iee_set_static_pgd, ++ (iee_func)_iee_set_bm_pte, ++ (iee_func)_iee_set_pte, ++ (iee_func)_iee_set_pmd, ++ (iee_func)_iee_set_pud, ++ (iee_func)_iee_set_p4d, ++ (iee_func)_iee_set_swapper_pgd, ++ (iee_func)_iee_set_xchg_relaxed, ++ (iee_func)_iee_set_pmd_xchg_relaxed, ++ (iee_func)_iee_set_cmpxchg_relaxed, ++ (iee_func)_iee_set_pmd_cmpxchg_relaxed, ++ (iee_func)_iee_set_sensitive_pte, ++ (iee_func)_iee_unset_sensitive_pte, ++#endif ++#ifdef CONFIG_KEYP ++ (iee_func)_iee_set_key_union, ++ (iee_func)_iee_set_key_struct, ++ (iee_func)_iee_set_key_payload, ++ (iee_func)_iee_set_key_usage, ++ (iee_func)_iee_set_key_serial, ++ (iee_func)_iee_set_key_watchers, ++ (iee_func)_iee_set_key_user, ++ (iee_func)_iee_set_key_security, ++ (iee_func)_iee_set_key_expiry, ++ (iee_func)_iee_set_key_revoked_at, ++ (iee_func)_iee_set_key_last_used_at, ++ (iee_func)_iee_set_key_uid, ++ (iee_func)_iee_set_key_gid, ++ (iee_func)_iee_set_key_perm, ++ (iee_func)_iee_set_key_quotalen, ++ (iee_func)_iee_set_key_datalen, ++ (iee_func)_iee_set_key_state, ++ (iee_func)_iee_set_key_magic, ++ (iee_func)_iee_set_key_flags, ++ (iee_func)_iee_set_key_index_key, ++ (iee_func)_iee_set_key_hash, ++ (iee_func)_iee_set_key_len_desc, ++ (iee_func)_iee_set_key_type, ++ (iee_func)_iee_set_key_domain_tag, ++ (iee_func)_iee_set_key_description, ++ (iee_func)_iee_set_key_restrict_link, ++ (iee_func)_iee_set_key_flag_bit, ++#endif ++#ifdef CONFIG_IEE_SELINUX_P ++ (iee_func)_iee_set_selinux_status_pg, ++ (iee_func)_iee_set_selinux_enforcing, ++ (iee_func)_iee_mark_selinux_initialized, ++ (iee_func)_iee_set_sel_policy_cap, ++ (iee_func)_iee_sel_rcu_assign_policy, ++#endif ++ NULL ++}; +diff --git a/arch/arm64/kernel/haoc/iee/Makefile b/arch/arm64/kernel/haoc/iee/Makefile +new file mode 100644 +index 000000000000..7ac317bc23bd +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/Makefile +@@ -0,0 +1,5 @@ ++# SPDX-License-Identifier: GPL-2.0 ++obj-$(CONFIG_IEE) += iee.o iee-gate.o iee-init.o iee-func.o iee-mmu.o ++obj-$(CONFIG_IEE_SIP) += iee-si-gate.o iee-si.o ++obj-$(CONFIG_IEE_PTRP) += iee-token.o ++ccflags-y += -I$(srctree)/mm +\ No newline at end of file +diff --git a/arch/arm64/kernel/haoc/iee/iee-func.c b/arch/arm64/kernel/haoc/iee/iee-func.c +new file mode 100644 +index 000000000000..51344ba709d3 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee-func.c +@@ -0,0 +1,479 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include "slab.h" ++ ++static inline bool iee_support_pmd_block(unsigned long addr, unsigned int order) +{ -+ unsigned long addr = __fix_to_virt(idx); -+ pte_t *pte; ++ u64 end = addr + (PAGE_SIZE << order); + -+ if (idx >= __end_of_fixed_addresses) { -+ BUG(); ++ return ((addr | end) & ~PMD_MASK) == 0; ++} ++ ++static inline bool iee_support_cont_pte(unsigned long addr, unsigned int order) ++{ ++ u64 end = addr + (PAGE_SIZE << order); ++ ++ return ((addr | end) & ~CONT_PTE_MASK) == 0; ++} ++ ++/* Would clear continuous bits or split pmd block descriptors if needed. */ ++static void iee_may_split_pmd(pud_t *pudp, unsigned long addr, unsigned int order) ++{ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ if (!pmd_leaf(*pmdp)) + return; ++ ++ /* Handling cont mapping for pmd blocks. */ ++ if ((pmd_val(*pmdp) & PTE_CONT)) { ++ /* Get the beginning of cont mapping. */ ++ pmd_t *cont_pmdp = pmd_offset(pudp, addr & CONT_PMD_MASK); ++ ++ for (int i = 0; i < CONT_PMDS; i++) { ++ set_pmd(cont_pmdp, __pmd(pmd_val(*cont_pmdp) & ~PTE_CONT)); ++ cont_pmdp++; ++ } + } -+ pte = early_ioremap_pte(addr); + -+ /* Sanitize 'prot' against any unsupported bits: */ -+ pgprot_val(flags) &= __supported_pte_mask; ++ /* May split Block Descriptor. */ ++ if (!iee_support_pmd_block(addr, order)) { ++ struct page *page = pmd_page(*pmdp); ++ pte_t *pgtable = pte_alloc_one_kernel(&init_mm); + -+ if (pgprot_val(flags)) -+ iee_set_pte_pre_init(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); -+ else -+ iee_set_pte_pre_init(pte, __pte(0)); -+ // pte_clear(&init_mm, addr, pte); -+ flush_tlb_one_kernel(addr); -+} -+#endif -diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c -index 230f1dee4f09..39d227daacfa 100644 ---- a/arch/x86/mm/kaslr.c -+++ b/arch/x86/mm/kaslr.c -@@ -156,6 +156,23 @@ void __init kernel_randomize_memory(void) - vaddr = round_up(vaddr + 1, PUD_SIZE); - remain_entropy -= entropy; - } ++ if (!pgtable) ++ panic("Alloc pgtable error.\n"); + -+ #ifdef CONFIG_IEE -+ extern unsigned long iee_offset; -+ extern unsigned long IEE_OFFSET; -+ iee_offset = *kaslr_regions[0].base - vaddr_start + IEE_OFFSET; -+ #endif /* CONFIG_IEE*/ -+ #ifdef CONFIG_KOI -+ extern unsigned long koi_offset; -+ extern unsigned long KOI_OFFSET; -+ koi_offset = *kaslr_regions[0].base - vaddr_start + KOI_OFFSET; -+ #endif /* CONFIG_KOI*/ ++ { ++ /* Iterate on the new page table. */ ++ pte_t *ptep = pgtable; + -+ #ifdef CONFIG_PTP -+ unsigned long *iee_ptdesc_kaslr_base = &iee_ptdesc_base; -+ *iee_ptdesc_kaslr_base = *kaslr_regions[2].base + (kaslr_regions[2].size_tb << TB_SHIFT); -+ pr_err("IEE: ptdesc base with kaslr: %lx\n", iee_ptdesc_base); -+ #endif - } - - void __meminit init_trampoline_kaslr(void) -diff --git a/arch/x86/mm/mm_internal.h b/arch/x86/mm/mm_internal.h -index 3f37b5c80bb3..2dc3f96fbc26 100644 ---- a/arch/x86/mm/mm_internal.h -+++ b/arch/x86/mm/mm_internal.h -@@ -14,6 +14,15 @@ unsigned long kernel_physical_mapping_init(unsigned long start, - unsigned long end, - unsigned long page_size_mask, - pgprot_t prot); -+#ifdef CONFIG_IEE -+unsigned long kernel_physical_mapping_init_for_iee(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, pgprot_t prot); -+#else -+unsigned long kernel_physical_mapping_init_for_koi(unsigned long paddr_start, -+ unsigned long paddr_end, -+ unsigned long page_size_mask, pgprot_t prot); -+#endif /* CONFIG_IEE*/ - unsigned long kernel_physical_mapping_change(unsigned long start, - unsigned long end, - unsigned long page_size_mask); -diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c -index 2d850f6bae70..067e3195782d 100644 ---- a/arch/x86/mm/pat/set_memory.c -+++ b/arch/x86/mm/pat/set_memory.c -@@ -37,6 +37,10 @@ - - #include "../mm_internal.h" - -+#ifdef CONFIG_PTP -+#include -+#endif ++ /* Try to support continuous mappings on pte. */ ++ for (int i = 0; i < PTRS_PER_PMD; i++, ptep++) { ++ pte_t entry; ++ pgprot_t pgprot = PAGE_KERNEL; + - /* - * The current flushing context - we pass it instead of 5 arguments: - */ -@@ -841,6 +845,9 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) - struct page *page; - - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; -@@ -1151,7 +1158,32 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, - * pagetable protections, the actual ptes set above control the - * primary protection behavior: - */ -+ #ifdef CONFIG_PTP -+ set_pmd((pmd_t *)kpte, __pmd(pte_val(mk_pte(base, __pgprot(_KERNPG_TABLE))))); -+ #ifdef CONFIG_X86_32 -+ if (!SHARED_KERNEL_PMD) { -+ struct page *page; -+ -+ list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif -+ pgd_t *pgd; -+ p4d_t *p4d; -+ pud_t *pud; -+ pmd_t *pmd; -+ -+ pgd = (pgd_t *)page_address(page) + pgd_index(address); -+ p4d = p4d_offset(pgd, address); -+ pud = pud_offset(p4d, address); -+ pmd = pmd_offset(pud, address); -+ set_pmd(pmd, mk_pmd(base, __pgprot(_KERNPG_TABLE))); ++ pgprot = __pgprot(pgprot_val(pgprot) | PTE_CONT); ++ ++ entry = mk_pte(page + i, pgprot); ++ set_pte(ptep, entry); ++ } + } -+ } -+ #endif -+ #else - __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE))); -+ #endif - - /* - * Do a global flush tlb after splitting the large page -@@ -1181,17 +1213,33 @@ static int split_large_page(struct cpa_data *cpa, pte_t *kpte, - unsigned long address) - { - struct page *base; -+ #ifdef CONFIG_PTP -+ pte_t *pbase; -+ #endif - - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); + -+ #ifdef CONFIG_PTP -+ /* without gfp_zero mask, set_pte cannot pass privilege check*/ -+ base = alloc_pages(GFP_KERNEL | __GFP_ZERO, 0); -+ pbase = (pte_t *)page_address(base); -+ set_iee_page((unsigned long)pbase, 0); -+ #else - base = alloc_pages(GFP_KERNEL, 0); -+ #endif ++ /* Ensure that this pmd hasn't be splited by other threads. */ ++ spinlock_t *ptl = pmd_lock(&init_mm, pmdp); + - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - if (!base) - return -ENOMEM; - -- if (__split_large_page(cpa, kpte, address, base)) -+ if (__split_large_page(cpa, kpte, address, base)) { -+ #ifdef CONFIG_PTP -+ unset_iee_page((unsigned long)pbase, 0); -+ #endif - __free_page(base); ++ if (pmd_leaf(READ_ONCE(*pmdp))) { ++ /* for sync. */ ++ smp_wmb(); ++ pmd_populate_kernel(&init_mm, pmdp, pgtable); ++ pgtable = NULL; ++ } ++ spin_unlock(ptl); ++ ++ if (pgtable) ++ pte_free_kernel(&init_mm, pgtable); + } - - return 0; - } -@@ -1204,7 +1252,11 @@ static bool try_to_free_pte_page(pte_t *pte) - if (!pte_none(pte[i])) - return false; - -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page(pte); -+ #else - free_page((unsigned long)pte); -+ #endif - return true; - } - -@@ -1339,6 +1391,10 @@ static int alloc_pte_page(pmd_t *pmd) - if (!pte) - return -1; - -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pte, 0); -+ #endif ++} + - set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE)); - return 0; - } -@@ -1348,6 +1404,10 @@ static int alloc_pmd_page(pud_t *pud) - pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL); - if (!pmd) - return -1; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pmd, 0); -+ #endif - - set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE)); - return 0; -@@ -1542,6 +1602,10 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr) - p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); - if (!p4d) - return -1; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)p4d, 0); -+ #endif - - set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE)); - } -@@ -1554,7 +1618,11 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr) - pud = (pud_t *)get_zeroed_page(GFP_KERNEL); - if (!pud) - return -1; -- -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pud, 0); -+ #endif -+ - set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); - } - -diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c -index 8e1ef5345b7a..5ee71c7282ed 100644 ---- a/arch/x86/mm/pgtable.c -+++ b/arch/x86/mm/pgtable.c -@@ -7,6 +7,12 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#include -+#include -+#endif ++/* ++ * Used to enforce or remove RO protection linear addresses of IEE objects. ++ * Not handling block descriptors except pmd blocks when change page tables, ++ * so DO NOT use larger block on kernel linear mappings. ++ */ ++void iee_set_logical_mem(unsigned long addr, unsigned int order, bool prot) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pmd_t *pmdp; ++ ++ /* Split pmd block if needed. */ ++ iee_may_split_pmd(pudp, addr, order); ++ ++ pmdp = pmd_offset(pudp, addr); ++ ++ if (pmd_leaf(READ_ONCE(*pmdp))) { ++ /* Only permits writing single pmd block right now. */ ++ if (order != (PMD_SHIFT - PAGE_SHIFT)) ++ panic("%s: error on linear mappings.", __func__); + - #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK - phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; - EXPORT_SYMBOL(physical_mask); -@@ -94,14 +100,22 @@ static inline void pgd_list_add(pgd_t *pgd) - { - struct ptdesc *ptdesc = virt_to_ptdesc(pgd); - -+ #ifdef CONFIG_PTP -+ list_add(&(iee_ptdesc(ptdesc)->pt_list), &pgd_list); -+ #else - list_add(&ptdesc->pt_list, &pgd_list); -+ #endif - } - - static inline void pgd_list_del(pgd_t *pgd) - { - struct ptdesc *ptdesc = virt_to_ptdesc(pgd); - -+ #ifdef CONFIG_PTP -+ list_del(&(iee_ptdesc(ptdesc)->pt_list)); -+ #else - list_del(&ptdesc->pt_list); -+ #endif - } - - #define UNSHARED_PTRS_PER_PGD \ -@@ -112,12 +126,22 @@ static inline void pgd_list_del(pgd_t *pgd) - - static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) - { -+ #ifdef CONFIG_PTP -+ struct ptdesc *ptdesc = virt_to_ptdesc(pgd); -+ (iee_ptdesc(ptdesc))->pt_mm = mm; -+ #else - virt_to_ptdesc(pgd)->pt_mm = mm; -+ #endif - } - - struct mm_struct *pgd_page_get_mm(struct page *page) - { -+ #ifdef CONFIG_PTP -+ struct ptdesc *ptdesc = page_ptdesc(page); -+ return (iee_ptdesc(ptdesc))->pt_mm; -+ #else - return page_ptdesc(page)->pt_mm; -+ #endif - } - - static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) -@@ -417,16 +441,44 @@ static inline void _pgd_free(pgd_t *pgd) - - static inline pgd_t *_pgd_alloc(void) - { -+ #ifdef CONFIG_PTP -+ return (pgd_t *)get_iee_pgd_page(GFP_PGTABLE_USER); -+ #else - return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, - PGD_ALLOCATION_ORDER); -+ #endif - } - - static inline void _pgd_free(pgd_t *pgd) - { -+ #ifdef CONFIG_PTP -+ struct page *page = virt_to_page((void *)pgd); -+ #if PGD_ALLOCATION_ORDER == 1 -+ page = virt_to_page((unsigned long)pgd + PAGE_SIZE); -+ #endif -+ free_iee_pgd_page(pgd); -+ #else - free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); -+ #endif - } - #endif /* CONFIG_X86_PAE */ - -+#ifdef CONFIG_KOI -+pgd_t *koi_pgd_alloc(void) ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ if (prot) ++ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); ++ else ++ pmd = __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY); ++ set_pmd(pmdp, pmd); ++ } else { ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte; ++ ++ /* Protect addresses one by one on this pte table.*/ ++ for (int i = 0; i < (1UL << order); i++) { ++ /* Clear continuous bits first. */ ++ if (pte_val(*ptep) & PTE_CONT && !iee_support_cont_pte(addr, order)) { ++ pte_t *cont_ptep = pte_offset_kernel(pmdp, addr & CONT_PTE_MASK); ++ ++ for (int j = 0; j < CONT_PTES; j++) { ++ set_pte(cont_ptep, __pte(pte_val(*cont_ptep) & ~PTE_CONT)); ++ cont_ptep++; ++ } ++ } ++ ++ pte = READ_ONCE(*ptep); ++ if (prot) ++ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); ++ else ++ pte = __pte(pte_val(pte) & ~PTE_RDONLY); ++ set_pte(ptep, pte); ++ ++ ptep++; ++ } ++ } ++} ++ ++void iee_set_logical_mem_ro(unsigned long addr) +{ -+ pgd_t *pgd; -+#ifdef CONFIG_PTP -+ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL); -+ unsigned long iee_addr = (unsigned long)__phys_to_iee(__pa(pgd)); -+ set_iee_page_valid(iee_addr); -+ iee_set_logical_mem_ro((unsigned long)pgd); -+#else -+ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL); -+#endif -+ return pgd; ++ iee_set_logical_mem(addr, 0, true); ++ __flush_tlb_kernel_pgtable(addr); ++ isb(); +} -+#endif + - pgd_t *pgd_alloc(struct mm_struct *mm) - { - pgd_t *pgd; -@@ -557,8 +609,13 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, - int ret = 0; - - if (pte_young(*ptep)) -+ #ifdef CONFIG_PTP -+ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, -+ (unsigned long *) &ptep->pte); -+ #else - ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, - (unsigned long *) &ptep->pte); -+ #endif - - return ret; - } -@@ -570,8 +627,13 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, - int ret = 0; - - if (pmd_young(*pmdp)) -+ #ifdef CONFIG_PTP -+ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, -+ (unsigned long *)pmdp); -+ #else - ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, - (unsigned long *)pmdp); -+ #endif - - return ret; - } -@@ -584,8 +646,13 @@ int pudp_test_and_clear_young(struct vm_area_struct *vma, - int ret = 0; - - if (pud_young(*pudp)) -+ #ifdef CONFIG_PTP -+ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, -+ (unsigned long *)pudp); -+ #else - ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, - (unsigned long *)pudp); -+ #endif - - return ret; - } -@@ -833,7 +900,11 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) - for (i = 0; i < PTRS_PER_PMD; i++) { - if (!pmd_none(pmd_sv[i])) { - pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page(pte); -+ #else - free_page((unsigned long)pte); -+ #endif - } - } - -@@ -863,7 +934,11 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) - /* INVLPG to clear all paging-structure caches */ - flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); - -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page(pte); -+ #else - free_page((unsigned long)pte); -+ #endif - - return 1; - } -diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c -index 83a6bdf0b498..759da9856751 100644 ---- a/arch/x86/mm/pti.c -+++ b/arch/x86/mm/pti.c -@@ -39,6 +39,10 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif ++void iee_set_logical_mem_rw(unsigned long addr) ++{ ++ iee_set_logical_mem(addr, 0, false); ++ __flush_tlb_kernel_pgtable(addr); ++ isb(); ++} + - #undef pr_fmt - #define pr_fmt(fmt) "Kernel/User page tables isolation: " fmt - -@@ -139,7 +143,11 @@ pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) - * The user page tables get the full PGD, accessible from - * userspace: - */ -+ #ifdef CONFIG_PTP -+ iee_rw_gate(IEE_OP_SET_PGD, kernel_to_user_pgdp(pgdp), pgd); -+ #else - kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; -+ #endif - - /* - * If this is normal user memory, make it NX in the kernel -@@ -182,6 +190,10 @@ static p4d_t *pti_user_pagetable_walk_p4d(unsigned long address) - unsigned long new_p4d_page = __get_free_page(gfp); - if (WARN_ON_ONCE(!new_p4d_page)) - return NULL; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)new_p4d_page, 0); -+ #endif - - set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); - } -@@ -211,6 +223,10 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) - unsigned long new_pud_page = __get_free_page(gfp); - if (WARN_ON_ONCE(!new_pud_page)) - return NULL; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)new_pud_page, 0); -+ #endif - - set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); - } -@@ -225,6 +241,10 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) - unsigned long new_pmd_page = __get_free_page(gfp); - if (WARN_ON_ONCE(!new_pmd_page)) - return NULL; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)new_pmd_page, 0); -+ #endif - - set_pud(pud, __pud(_KERNPG_TABLE | __pa(new_pmd_page))); - } -@@ -266,6 +286,10 @@ static pte_t *pti_user_pagetable_walk_pte(unsigned long address, bool late_text) - unsigned long new_pte_page = __get_free_page(gfp); - if (!new_pte_page) - return NULL; -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)new_pte_page, 0); -+ #endif - - set_pmd(pmd, __pmd(_KERNPG_TABLE | __pa(new_pte_page))); - } -@@ -292,7 +316,11 @@ static void __init pti_setup_vsyscall(void) - if (WARN_ON(!target_pte)) - return; - -+ #ifdef CONFIG_PTP -+ set_pte(target_pte, *pte); -+ #else - *target_pte = *pte; -+ #endif - set_vsyscall_pgtable_user_bits(kernel_to_user_pgdp(swapper_pg_dir)); - } - #else -@@ -370,14 +398,22 @@ pti_clone_pgtable(unsigned long start, unsigned long end, - * code that only set this bit when supported. - */ - if (boot_cpu_has(X86_FEATURE_PGE)) -+ #ifdef CONFIG_PTP -+ set_pmd(pmd, pmd_set_flags(*pmd, _PAGE_GLOBAL)); -+ #else - *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); -+ #endif - - /* - * Copy the PMD. That is, the kernelmode and usermode - * tables will share the last-level page tables of this - * address range - */ -+ #ifdef CONFIG_PTP -+ set_pmd(target_pmd, *pmd); -+ #else - *target_pmd = *pmd; -+ #endif - - addr = round_up(addr + 1, PMD_SIZE); - -@@ -401,10 +437,18 @@ pti_clone_pgtable(unsigned long start, unsigned long end, - - /* Set GLOBAL bit in both PTEs */ - if (boot_cpu_has(X86_FEATURE_PGE)) -+ #ifdef CONFIG_PTP -+ set_pte(pte, pte_set_flags(*pte, _PAGE_GLOBAL)); -+ #else - *pte = pte_set_flags(*pte, _PAGE_GLOBAL); -+ #endif - - /* Clone the PTE */ -+ #ifdef CONFIG_PTP -+ set_pte(target_pte, *pte); -+ #else - *target_pte = *pte; -+ #endif - - addr = round_up(addr + 1, PAGE_SIZE); - -@@ -430,7 +474,11 @@ static void __init pti_clone_p4d(unsigned long addr) - - kernel_pgd = pgd_offset_k(addr); - kernel_p4d = p4d_offset(kernel_pgd, addr); -+ #ifdef CONFIG_PTP -+ set_p4d(user_p4d, *kernel_p4d); -+ #else - *user_p4d = *kernel_p4d; -+ #endif - } - - /* -@@ -461,7 +509,11 @@ static void __init pti_clone_user_shared(void) - if (WARN_ON(!target_pte)) - return; - -+ #ifdef CONFIG_PTP -+ set_pte(target_pte, pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL)); -+ #else - *target_pte = pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL); -+ #endif - } - } - -diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c -index d61f04743aad..5196ddd91e22 100644 ---- a/arch/x86/net/bpf_jit_comp.c -+++ b/arch/x86/net/bpf_jit_comp.c -@@ -16,6 +16,9 @@ - #include - #include - #include -+#ifdef CONFIG_HIVE -+#include -+#endif - - static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) - { -@@ -53,6 +56,11 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) - #define EMIT_ENDBR() - #endif - -+#ifdef CONFIG_HIVE -+static void emit_mov_imm64(u8 **pprog, u32 dst_reg, -+ const u32 imm32_hi, const u32 imm32_lo); -+#endif ++/* ++ * IEE addresses are mapped in page granule so we only need to valid or invalid ++ * the pte entries of these mappings to mark whether a physical page is inside IEE. ++ */ ++void set_iee_address(unsigned long addr, unsigned int order, bool valid) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ unsigned long end_addr = addr + (PAGE_SIZE << order); ++ ++ if ((addr < (PAGE_OFFSET + IEE_OFFSET)) | ++ (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) { ++ pr_err("IEE: Invalid address to valid in IEE."); ++ return; ++ } ++ ++ if (addr != ALIGN(addr, PAGE_SIZE) ++ || end_addr > ALIGN(addr + 1, PMD_SIZE)) ++ panic("%s: invalid input address range 0x%lx-0x%lx.", __func__, ++ addr, end_addr); + - static bool is_imm8(int value) - { - return value <= 127 && value >= -128; -@@ -183,6 +191,9 @@ static const int reg2hex[] = { - [BPF_REG_AX] = 2, /* R10 temp register */ - [AUX_REG] = 3, /* R11 temp register */ - [X86_REG_R9] = 1, /* R9 register, 6th function argument */ -+#ifdef CONFIG_HIVE -+ [BPF_REG_BASE] = 4, /* R12 callee saved */ -+#endif - }; - - static const int reg2pt_regs[] = { -@@ -205,6 +216,16 @@ static const int reg2pt_regs[] = { - */ - static bool is_ereg(u32 reg) - { -+#ifdef CONFIG_HIVE -+ return (1 << reg) & (BIT(BPF_REG_5) | -+ BIT(AUX_REG) | -+ BIT(BPF_REG_7) | -+ BIT(BPF_REG_8) | -+ BIT(BPF_REG_9) | -+ BIT(X86_REG_R9) | -+ BIT(BPF_REG_AX) | -+ BIT(BPF_REG_BASE)); ++ for (int i = 0; i < (1UL << order); i++) { ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if (valid) ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ else ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ ++ set_pte(ptep, pte); ++ ptep++; ++ } ++} ++ ++// TODO: Delete when allocate page table from pool ++void set_iee_address_valid(unsigned long lm_addr, unsigned int order) ++{ ++ set_iee_address(__virt_to_iee(lm_addr), order, true); ++} ++ ++void set_iee_address_invalid(unsigned long lm_addr, unsigned int order) ++{ ++ set_iee_address(__virt_to_iee(lm_addr), order, false); ++} ++// TODO END ++ ++/* Modify linear and IEE mappings of each address at the same time to avoid ++ * synchronization problems. ++ */ ++static void iee_set_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, ++ int use_block_pmd) ++{ ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, use_block_pmd); +#else - return (1 << reg) & (BIT(BPF_REG_5) | - BIT(AUX_REG) | - BIT(BPF_REG_7) | -@@ -212,6 +233,7 @@ static bool is_ereg(u32 reg) - BIT(BPF_REG_9) | - BIT(X86_REG_R9) | - BIT(BPF_REG_AX)); ++ int i; ++ ++ if (use_block_pmd) { ++ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); ++ ++ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); ++ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*iee_ptep); ++ ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ iee_ptep++; ++ } ++ } else { ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*lm_ptep); ++ ++ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); ++ WRITE_ONCE(*lm_ptep, pte); ++ pte = READ_ONCE(*iee_ptep); ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ lm_ptep++; ++ iee_ptep++; ++ } ++ } ++ dsb(ishst); ++ isb(); +#endif - } - - /* -@@ -371,6 +393,26 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, - *pprog = prog; - } - -+#ifdef CONFIG_HIVE -+static void emit_sfi_prologue(u8 **pprog, struct bpf_prog *bpf_prog) ++} ++ ++static void iee_unset_sensitive_pte(pte_t *lm_ptep, pte_t *iee_ptep, int order, int use_block_pmd) +{ -+ u8 *prog = *pprog; -+ u64 base_addr, stack_addr; ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_UNSET_SENSITIVE_PTE, lm_ptep, iee_ptep, order, use_block_pmd); ++#else ++ int i; ++ ++ if (use_block_pmd) { ++ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + -+ /* assign addr to base and rsp */ -+ EMIT2(0x41, 0x54); /* push %r12 */ -+ EMIT1(0x55); /* push %rbp */ ++ pmd = __pmd(pmd_val(pmd) | PTE_DBM); ++ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*iee_ptep); + -+ base_addr = (u64)bpf_prog->shadow_region_addr; -+ emit_mov_imm64(&prog, BPF_REG_BASE, (u32)(base_addr >> 32), (u32)base_addr); -+ -+ stack_addr = (u64)bpf_prog->shadow_stack_addr; -+ emit_mov_imm64(&prog, BPF_REG_FP, (u32)(stack_addr >> 32), (u32)stack_addr); ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ iee_ptep++; ++ } ++ } else { ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*lm_ptep); + -+ *pprog = prog; -+} ++ pte = __pte(pte_val(pte) | PTE_DBM); ++ WRITE_ONCE(*lm_ptep, pte); ++ pte = READ_ONCE(*iee_ptep); ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ lm_ptep++; ++ iee_ptep++; ++ } ++ } ++ dsb(ishst); ++ isb(); +#endif ++} + - static int emit_patch(u8 **pprog, void *func, void *ip, u8 opcode) - { - u8 *prog = *pprog; -@@ -503,6 +545,18 @@ static void emit_return(u8 **pprog, u8 *ip) - *pprog = prog; - } - -+#ifdef CONFIG_HIVE -+static void emit_sfi_epilogue(u8 **pprog) ++/* Only support address range smaller then one PMD block. */ ++void put_pages_into_iee_block(unsigned long addr, int order) +{ -+ u8 *prog = *pprog; ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pmd_t *pmdp; ++ pte_t *lm_ptep; ++ pte_t *iee_ptep; ++ unsigned long iee_addr; ++ int use_block_pmd = 0; ++ ++ /* Split pmd block if needed. */ ++ iee_may_split_pmd(pudp, addr, order); ++ ++ pmdp = pmd_offset(pudp, addr); ++ ++ use_block_pmd = pmd_leaf(READ_ONCE(*pmdp)); ++ ++ if (use_block_pmd) ++ lm_ptep = (pte_t *)pmdp; ++ else ++ lm_ptep = pte_offset_kernel(pmdp, addr); ++ ++ // Handling cont mapping. ++ if (((1 << order) < CONT_PTES) && (pte_val(*lm_ptep) & PTE_CONT)) { ++ // The beginning of cont mapping. ++ int i; ++ pte_t *ptep = pte_offset_kernel(pmdp, addr & CONT_PTE_MASK); + -+ EMIT1(0x5d); /* pop %rbp */ -+ EMIT2(0x41, 0x5c); /* pop %r12 */ ++ if (order < CONFIG_ARM64_CONT_PTE_SHIFT) { ++ for (i = 0; i < CONT_PTES; i++) { ++ set_pte(ptep, __pte(pte_val(*ptep) & ~PTE_CONT)); ++ ptep++; ++ } ++ } ++ } + -+ *pprog = prog; ++ iee_addr = ((unsigned long)addr | IEE_OFFSET); ++ pgdp = pgd_offset_pgd(pgdir, iee_addr); ++ p4dp = p4d_offset(pgdp, iee_addr); ++ pudp = pud_offset(p4dp, iee_addr); ++ pmdp = pmd_offset(pudp, iee_addr); ++ iee_ptep = pte_offset_kernel(pmdp, iee_addr); ++ iee_set_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); ++ isb(); +} -+#endif + - /* - * Generate the following code: - * -@@ -1097,6 +1151,10 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image - bpf_prog_was_classic(bpf_prog), tail_call_reachable, - bpf_prog->aux->func_idx != 0); - push_callee_regs(&prog, callee_regs_used); -+#ifdef CONFIG_HIVE -+ if (bpf_prog->aux->func_idx == 0) // not subprog -+ emit_sfi_prologue(&prog, bpf_prog); -+#endif - - ilen = prog - temp; - if (rw_image) -@@ -1945,6 +2003,10 @@ st: if (is_imm8(insn->off)) - seen_exit = true; - /* Update cleanup_addr */ - ctx->cleanup_addr = proglen; -+#ifdef CONFIG_HIVE -+ if (bpf_prog->aux->func_idx == 0) // not sub_prog -+ emit_sfi_epilogue(&prog); -+#endif - pop_callee_regs(&prog, callee_regs_used); - EMIT1(0xC9); /* leave */ - emit_return(&prog, image + addrs[i - 1] + (prog - temp)); -diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c -index 91d31ac422d6..e7c782f0cde5 100644 ---- a/arch/x86/platform/efi/efi_64.c -+++ b/arch/x86/platform/efi/efi_64.c -@@ -49,6 +49,10 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif ++void put_pages_into_iee(unsigned long addr, int order) ++{ ++ unsigned long end = addr + (PAGE_SIZE << order); + - /* - * We allocate runtime services regions top-down, starting from -4G, i.e. - * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G. -@@ -74,6 +78,11 @@ int __init efi_alloc_page_tables(void) - - gfp_mask = GFP_KERNEL | __GFP_ZERO; - efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); -+ -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER); -+ #endif -+ - if (!efi_pgd) - goto fail; - -@@ -116,7 +125,11 @@ void efi_sync_low_kernel_mappings(void) - pgd_k = pgd_offset_k(PAGE_OFFSET); - - num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); -+ #ifdef CONFIG_PTP -+ iee_memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); -+ #else - memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); -+ #endif - - pgd_efi = efi_pgd + pgd_index(EFI_VA_END); - pgd_k = pgd_offset_k(EFI_VA_END); -@@ -124,7 +137,11 @@ void efi_sync_low_kernel_mappings(void) - p4d_k = p4d_offset(pgd_k, 0); - - num_entries = p4d_index(EFI_VA_END); -+ #ifdef CONFIG_PTP -+ iee_memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); -+ #else - memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); -+ #endif - - /* - * We share all the PUD entries apart from those that map the -@@ -139,13 +156,21 @@ void efi_sync_low_kernel_mappings(void) - pud_k = pud_offset(p4d_k, 0); - - num_entries = pud_index(EFI_VA_END); -+ #ifdef CONFIG_PTP -+ iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); -+ #else - memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); -+ #endif - - pud_efi = pud_offset(p4d_efi, EFI_VA_START); - pud_k = pud_offset(p4d_k, EFI_VA_START); - - num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); -+ #ifdef CONFIG_PTP -+ iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); -+ #else - memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); -+ #endif - } - - /* -diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c -index a595953f1d6d..a83a51431b60 100644 ---- a/arch/x86/power/hibernate_64.c -+++ b/arch/x86/power/hibernate_64.c -@@ -55,15 +55,24 @@ static int set_up_temporary_text_mapping(pgd_t *pgd) - p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); - if (!p4d) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)p4d, 0); -+ #endif - } - - pud = (pud_t *)get_safe_page(GFP_ATOMIC); - if (!pud) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pud, 0); -+ #endif - - pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); - if (!pmd) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pmd, 0); -+ #endif - - set_pmd(pmd + pmd_index(restore_jump_address), - __pmd((jump_address_phys & PMD_MASK) | pgprot_val(pmd_text_prot))); -@@ -86,7 +95,13 @@ static int set_up_temporary_text_mapping(pgd_t *pgd) - - static void *alloc_pgt_page(void *context) - { -+ #ifdef CONFIG_PTP -+ unsigned long new = get_safe_page(GFP_ATOMIC); -+ set_iee_page((unsigned long)new, 0); -+ return (void *)new; -+ #else - return (void *)get_safe_page(GFP_ATOMIC); -+ #endif - } - - static int set_up_temporary_mappings(void) -@@ -104,6 +119,9 @@ static int set_up_temporary_mappings(void) - pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); - if (!pgd) - return -ENOMEM; -+ #ifdef CONFIG_PTP -+ set_iee_page((unsigned long)pgd, 0); -+ #endif - - /* Prepare a temporary mapping for the kernel text */ - result = set_up_temporary_text_mapping(pgd); -diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c -index 6b201e64d8ab..ee450d16c687 100644 ---- a/arch/x86/xen/mmu_pv.c -+++ b/arch/x86/xen/mmu_pv.c -@@ -785,6 +785,9 @@ void xen_mm_pin_all(void) - spin_lock(&pgd_lock); - - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - if (!PagePinned(page)) { - __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); - SetPageSavePinned(page); -@@ -891,6 +894,9 @@ void xen_mm_unpin_all(void) - spin_lock(&pgd_lock); - - list_for_each_entry(page, &pgd_list, lru) { -+ #ifdef CONFIG_PTP -+ page = iee_ptdesc_to_page(page); -+ #endif - if (PageSavePinned(page)) { - BUG_ON(!PagePinned(page)); - __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); -diff --git a/block/sed-opal.c b/block/sed-opal.c -index 1a1cb35bf4b7..7149485a808c 100644 ---- a/block/sed-opal.c -+++ b/block/sed-opal.c -@@ -316,7 +316,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) - return PTR_ERR(kref); - - key = key_ref_to_ptr(kref); -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - ret = key_validate(key); - if (ret == 0) { - if (buflen > key->datalen) -@@ -324,7 +328,11 @@ static int read_sed_opal_key(const char *key_name, u_char *buffer, int buflen) - - ret = key->type->read(key, (char *)buffer, buflen); - } -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - - key_ref_put(kref); - -diff --git a/certs/blacklist.c b/certs/blacklist.c -index 675dd7a8f07a..4db9c6cb7732 100644 ---- a/certs/blacklist.c -+++ b/certs/blacklist.c -@@ -19,6 +19,10 @@ - #include - #include "blacklist.h" - -+#ifdef CONFIG_KEYP -+#include ++ /* Split the address range if needed. */ ++ if (order < IEE_DATA_ORDER) ++ put_pages_into_iee_block(addr, order); ++ else { ++ if (addr != ALIGN(addr, PMD_SIZE)) ++ panic("IEE: Invalid input addr 0x%lx order %d for %s", ++ addr, order, __func__); ++ while (addr < end) { ++ put_pages_into_iee_block(addr, IEE_DATA_ORDER); ++ addr += PMD_SIZE; ++ } ++ } ++} ++ ++void remove_pages_from_iee(unsigned long addr, int order) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ pte_t *lm_ptep; ++ pte_t *iee_ptep; ++ unsigned long iee_addr; ++ int use_block_pmd = 0; ++ ++ // Use Block Descriptor. ++ if (pmd_leaf(*pmdp)) { ++ use_block_pmd = 1; ++ lm_ptep = (pte_t *)pmdp; ++ } else ++ lm_ptep = pte_offset_kernel(pmdp, addr); ++ ++ iee_addr = ((unsigned long)addr | IEE_OFFSET); ++ pgdp = pgd_offset_pgd(pgdir, iee_addr); ++ p4dp = p4d_offset(pgdp, iee_addr); ++ pudp = pud_offset(p4dp, iee_addr); ++ pmdp = pmd_offset(pudp, iee_addr); ++ iee_ptep = pte_offset_kernel(pmdp, iee_addr); ++ iee_unset_sensitive_pte(lm_ptep, iee_ptep, order, use_block_pmd); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE*(1 << order)); ++ flush_tlb_kernel_range(iee_addr, iee_addr+PAGE_SIZE*(1 << order)); ++ isb(); ++} ++ ++void set_iee_page(unsigned long addr, int order, enum HAOC_BITMAP_TYPE type) ++{ ++ put_pages_into_iee(addr, order); ++ iee_set_bitmap_type(addr, 1UL << order, type); ++} ++ ++void unset_iee_page(unsigned long addr, int order) ++{ ++ remove_pages_from_iee(addr, order); ++ iee_set_bitmap_type(addr, 1UL << order, IEE_NORMAL); ++} ++ ++void iee_print_pgtables(u64 addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ pgd_t pgd = READ_ONCE(*pgdp); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pud_t pud = READ_ONCE(*pudp); ++ pmd_t *pmdp; ++ pmd_t pmd; ++ pte_t *ptep; ++ pte_t pte; ++ ++ if ((pud_val(pud) & PUD_TYPE_MASK) == PUD_TYPE_SECT) { ++ pr_info("IEE TEST: address 0x%llx: pgd 0x%llx, pud 0x%llx", ++ addr, pgd_val(pgd), pud_val(pud)); ++ return; ++ } ++ ++ pmdp = pmd_offset(pudp, addr); ++ pmd = READ_ONCE(*pmdp); ++ ++ pr_info("IEE TEST: address 0x%llx: pgd 0x%llx, pud 0x%llx, pmd 0x%llx", ++ addr, pgd_val(pgd), pud_val(pud), pmd_val(pmd)); ++ ++ if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) ++ return; ++ ++ ptep = pte_offset_kernel(pmdp, addr); ++ pte = READ_ONCE(*ptep); ++ pr_info("IEE TEST: address 0x%llx: pte 0x%llx", addr, pte_val(pte)); ++} ++ ++unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) ++{ ++#ifdef CONFIG_IEE_PTRP ++ if (strcmp(s->name, "task_struct") == 0) ++ return IEE_DATA_ORDER; +#endif ++ return order; ++} + - /* - * According to crypto/asymmetric_keys/x509_cert_parser.c:x509_note_pkey_algo(), - * the size of the currently longest supported hash algorithm is 512 bits, -@@ -91,7 +95,11 @@ static int blacklist_key_instantiate(struct key *key, - #endif - - /* Sets safe default permissions for keys loaded by user space. */ -+ #ifdef CONFIG_KEYP -+ iee_set_key_perm(key, BLACKLIST_KEY_PERM); -+ #else - key->perm = BLACKLIST_KEY_PERM; -+ #endif - - /* - * Skips the authentication step for builtin hashes, they are not -diff --git a/certs/system_keyring.c b/certs/system_keyring.c -index a803524044bd..eab10ce55564 100644 ---- a/certs/system_keyring.c -+++ b/certs/system_keyring.c -@@ -96,9 +96,15 @@ int restrict_link_by_builtin_and_secondary_trusted( - /* If we have a secondary trusted keyring, then that contains a link - * through to the builtin keyring and the search will follow that link. - */ -+ #ifdef CONFIG_KEYP -+ if (type == &key_type_keyring && -+ dest_keyring == secondary_trusted_keys && -+ payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) -+ #else - if (type == &key_type_keyring && - dest_keyring == secondary_trusted_keys && - payload == &builtin_trusted_keys->payload) -+ #endif - /* Allow the builtin keyring to be added to the secondary */ - return 0; - -@@ -125,9 +131,15 @@ int restrict_link_by_digsig_builtin_and_secondary(struct key *dest_keyring, - /* If we have a secondary trusted keyring, then that contains a link - * through to the builtin keyring and the search will follow that link. - */ -+ #ifdef CONFIG_KEYP -+ if (type == &key_type_keyring && -+ dest_keyring == secondary_trusted_keys && -+ payload == (union key_payload *)(builtin_trusted_keys->name_link.next)) -+ #else - if (type == &key_type_keyring && - dest_keyring == secondary_trusted_keys && - payload == &builtin_trusted_keys->payload) -+ #endif - /* Allow the builtin keyring to be added to the secondary */ - return 0; - -@@ -212,9 +224,15 @@ int restrict_link_by_builtin_secondary_and_machine( - const union key_payload *payload, - struct key *restrict_key) - { -+ #ifdef CONFIG_KEYP -+ if (machine_trusted_keys && type == &key_type_keyring && -+ dest_keyring == secondary_trusted_keys && -+ payload == (union key_payload *)(machine_trusted_keys->name_link.next)) -+ #else - if (machine_trusted_keys && type == &key_type_keyring && - dest_keyring == secondary_trusted_keys && - payload == &machine_trusted_keys->payload) -+ #endif - /* Allow the machine keyring to be added to the secondary */ - return 0; - -diff --git a/crypto/af_alg.c b/crypto/af_alg.c -index 68cc9290cabe..6d4415de9ee4 100644 ---- a/crypto/af_alg.c -+++ b/crypto/af_alg.c -@@ -304,7 +304,11 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, - if (IS_ERR(key)) - return PTR_ERR(key); - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - - ret = ERR_PTR(-ENOPROTOOPT); - if (!strcmp(key->type->name, "user") || -@@ -319,21 +323,33 @@ static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, - } - - if (IS_ERR(ret)) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - return PTR_ERR(ret); - } - - key_data = sock_kmalloc(&ask->sk, key_datalen, GFP_KERNEL); - if (!key_data) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - return -ENOMEM; - } - - memcpy(key_data, ret, key_datalen); - -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - - err = type->setkey(ask->private, key_data, key_datalen); -diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c -index 43af5fa510c0..134f38bd1be9 100644 ---- a/crypto/asymmetric_keys/asymmetric_type.c -+++ b/crypto/asymmetric_keys/asymmetric_type.c -@@ -16,6 +16,9 @@ - #include - #include - #include "asymmetric_keys.h" -+#ifdef CONFIG_KEYP -+#include ++void iee_set_min_partial(struct kmem_cache *s) ++{ ++#ifdef CONFIG_IEE_PTRP ++ if (strcmp(s->name, "task_struct") == 0) ++ s->min_partial *= 16; +#endif - - - const char *const key_being_used_for[NR__KEY_BEING_USED_FOR] = { -@@ -467,6 +470,17 @@ static void asymmetric_key_free_preparse(struct key_preparsed_payload *prep) - static void asymmetric_key_destroy(struct key *key) - { - struct asymmetric_key_subtype *subtype = asymmetric_key_subtype(key); -+ #ifdef CONFIG_KEYP -+ struct asymmetric_key_ids *kids = ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; -+ void *data = ((union key_payload *)(key->name_link.next))->data[asym_crypto]; -+ void *auth = ((union key_payload *)(key->name_link.next))->data[asym_auth]; -+ -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[asym_crypto] = NULL; -+ key_payload->data[asym_subtype] = NULL; -+ key_payload->data[asym_key_ids] = NULL; -+ key_payload->data[asym_auth] = NULL; -+ #else - struct asymmetric_key_ids *kids = key->payload.data[asym_key_ids]; - void *data = key->payload.data[asym_crypto]; - void *auth = key->payload.data[asym_auth]; -@@ -475,6 +489,7 @@ static void asymmetric_key_destroy(struct key *key) - key->payload.data[asym_subtype] = NULL; - key->payload.data[asym_key_ids] = NULL; - key->payload.data[asym_auth] = NULL; -+ #endif - - if (subtype) { - subtype->destroy(data, auth); -@@ -580,8 +595,13 @@ int asymmetric_key_eds_op(struct kernel_pkey_params *params, - if (key->type != &key_type_asymmetric) - return -EINVAL; - subtype = asymmetric_key_subtype(key); -+ #ifdef CONFIG_KEYP -+ if (!subtype || -+ !((union key_payload *)(key->name_link.next))->data[0]) -+ #else - if (!subtype || - !key->payload.data[0]) -+ #endif - return -EINVAL; - if (!subtype->eds_op) - return -ENOTSUPP; -diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c -index 1dcab27986a6..490ba98995c1 100644 ---- a/crypto/asymmetric_keys/public_key.c -+++ b/crypto/asymmetric_keys/public_key.c -@@ -30,7 +30,11 @@ MODULE_LICENSE("GPL"); - static void public_key_describe(const struct key *asymmetric_key, - struct seq_file *m) - { -+ #ifdef CONFIG_KEYP -+ struct public_key *key = ((union key_payload *)(asymmetric_key->name_link.next))->data[asym_crypto]; -+ #else - struct public_key *key = asymmetric_key->payload.data[asym_crypto]; -+ #endif - - if (key) - seq_printf(m, "%s.%s", key->id_type, key->pkey_algo); -@@ -158,7 +162,11 @@ static int software_key_query(const struct kernel_pkey_params *params, - struct kernel_pkey_query *info) - { - struct crypto_akcipher *tfm; -+ #ifdef CONFIG_KEYP -+ struct public_key *pkey = ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; -+ #else - struct public_key *pkey = params->key->payload.data[asym_crypto]; -+ #endif - char alg_name[CRYPTO_MAX_ALG_NAME]; - struct crypto_sig *sig; - u8 *key, *ptr; -@@ -273,7 +281,11 @@ static int software_key_query(const struct kernel_pkey_params *params, - static int software_key_eds_op(struct kernel_pkey_params *params, - const void *in, void *out) - { -+ #ifdef CONFIG_KEYP -+ const struct public_key *pkey = ((union key_payload *)(params->key->name_link.next))->data[asym_crypto]; -+ #else - const struct public_key *pkey = params->key->payload.data[asym_crypto]; -+ #endif - char alg_name[CRYPTO_MAX_ALG_NAME]; - struct crypto_akcipher *tfm; - struct crypto_sig *sig; -@@ -453,7 +465,11 @@ EXPORT_SYMBOL_GPL(public_key_verify_signature); - static int public_key_verify_signature_2(const struct key *key, - const struct public_key_signature *sig) - { -+ #ifdef CONFIG_KEYP -+ const struct public_key *pk = ((union key_payload *)(key->name_link.next))->data[asym_crypto]; -+ #else - const struct public_key *pk = key->payload.data[asym_crypto]; -+ #endif - return public_key_verify_signature(pk, sig); - } - -diff --git a/crypto/asymmetric_keys/signature.c b/crypto/asymmetric_keys/signature.c -index 2deff81f8af5..696fa2c4e708 100644 ---- a/crypto/asymmetric_keys/signature.c -+++ b/crypto/asymmetric_keys/signature.c -@@ -51,8 +51,13 @@ int query_asymmetric_key(const struct kernel_pkey_params *params, - if (key->type != &key_type_asymmetric) - return -EINVAL; - subtype = asymmetric_key_subtype(key); -+ #ifdef CONFIG_KEYP -+ if (!subtype || -+ !((union key_payload *)(key->name_link.next))->data[0]) -+ #else - if (!subtype || - !key->payload.data[0]) -+ #endif - return -EINVAL; - if (!subtype->query) - return -ENOTSUPP; -@@ -145,8 +150,13 @@ int verify_signature(const struct key *key, - if (key->type != &key_type_asymmetric) - return -EINVAL; - subtype = asymmetric_key_subtype(key); -+ #ifdef CONFIG_KEYP -+ if (!subtype || -+ !((union key_payload *)(key->name_link.next))->data[0]) -+ #else - if (!subtype || - !key->payload.data[0]) -+ #endif - return -EINVAL; - if (!subtype->verify_signature) - return -ENOTSUPP; -diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c -index 83092d93f36a..fb12e7d0660a 100644 ---- a/drivers/firmware/efi/arm-runtime.c -+++ b/drivers/firmware/efi/arm-runtime.c -@@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void) - return 0; - } - -+ #ifdef CONFIG_PTP -+ efi_memmap_unmap_after_init(); -+ #else - efi_memmap_unmap(); -+ #endif - - mapsize = efi.memmap.desc_size * efi.memmap.nr_map; - -diff --git a/drivers/firmware/efi/memmap.c b/drivers/firmware/efi/memmap.c -index 77dd20f9df31..ea8ebecaa153 100644 ---- a/drivers/firmware/efi/memmap.c -+++ b/drivers/firmware/efi/memmap.c -@@ -96,6 +96,26 @@ void __init efi_memmap_unmap(void) - clear_bit(EFI_MEMMAP, &efi.flags); - } - -+#ifdef CONFIG_PTP -+void __init efi_memmap_unmap_after_init(void) ++} ++ ++ ++#include ++#include ++#include ++ ++extern void arm64_enter_nmi(struct pt_regs *regs); ++extern const char *esr_get_class_string(unsigned long esr); ++static char *handler[] = { ++ "ELR_EL1", ++ "TCR_EL1", ++}; ++ ++asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, ++ unsigned int esr, unsigned long info) +{ -+ if (!efi_enabled(EFI_MEMMAP)) -+ return; ++ arm64_enter_nmi(regs); + -+ if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) { -+ unsigned long size; ++ console_verbose(); + -+ size = efi.memmap.desc_size * efi.memmap.nr_map; -+ early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size); -+ } else { -+ memunmap(efi.memmap.map); ++ pr_crit("IEE : Bad mode in %s check detected on CPU%d, code 0x%08x -- %s\n", ++ handler[reason], smp_processor_id(), esr, ++ esr_get_class_string(esr)); ++ ++ __show_regs(regs); ++ local_daif_mask(); ++ ++ switch (reason) { ++ case 0: ++ pr_crit("IEE: Bad ELR_EL1 0x%llx\n", (u64)info); ++ break; ++ case 1: ++ pr_crit("IEE: Bad TCR_EL1 0x%llx\n", (u64)info); ++ break; + } + -+ efi.memmap.map = NULL; -+ clear_bit(EFI_MEMMAP, &efi.flags); ++ panic("bad mode"); +} -+#endif +diff --git a/arch/arm64/kernel/haoc/iee/iee-gate.S b/arch/arm64/kernel/haoc/iee/iee-gate.S +new file mode 100644 +index 000000000000..067a77d5b13b +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee-gate.S +@@ -0,0 +1,92 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#include ++#include ++#include ++#include ++SYM_FUNC_START(iee_rw_gate) ++ /* save daif, close irq */ ++ mrs x13, daif ++ msr daifset, #0x2 ++ isb ++ /* save lr */ ++ stp x29, x30, [sp, #-16]! ++ bl iee_protected_rw_gate ++ /* restore lr */ ++ ldp x29, x30, [sp], #16 ++ /* restore daif */ ++ msr daif, x13 ++ ret ++SYM_FUNC_END(iee_rw_gate) + - /** - * efi_memmap_init_late - Map efi.memmap with memremap() - * @phys_addr: Physical address of the new EFI memory map -diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c -index 6c0621f6f572..0ab9f53a7841 100644 ---- a/drivers/iommu/amd/io_pgtable.c -+++ b/drivers/iommu/amd/io_pgtable.c -@@ -187,7 +187,11 @@ static bool increase_address_space(struct protection_domain *domain, - - out: - spin_unlock_irqrestore(&domain->lock, flags); -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page(pte); -+ #else - free_page((unsigned long)pte); -+ #endif - - return ret; - } -diff --git a/drivers/md/Makefile b/drivers/md/Makefile -index 84291e38dca8..b3af9d419b6e 100644 ---- a/drivers/md/Makefile -+++ b/drivers/md/Makefile -@@ -71,6 +71,7 @@ obj-$(CONFIG_DM_PERSISTENT_DATA) += persistent-data/ - obj-$(CONFIG_DM_MIRROR) += dm-mirror.o dm-log.o dm-region-hash.o - obj-$(CONFIG_DM_LOG_USERSPACE) += dm-log-userspace.o - obj-$(CONFIG_DM_ZERO) += dm-zero.o -+obj-$(CONFIG_DM_ZERO_KOI) += dm-zero_koi.o - obj-$(CONFIG_DM_RAID) += dm-raid.o - obj-$(CONFIG_DM_THIN_PROVISIONING) += dm-thin-pool.o - obj-$(CONFIG_DM_VERITY) += dm-verity.o -diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c -index aa6bb5b4704b..ecf777e5d176 100644 ---- a/drivers/md/dm-crypt.c -+++ b/drivers/md/dm-crypt.c -@@ -2512,7 +2512,11 @@ static int set_key_encrypted(struct crypt_config *cc, struct key *key) - { - const struct encrypted_key_payload *ekp; - -+ #ifdef CONFIG_KEYP -+ ekp = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - ekp = key->payload.data[0]; -+ #endif - if (!ekp) - return -EKEYREVOKED; - -@@ -2528,7 +2532,11 @@ static int set_key_trusted(struct crypt_config *cc, struct key *key) - { - const struct trusted_key_payload *tkp; - -+ #ifdef CONFIG_KEYP -+ tkp = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - tkp = key->payload.data[0]; -+ #endif - if (!tkp) - return -EKEYREVOKED; - -@@ -2590,17 +2598,29 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string - return PTR_ERR(key); - } - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - - ret = set_key(cc, key); - if (ret < 0) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - kfree_sensitive(new_key_string); - return ret; - } - -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - - /* clear the flag since following operations may invalidate previously valid key */ -diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c -index 4836508ea50c..d52d5bccc190 100644 ---- a/drivers/md/dm-verity-verify-sig.c -+++ b/drivers/md/dm-verity-verify-sig.c -@@ -40,7 +40,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, - if (IS_ERR(key)) - return PTR_ERR(key); - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - - ukp = user_key_payload_locked(key); - if (!ukp) { -@@ -58,7 +62,11 @@ static int verity_verify_get_sig_from_key(const char *key_desc, - memcpy(sig_opts->sig, ukp->data, sig_opts->sig_size); - - end: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - - return ret; -diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c -index 3b13e6eb1aa4..a93ac50440c6 100644 ---- a/drivers/md/dm-zero.c -+++ b/drivers/md/dm-zero.c -@@ -79,4 +79,4 @@ module_dm(zero); - - MODULE_AUTHOR("Jana Saout "); - MODULE_DESCRIPTION(DM_NAME " dummy target returning zeros"); --MODULE_LICENSE("GPL"); -+MODULE_LICENSE("GPL"); -\ No newline at end of file -diff --git a/drivers/nvdimm/security.c b/drivers/nvdimm/security.c -index a03e3c45f297..71bccea085e9 100644 ---- a/drivers/nvdimm/security.c -+++ b/drivers/nvdimm/security.c -@@ -28,7 +28,11 @@ static void *key_data(struct key *key) - { - struct encrypted_key_payload *epayload = dereference_key_locked(key); - -+ #ifdef CONFIG_KEYP -+ lockdep_assert_held_read(&KEY_SEM(key)); -+ #else - lockdep_assert_held_read(&key->sem); -+ #endif - - return epayload->decrypted_data; - } -@@ -38,7 +42,11 @@ static void nvdimm_put_key(struct key *key) - if (!key) - return; - -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - } - -@@ -65,10 +73,18 @@ static struct key *nvdimm_request_key(struct nvdimm *nvdimm) - } else { - struct encrypted_key_payload *epayload; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - epayload = dereference_key_locked(key); - if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - key = NULL; - } -@@ -107,10 +123,18 @@ static struct key *nvdimm_lookup_user_key(struct nvdimm *nvdimm, - - dev_dbg(dev, "%s: key found: %#x\n", __func__, key_serial(key)); - -+ #ifdef CONFIG_KEYP -+ down_read_nested(&KEY_SEM(key), subclass); -+ #else - down_read_nested(&key->sem, subclass); -+ #endif - epayload = dereference_key_locked(key); - if (epayload->decrypted_datalen != NVDIMM_PASSPHRASE_LEN) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - key = NULL; - } -diff --git a/drivers/rtc/rtc-test.c b/drivers/rtc/rtc-test.c -index 7e0d8fb26465..950ee6396659 100644 ---- a/drivers/rtc/rtc-test.c -+++ b/drivers/rtc/rtc-test.c -@@ -198,4 +198,4 @@ MODULE_DESCRIPTION("RTC test driver/device"); - MODULE_LICENSE("GPL v2"); - - module_init(test_init); --module_exit(test_exit); -+module_exit(test_exit); -\ No newline at end of file -diff --git a/drivers/rtc/rtc-test_glue.h b/drivers/rtc/rtc-test_glue.h ++EXPORT_SYMBOL(iee_rw_gate) ++ ++ .pushsection ".iee.text", "ax" ++ ++.global iee_tcr_mask ++iee_tcr_mask: ++ .quad IEE_TCR_MASK ++ ++.global iee_tcr ++iee_tcr: ++ .quad 0 ++ ++.global kernel_tcr ++kernel_tcr: ++ .quad 0 ++ ++.global __iee_code_start ++__iee_code_start: ++ ++SYM_FUNC_START(iee_protected_rw_gate) ++ /* entry gate */ ++ mrs x12, tcr_el1 ++ orr x12, x12, #TCR_HPD1 ++ orr x12, x12, #TCR_A1 ++ msr tcr_el1, x12 ++ isb ++ /* Check TCR */ ++ ldr x9, iee_tcr ++ ldr x10, iee_tcr_mask ++ and x12, x12, x10 ++ cmp x9, x12 ++ b.ne 1f ++ ++ /* Switch to iee stack by per cpu ptr. */ ++ ldr_this_cpu x9, iee_cpu_stack_ptr, x10 ++ mov x14, sp ++ mov sp, x9 ++ ++ stp x13, x14, [sp, #-16]! ++ /* call iee func */ ++ adrp x12, iee_funcs ++ add x12, x12, x0, lsl #3 ++ ldr x12, [x12, #:lo12:iee_funcs] ++ stp x29, x30, [sp, #-16]! ++ blr x12 ++ ldp x29, x30, [sp], #16 ++ /* Switch to kernel stack */ ++ ldp x13, x14, [sp], #16 ++ mov sp, x14 ++ /* exit gate */ ++ mrs x12, tcr_el1 ++ bic x12, x12, #TCR_HPD1 ++ bic x12, x12, #TCR_A1 ++ msr tcr_el1, x12 ++ isb ++ /* Check TCR */ ++ ldr x9, kernel_tcr ++ ldr x10, iee_tcr_mask ++ and x12, x12, x10 ++ cmp x9, x12 ++ b.ne 1f ++ ret ++1: ++ mov x0, sp ++ mov x1, #BAD_TCR_EL1 ++ mrs x2, esr_el1 ++ mrs x3, tcr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++SYM_FUNC_END(iee_protected_rw_gate) ++ ++ .popsection +diff --git a/arch/arm64/kernel/haoc/iee/iee-init.c b/arch/arm64/kernel/haoc/iee/iee-init.c new file mode 100644 -index 000000000000..aee28f9680b2 +index 000000000000..42f6cd0fb2a8 --- /dev/null -+++ b/drivers/rtc/rtc-test_glue.h -@@ -0,0 +1,13 @@ ++++ b/arch/arm64/kernel/haoc/iee/iee-init.c +@@ -0,0 +1,56 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif ++#include + -+static int test_rtc_read_time_glue(struct device* dev, struct rtc_time* tm); -+static int test_rtc_set_time_glue(struct device* dev, struct rtc_time* tm); -+static int test_rtc_read_alarm_glue(struct device* dev, struct rtc_wkalrm* alrm); -+static int test_rtc_set_alarm_glue(struct device* dev, struct rtc_wkalrm* alrm); -+static int test_rtc_alarm_irq_enable_glue(struct device* dev, unsigned int enable); ++__aligned(PAGE_SIZE) DEFINE_PER_CPU(u64*[(PAGE_SIZE/8)], ++ iee_cpu_stack_ptr); + ++bool __ro_after_init iee_init_done; + -+static void add_timer_glue(struct timer_list* tim_0); -+static time64_t rtc_tm_to_time64_glue(struct rtc_time* rtc_0); -+static time64_t ktime_get_real_seconds_glue(void); -+static void rtc_time64_to_tm_glue(time64_t val_0, struct rtc_time* rtc_1); -+static noinline int del_timer_glue(struct timer_list *timer); -\ No newline at end of file -diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c -index a5fbb6ed38ae..81428783b9da 100644 ---- a/drivers/tty/serial/earlycon.c -+++ b/drivers/tty/serial/earlycon.c -@@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size) - { - void __iomem *base; - #ifdef CONFIG_FIX_EARLYCON_MEM -+ #ifdef CONFIG_PTP -+ __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO); -+ #else - set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK); ++/* Allocate pages from IEE data pool to use as per-cpu IEE stack. */ ++static void __init iee_stack_alloc(void) ++{ ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ u64 *cpu_stack_ptr = (u64 *)(SHIFT_PERCPU_PTR(iee_cpu_stack_ptr, ++ __per_cpu_offset[cpu])); ++ u64 *new_pages = __va(early_iee_stack_alloc(IEE_STACK_ORDER)); ++ ++ *cpu_stack_ptr = __virt_to_iee((u64)new_pages + IEE_STACK_SIZE); ++ } ++ ++ flush_tlb_all(); ++} ++ ++void __init iee_init_post(void) ++{ ++ /* Flush tlb to enable IEE. */ ++ flush_tlb_all(); ++ ++ #ifdef CONFIG_IEE_PTRP ++ iee_prepare_init_task_token(); + #endif - base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE); - base += paddr & ~PAGE_MASK; - #else -diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c -index 45b42d8f6453..b71072d6957e 100644 ---- a/drivers/usb/early/ehci-dbgp.c -+++ b/drivers/usb/early/ehci-dbgp.c -@@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s) - * FIXME I don't have the bar size so just guess PAGE_SIZE is more - * than enough. 1K is the biggest I have seen. - */ -+ #ifdef CONFIG_PTP -+ __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE); -+ #else - set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK); ++ ++ iee_init_done = true; ++ ++ // haoc_bitmap_sparse_init(); ++ ++ // haoc_bitmap_setup(); ++ ++ #ifdef CONFIG_IEE_SIP ++ extern void iee_si_init(void); ++ iee_si_init(); + #endif - ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE); - ehci_bar += bar_val & ~PAGE_MASK; - dbgp_printk("ehci_bar: %p\n", ehci_bar); -diff --git a/fs/coredump.c b/fs/coredump.c -index 9d235fa14ab9..72be355903ca 100644 ---- a/fs/coredump.c -+++ b/fs/coredump.c -@@ -53,6 +53,10 @@ - - #include - -+#ifdef CONFIG_CREDP -+#include ++} ++ ++void __init iee_stack_init(void) ++{ ++ iee_stack_alloc(); ++} +diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c +new file mode 100644 +index 000000000000..791956d77c3c +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c +@@ -0,0 +1,867 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_PTP ++#include +#endif + - static bool dump_vma_snapshot(struct coredump_params *cprm); - static void free_vma_snapshot(struct coredump_params *cprm); - -@@ -564,7 +568,11 @@ void do_coredump(const kernel_siginfo_t *siginfo) - */ - if (__get_dumpable(cprm.mm_flags) == SUID_DUMP_ROOT) { - /* Setuid core dump mode */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(cred,GLOBAL_ROOT_UID); -+ #else - cred->fsuid = GLOBAL_ROOT_UID; /* Dump root private */ -+ #endif - need_suid_safe = true; - } - -diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c -index 7cbb1fd872ac..a8425a83aabf 100644 ---- a/fs/crypto/keyring.c -+++ b/fs/crypto/keyring.c -@@ -590,8 +590,13 @@ static void fscrypt_provisioning_key_describe(const struct key *key, - { - seq_puts(m, key->description); - if (key_is_positive(key)) { -+ #ifdef CONFIG_KEYP -+ const struct fscrypt_provisioning_key_payload *payload = -+ ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - const struct fscrypt_provisioning_key_payload *payload = - key->payload.data[0]; -+ #endif - - seq_printf(m, ": %u [%u]", key->datalen, payload->type); - } -@@ -599,7 +604,11 @@ static void fscrypt_provisioning_key_describe(const struct key *key, - - static void fscrypt_provisioning_key_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - kfree_sensitive(key->payload.data[0]); -+ #endif - } - - static struct key_type key_type_fscrypt_provisioning = { -@@ -641,7 +650,11 @@ static int get_keyring_key(u32 key_id, u32 type, - - if (key->type != &key_type_fscrypt_provisioning) - goto bad_key; -+ #ifdef CONFIG_KEYP -+ payload = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - payload = key->payload.data[0]; -+ #endif - - /* Don't allow fscrypt v1 keys to be used as v2 keys and vice versa. */ - if (payload->type != type) -diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c -index 75dabd9b27f9..9d0cc6ef8e48 100644 ---- a/fs/crypto/keysetup_v1.c -+++ b/fs/crypto/keysetup_v1.c -@@ -109,7 +109,11 @@ find_and_lock_process_key(const char *prefix, - if (IS_ERR(key)) - return key; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - ukp = user_key_payload_locked(key); - - if (!ukp) /* was the key revoked before we acquired its semaphore? */ -@@ -136,7 +140,11 @@ find_and_lock_process_key(const char *prefix, - return key; - - invalid: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - return ERR_PTR(-ENOKEY); - } -@@ -315,7 +323,11 @@ int fscrypt_setup_v1_file_key_via_subscribed_keyrings(struct fscrypt_info *ci) - return PTR_ERR(key); - - err = fscrypt_setup_v1_file_key(ci, payload->raw); -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - return err; - } -diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h -index f2ed0c0266cb..1a24f8dcff5c 100644 ---- a/fs/ecryptfs/ecryptfs_kernel.h -+++ b/fs/ecryptfs/ecryptfs_kernel.h -@@ -82,7 +82,11 @@ ecryptfs_get_encrypted_key_payload_data(struct key *key) - if (key->type != &key_type_encrypted) - return NULL; - -+ #ifdef CONFIG_KEYP -+ payload = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - payload = key->payload.data[0]; -+ #endif - if (!payload) - return ERR_PTR(-EKEYREVOKED); - -diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c -index 7f9f68c00ef6..a4859a7d3b5d 100644 ---- a/fs/ecryptfs/keystore.c -+++ b/fs/ecryptfs/keystore.c -@@ -512,7 +512,11 @@ ecryptfs_find_global_auth_tok_for_sig( - goto out_invalid_auth_tok; - } - -+ #ifdef CONFIG_KEYP -+ down_write(&(KEY_SEM(walker->global_auth_tok_key))); -+ #else - down_write(&(walker->global_auth_tok_key->sem)); -+ #endif - rc = ecryptfs_verify_auth_tok_from_key( - walker->global_auth_tok_key, auth_tok); - if (rc) -@@ -525,7 +529,11 @@ ecryptfs_find_global_auth_tok_for_sig( - rc = -ENOENT; - goto out; - out_invalid_auth_tok_unlock: -+ #ifdef CONFIG_KEYP -+ up_write(&(KEY_SEM(walker->global_auth_tok_key))); -+ #else - up_write(&(walker->global_auth_tok_key->sem)); -+ #endif - out_invalid_auth_tok: - printk(KERN_WARNING "Invalidating auth tok with sig = [%s]\n", sig); - walker->flags |= ECRYPTFS_AUTH_TOK_INVALID; -@@ -846,7 +854,11 @@ ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, - mutex_unlock(s->tfm_mutex); - out: - if (auth_tok_key) { -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - } - skcipher_request_free(s->skcipher_req); -@@ -1088,7 +1100,11 @@ ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, - (*filename) = NULL; - } - if (auth_tok_key) { -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - } - skcipher_request_free(s->skcipher_req); -@@ -1625,10 +1641,18 @@ int ecryptfs_keyring_auth_tok_for_sig(struct key **auth_tok_key, - goto out; - } - } -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM((*auth_tok_key))); -+ #else - down_write(&(*auth_tok_key)->sem); -+ #endif - rc = ecryptfs_verify_auth_tok_from_key(*auth_tok_key, auth_tok); - if (rc) { -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM((*auth_tok_key))); -+ #else - up_write(&(*auth_tok_key)->sem); -+ #endif - key_put(*auth_tok_key); - (*auth_tok_key) = NULL; - goto out; -@@ -1901,7 +1925,11 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - memcpy(&(candidate_auth_tok->token.private_key), - &(matching_auth_tok->token.private_key), - sizeof(struct ecryptfs_private_key)); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - rc = decrypt_pki_encrypted_session_key(candidate_auth_tok, - crypt_stat); -@@ -1909,12 +1937,20 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - memcpy(&(candidate_auth_tok->token.password), - &(matching_auth_tok->token.password), - sizeof(struct ecryptfs_password)); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - rc = decrypt_passphrase_encrypted_session_key( - candidate_auth_tok, crypt_stat); - } else { -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - rc = -EINVAL; - } -@@ -1976,7 +2012,11 @@ pki_encrypt_session_key(struct key *auth_tok_key, - crypt_stat->cipher, - crypt_stat->key_size), - crypt_stat, &payload, &payload_len); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - if (rc) { - ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet\n"); -@@ -2040,7 +2080,11 @@ write_tag_1_packet(char *dest, size_t *remaining_bytes, - memcpy(key_rec->enc_key, - auth_tok->session_key.encrypted_key, - auth_tok->session_key.encrypted_key_size); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - goto encrypted_session_key_set; - } -@@ -2438,7 +2482,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, - &max, auth_tok, - crypt_stat, key_rec, - &written); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - if (rc) { - ecryptfs_printk(KERN_WARNING, "Error " -@@ -2467,7 +2515,11 @@ ecryptfs_generate_key_packet_set(char *dest_base, - } - (*len) += written; - } else { -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(auth_tok_key)); -+ #else - up_write(&(auth_tok_key->sem)); -+ #endif - key_put(auth_tok_key); - ecryptfs_printk(KERN_WARNING, "Unsupported " - "authentication token type\n"); -diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c -index 2dc927ba067f..39c6f1d1438c 100644 ---- a/fs/ecryptfs/main.c -+++ b/fs/ecryptfs/main.c -@@ -202,7 +202,11 @@ static int ecryptfs_init_global_auth_toks( - goto out; - } else { - global_auth_tok->flags &= ~ECRYPTFS_AUTH_TOK_INVALID; -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(global_auth_tok->global_auth_tok_key)); -+ #else - up_write(&(global_auth_tok->global_auth_tok_key)->sem); -+ #endif - } - } - out: -diff --git a/fs/exec.c b/fs/exec.c -index eaec57f79aa1..5688b11ec9d4 100644 ---- a/fs/exec.c -+++ b/fs/exec.c -@@ -76,6 +76,14 @@ - - #include - -+#ifdef CONFIG_CREDP -+#include ++// #define DEBUG ++ ++#define IEE_EARLY_BLOCK_NR 126 ++ ++struct iee_block { ++ phys_addr_t start; ++ unsigned int order; ++}; ++ ++struct iee_early_alloc { ++ phys_addr_t begin; ++ phys_addr_t end; ++ int pos; ++ struct iee_block blocks[IEE_EARLY_BLOCK_NR]; ++ int curr_block_nr; ++ char *name; ++}; ++static struct iee_early_alloc iee_data = { ++ .name = "iee_early_data", ++ .curr_block_nr = -1 ++}; ++static struct iee_early_alloc iee_stack = { ++ .name = "iee_stack", ++ .curr_block_nr = -1 ++}; ++ ++#ifdef CONFIG_PTP ++static struct iee_early_alloc iee_pgtable = { ++ .name = "iee_early_pgtable", ++ .curr_block_nr = -1 ++}; +#endif + -+#ifdef CONFIG_IEE -+#include ++DEFINE_SPINLOCK(swapper_pgdir_lock); ++DEFINE_MUTEX(fixmap_lock); ++ ++__aligned(PAGE_SIZE) DECLARE_PER_CPU(u64*[(PAGE_SIZE/8)], ++ iee_cpu_stack_ptr); ++ ++/* reserve 8 pages for iee init stack. */ ++__aligned(IEE_STACK_SIZE) __initdata u64 iee_init_stack[IEE_STACK_SIZE/8]; ++ ++#ifdef CONFIG_PTP ++void __init iee_init_tcr(void) ++{ ++ unsigned long ptr = pte_set_fixmap_pre_init(__pa_symbol(&kernel_tcr)); ++ *((u64 *)ptr) = read_sysreg(tcr_el1) & IEE_TCR_MASK; ++ pte_clear_fixmap_pre_init(); ++ ptr = pte_set_fixmap_pre_init(__pa_symbol(&iee_tcr)); ++ *((u64 *)ptr) = kernel_tcr | TCR_HPD1 | TCR_A1; ++ pte_clear_fixmap_pre_init(); ++} ++#else ++void __init iee_init_tcr(void) ++{ ++ unsigned long ptr = (unsigned long)(fix_to_virt(FIX_PTE)); ++ ++ __set_fixmap(FIX_PTE, __pa_symbol(&kernel_tcr), FIXMAP_PAGE_NORMAL); ++ ptr += (unsigned long)(&kernel_tcr) & (PAGE_SIZE - 1); ++ *((u64 *)ptr) = read_sysreg(tcr_el1) & IEE_TCR_MASK; ++ clear_fixmap(FIX_PTE); ++ ptr = (unsigned long)(fix_to_virt(FIX_PTE)); ++ __set_fixmap(FIX_PTE, __pa_symbol(&iee_tcr), FIXMAP_PAGE_NORMAL); ++ ptr += (unsigned long)(&iee_tcr) & (PAGE_SIZE - 1); ++ *((u64 *)ptr) = kernel_tcr | TCR_HPD1 | TCR_A1; ++ clear_fixmap(FIX_PTE); ++} +#endif + - static int bprm_creds_from_file(struct linux_binprm *bprm); - - int suid_dumpable = 0; -@@ -733,7 +741,11 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) - free_pgd_range(&tlb, old_start, old_end, new_end, - next ? next->vm_start : USER_PGTABLES_CEILING); - } -+ #ifdef CONFIG_PTP -+ iee_tlb_finish_mmu(&tlb); -+ #else - tlb_finish_mmu(&tlb); -+ #endif - - vma_prev(&vmi); - /* Shrink the vma to just the new range */ -@@ -1023,6 +1035,9 @@ static int exec_mmap(struct mm_struct *mm) - if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) - local_irq_enable(); - activate_mm(active_mm, mm); -+ #ifdef CONFIG_IEE -+ iee_set_token_pgd(tsk, mm->pgd); -+ #endif - if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) - local_irq_enable(); - lru_gen_add_mm(mm); -@@ -1678,12 +1693,20 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) - - if (mode & S_ISUID) { - bprm->per_clear |= PER_CLEAR_ON_SETID; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_euid(bprm->cred, vfsuid_into_kuid(vfsuid)); -+ #else - bprm->cred->euid = vfsuid_into_kuid(vfsuid); -+ #endif - } - - if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { - bprm->per_clear |= PER_CLEAR_ON_SETID; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_egid(bprm->cred, vfsgid_into_kgid(vfsgid)); -+ #else - bprm->cred->egid = vfsgid_into_kgid(vfsgid); -+ #endif - } - } - -diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c -index 3e724cb7ef01..e32e136e4271 100644 ---- a/fs/nfs/flexfilelayout/flexfilelayout.c -+++ b/fs/nfs/flexfilelayout/flexfilelayout.c -@@ -15,6 +15,10 @@ - - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif -+ - #include "flexfilelayout.h" - #include "../nfs4session.h" - #include "../nfs4idmap.h" -@@ -502,8 +506,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, - rc = -ENOMEM; - if (!kcred) - goto out_err_free; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(kcred,uid); -+ iee_set_cred_fsgid(kcred,gid); ++static void __init iee_setup_bootcpu_stack(void) ++{ ++ u64 *cpu_stack_ptr = (u64 *)(SHIFT_PERCPU_PTR(iee_cpu_stack_ptr, ++ __per_cpu_offset[0])); ++ ++ /* Simply use kernel image address here. */ ++ *cpu_stack_ptr = (u64)(&iee_init_stack) + IEE_STACK_SIZE; ++} ++ ++/* Allocate memory block for iee early data pool. */ ++static phys_addr_t __init iee_mem_pool_early_alloc(struct iee_early_alloc *cache, ++ unsigned int order) ++{ ++ phys_addr_t phys = 0; ++ void *ptr; ++ int i; ++ unsigned long block_size = (PAGE_SIZE << (order)); ++ /* Try smaller block if alloc failed. */ ++ while(!phys && order >= IEE_DATA_ORDER) { ++ phys = memblock_phys_alloc_range(block_size, ++ block_size, 0, MEMBLOCK_ALLOC_NOLEAKTRACE); ++ if (!phys) { ++ order--; ++ block_size = (PAGE_SIZE << (order)); ++ } ++ } ++ ++ if (!phys) ++ panic("Failed to allocate %s page\n", cache->name); ++ ++ /* ++ * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE ++ * slot will be free, so we can (ab)use the FIX_PTE slot to initialise ++ * any level of table. ++ */ ++ for (i = 0; i < (1 << (order)); i++) { ++ #ifdef CONFIG_PTP ++ ptr = pte_set_fixmap_pre_init(phys + i * PAGE_SIZE); + #else - kcred->fsuid = uid; - kcred->fsgid = gid; ++ ptr = pte_set_fixmap(phys + i * PAGE_SIZE); + #endif - cred = RCU_INITIALIZER(kcred); - - if (lgr->range.iomode == IOMODE_READ) -diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c -index 25a7c771cfd8..c11c2e78e782 100644 ---- a/fs/nfs/nfs4idmap.c -+++ b/fs/nfs/nfs4idmap.c -@@ -48,6 +48,13 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif -+#ifdef CONFIG_KEYP -+#include -+#endif -+ - #include "internal.h" - #include "netns.h" - #include "nfs4idmap.h" -@@ -225,9 +232,18 @@ int nfs_idmap_init(void) - if (ret < 0) - goto failed_reg_legacy; - -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(cred,keyring); -+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); -+ #else - cred->thread_keyring = keyring; - cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; -+ #endif - id_resolver_cache = cred; - return 0; - -@@ -296,7 +312,11 @@ static struct key *nfs_idmap_request_key(const char *name, size_t namelen, - mutex_unlock(&idmap->idmap_mutex); - } - if (!IS_ERR(rkey)) -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(rkey, KEY_FLAG_ROOT_CAN_INVAL, SET_BIT_OP); ++ ++ memset(ptr, 0, PAGE_SIZE); ++ ++ /* ++ * Implicit barriers also ensure the zeroed page is visible to the page ++ * table walker ++ */ ++ #ifdef CONFIG_PTP ++ pte_clear_fixmap_pre_init(); + #else - set_bit(KEY_FLAG_ROOT_CAN_INVAL, &rkey->flags); ++ pte_clear_fixmap(); + #endif - - kfree(desc); - return rkey; -@@ -321,7 +341,11 @@ static ssize_t nfs_idmap_get_key(const char *name, size_t namelen, - } - - rcu_read_lock(); -+ #ifdef CONFIG_KEYP -+ iee_set_key_perm(rkey, rkey->perm | KEY_USR_VIEW); -+ #else - rkey->perm |= KEY_USR_VIEW; ++ } ++ ++ cache->begin = phys; ++ cache->end = phys + block_size; ++ /* Reset curr free page position. */ ++ cache->pos = 0; ++ cache->curr_block_nr++; ++ if (cache->curr_block_nr > IEE_EARLY_BLOCK_NR) ++ panic("IEE: early data too large."); ++ if (!haoc_bitmap_ready) { ++ /* Record allocated blocks before IEE initialization finishied. */ ++ cache->blocks[cache->curr_block_nr].start = phys; ++ cache->blocks[cache->curr_block_nr].order = order; ++ } ++ #ifdef DEBUG ++ pr_info("IEE: early alloc %s. curr block nr: %d, order %u 0x%lx-0x%lx,", ++ cache->name, cache->curr_block_nr, ++ cache->blocks[cache->curr_block_nr].order, ++ __va(phys), __va(phys + block_size)); + #endif - - ret = key_validate(rkey); - if (ret < 0) -diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c -index e6beaaf4f170..e89385fd81f1 100644 ---- a/fs/nfsd/auth.c -+++ b/fs/nfsd/auth.c -@@ -2,6 +2,9 @@ - /* Copyright (C) 1995, 1996 Olaf Kirch */ - - #include -+#ifdef CONFIG_CREDP -+#include ++ return phys; ++} ++ ++static unsigned long get_size(unsigned long data) ++{ ++ int i = 0; ++ while(data >> i) ++ { ++ i++; ++ } ++ return i + 1; ++} ++ ++/* Calculate the reserved size for early data. */ ++static unsigned int get_iee_alloc_order(int shift) ++{ ++ phys_addr_t start, end; ++ u64 i; ++ unsigned long size = 0; ++ ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; ++ size += (end - start); ++ } ++ return IEE_DATA_ORDER + get_size(size >> 36); ++} ++ ++/* Prepare one block for each early page pool. */ ++void __init early_iee_data_cache_init(void) ++{ ++ unsigned int order; ++ /* Calculate IEE stack alloc block size. */ ++ order = get_order(CONFIG_NR_CPUS * IEE_STACK_SIZE); ++ if (order < IEE_DATA_ORDER) ++ order = IEE_DATA_ORDER; ++ iee_mem_pool_early_alloc(&iee_stack, order); ++ /* Calculate IEE data alloc block size. */ ++ iee_mem_pool_early_alloc(&iee_data, get_iee_alloc_order(1)); ++ #ifdef CONFIG_PTP ++ iee_mem_pool_early_alloc(&iee_pgtable, get_iee_alloc_order(0)); ++ #endif ++} ++ ++phys_addr_t __init iee_early_alloc(struct iee_early_alloc *cache, ++ int order, enum HAOC_BITMAP_TYPE type) ++{ ++ phys_addr_t phys; ++ phys_addr_t expand_phys; ++ unsigned int block_order, expand_order; ++ ++redo: ++ if ((cache->begin + cache->pos * PAGE_SIZE + (PAGE_SIZE << order)) ++ <= cache->end) { ++ phys = cache->begin + cache->pos * PAGE_SIZE; ++ cache->pos += (1 << order); ++ } else { ++ /* Use current order to expand. */ ++ expand_order = cache->blocks[cache->curr_block_nr].order; ++ expand_phys = iee_mem_pool_early_alloc(cache, expand_order); ++ ++ /* Put the expanded memory into IEE, and mark bitmaps if late enough. */ ++ block_order = cache->blocks[cache->curr_block_nr].order; ++ if (haoc_bitmap_ready) ++ set_iee_page((unsigned long)__va(expand_phys), block_order, type); ++ else if (iee_init_done) ++ put_pages_into_iee((unsigned long)__va(expand_phys), block_order); ++ goto redo; ++ } ++ return phys; ++} ++ ++phys_addr_t __init early_iee_stack_alloc(int order) ++{ ++ return iee_early_alloc(&iee_stack, order, IEE_DATA); ++} ++ ++phys_addr_t __init early_iee_data_alloc(int shift) ++{ ++ return iee_early_alloc(&iee_data, 0, IEE_DATA); ++} ++ ++#ifdef CONFIG_PTP ++phys_addr_t __init early_iee_pgtable_alloc(int shift) ++{ ++ return iee_early_alloc(&iee_pgtable, 0, IEE_PGTABLE); ++} +#endif - #include "nfsd.h" - #include "auth.h" - -@@ -32,22 +35,40 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) - if (!new) - return -ENOMEM; - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,rqstp->rq_cred.cr_uid); -+ iee_set_cred_fsgid(new,rqstp->rq_cred.cr_gid); ++ ++void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ pgd_t *fixmap_pgdp; ++ ++ spin_lock(&swapper_pgdir_lock); ++ fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); ++ WRITE_ONCE(*fixmap_pgdp, pgd); ++ /* ++ * We need dsb(ishst) here to ensure the page-table-walker sees ++ * our new entry before set_p?d() returns. The fixmap's ++ * flush_tlb_kernel_range() via clear_fixmap() does this for us. ++ */ ++ pgd_clear_fixmap(); ++ spin_unlock(&swapper_pgdir_lock); ++} ++ ++pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, ++ unsigned long size, pgprot_t vma_prot) ++{ ++ if (!pfn_is_map_memory(pfn)) ++ return pgprot_noncached(vma_prot); ++ else if (file->f_flags & O_SYNC) ++ return pgprot_writecombine(vma_prot); ++ return vma_prot; ++} ++EXPORT_SYMBOL(phys_mem_access_prot); ++ ++phys_addr_t __init early_pgtable_alloc(int shift) ++{ ++ #ifdef CONFIG_PTP ++ return early_iee_pgtable_alloc(shift); + #else - new->fsuid = rqstp->rq_cred.cr_uid; - new->fsgid = rqstp->rq_cred.cr_gid; ++ phys_addr_t phys; ++ void *ptr; ++ ++ phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, ++ MEMBLOCK_ALLOC_NOLEAKTRACE); ++ if (!phys) ++ panic("Failed to allocate page table page\n"); ++ ++ /* ++ * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE ++ * slot will be free, so we can (ab)use the FIX_PTE slot to initialise ++ * any level of table. ++ */ ++ ptr = pte_set_fixmap(phys); ++ ++ memset(ptr, 0, PAGE_SIZE); ++ ++ /* ++ * Implicit barriers also ensure the zeroed page is visible to the page ++ * table walker ++ */ ++ pte_clear_fixmap(); ++ ++ return phys; + #endif - - rqgi = rqstp->rq_cred.cr_group_info; - - if (flags & NFSEXP_ALLSQUASH) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,exp->ex_anon_uid); -+ iee_set_cred_fsgid(new,exp->ex_anon_gid); -+ #else - new->fsuid = exp->ex_anon_uid; - new->fsgid = exp->ex_anon_gid; -+ #endif - gi = groups_alloc(0); - if (!gi) - goto oom; - } else if (flags & NFSEXP_ROOTSQUASH) { - if (uid_eq(new->fsuid, GLOBAL_ROOT_UID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,exp->ex_anon_uid); -+ #else - new->fsuid = exp->ex_anon_uid; -+ #endif - if (gid_eq(new->fsgid, GLOBAL_ROOT_GID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsgid(new,exp->ex_anon_gid); -+ #else - new->fsgid = exp->ex_anon_gid; -+ #endif - - gi = groups_alloc(rqgi->ngroups); - if (!gi) -@@ -67,18 +88,35 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) - } - - if (uid_eq(new->fsuid, INVALID_UID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,exp->ex_anon_uid); -+ #else - new->fsuid = exp->ex_anon_uid; -+ #endif - if (gid_eq(new->fsgid, INVALID_GID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsgid(new,exp->ex_anon_gid); -+ #else - new->fsgid = exp->ex_anon_gid; -+ #endif - - set_groups(new, gi); - put_group_info(gi); - - if (!uid_eq(new->fsuid, GLOBAL_ROOT_UID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,cap_drop_nfsd_set(new->cap_effective)); -+ #else - new->cap_effective = cap_drop_nfsd_set(new->cap_effective); -+ #endif - else -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective, -+ new->cap_permitted)); -+ #else - new->cap_effective = cap_raise_nfsd_set(new->cap_effective, - new->cap_permitted); -+ #endif - put_cred(override_creds(new)); - put_cred(new); - return 0; -diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c -index 49a88dde9631..4fde340b8471 100644 ---- a/fs/nfsd/nfs4callback.c -+++ b/fs/nfsd/nfs4callback.c -@@ -41,6 +41,9 @@ - #include "trace.h" - #include "xdr4cb.h" - #include "xdr4.h" -+#ifdef CONFIG_CREDP -+#include -+#endif - - #define NFSDDBG_FACILITY NFSDDBG_PROC - -@@ -946,8 +949,13 @@ static const struct cred *get_backchannel_cred(struct nfs4_client *clp, struct r - if (!kcred) - return NULL; - -- kcred->fsuid = ses->se_cb_sec.uid; -- kcred->fsgid = ses->se_cb_sec.gid; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_uid(kcred,ses->se_cb_sec.uid); -+ iee_set_cred_gid(kcred,ses->se_cb_sec.gid); -+ #else -+ kcred->uid = ses->se_cb_sec.uid; -+ kcred->gid = ses->se_cb_sec.gid; -+ #endif - return kcred; - } - } -diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c -index 4395577825a7..8c34cfe01075 100644 ---- a/fs/nfsd/nfs4recover.c -+++ b/fs/nfsd/nfs4recover.c -@@ -44,6 +44,10 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif ++} ++ ++bool pgattr_change_is_safe(u64 old, u64 new) ++{ ++ /* ++ * The following mapping attributes may be updated in live ++ * kernel mappings without the need for break-before-make. ++ */ ++ pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG; ++ ++ /* creating or taking down mappings is always safe */ ++ if (!pte_valid(__pte(old)) || !pte_valid(__pte(new))) ++ return true; ++ ++ /* A live entry's pfn should not change */ ++ if (pte_pfn(__pte(old)) != pte_pfn(__pte(new))) ++ return false; ++ ++ /* live contiguous mappings may not be manipulated at all */ ++ if ((old | new) & PTE_CONT) ++ return false; ++ ++ /* Transitioning from Non-Global to Global is unsafe */ ++ if (old & ~new & PTE_NG) ++ return false; ++ ++ /* ++ * Changing the memory type between Normal and Normal-Tagged is safe ++ * since Tagged is considered a permission attribute from the ++ * mismatched attribute aliases perspective. ++ */ ++ if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || ++ (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) && ++ ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) || ++ (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED))) ++ mask |= PTE_ATTRINDX_MASK; ++ ++ return ((old ^ new) & ~mask) == 0; ++} ++ ++static void iee_init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot) ++{ ++ pte_t *ptep; ++ ++ ptep = pte_set_fixmap_offset(pmdp, addr); ++ do { ++ pte_t old_pte = __ptep_get(ptep); ++ ++ __set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); ++ ++ /* ++ * After the PTE entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ IEE_CHECK(!pgattr_change_is_safe(pte_val(old_pte), ++ pte_val(__ptep_get(ptep)))); ++ ++ phys += PAGE_SIZE; ++ } while (ptep++, addr += PAGE_SIZE, addr != end); ++ ++ pte_clear_fixmap(); ++} ++ ++static void iee_alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long next; ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ IEE_CHECK(pmd_sect(pmd)); ++ if (pmd_none(pmd)) { ++ pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; ++ phys_addr_t pte_phys; ++ ++ if (flags & NO_EXEC_MAPPINGS) ++ pmdval |= PMD_TABLE_PXN; ++ IEE_CHECK(!pgtable_alloc); ++ pte_phys = pgtable_alloc(PAGE_SHIFT); ++ __pmd_populate(pmdp, pte_phys, pmdval); ++ pmd = READ_ONCE(*pmdp); ++ } ++ IEE_CHECK(pmd_bad(pmd)); ++ ++ do { ++ pgprot_t __prot = prot; ++ ++ next = pte_cont_addr_end(addr, end); ++ ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ++ ++ iee_init_pte(pmdp, addr, next, phys, __prot); ++ ++ phys += next - addr; ++ } while (addr = next, addr != end); ++} ++ ++static void iee_init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) ++{ ++ unsigned long next; ++ pmd_t *pmdp; ++ ++ pmdp = pmd_set_fixmap_offset(pudp, addr); ++ do { ++ pmd_t old_pmd = READ_ONCE(*pmdp); ++ ++ next = pmd_addr_end(addr, end); ++ ++ /* try section mapping first */ ++ if (((addr | next | phys) & ~PMD_MASK) == 0 && ++ (flags & NO_BLOCK_MAPPINGS) == 0) { ++ pmd_set_huge(pmdp, phys, prot); ++ ++ /* ++ * After the PMD entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ IEE_CHECK(!pgattr_change_is_safe(pmd_val(old_pmd), ++ READ_ONCE(pmd_val(*pmdp)))); ++ } else { ++ iee_alloc_init_cont_pte(pmdp, addr, next, phys, prot, ++ pgtable_alloc, flags); ++ ++ IEE_CHECK(pmd_val(old_pmd) != 0 && ++ pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); ++ } ++ phys += next - addr; ++ } while (pmdp++, addr = next, addr != end); ++ ++ pmd_clear_fixmap(); ++} ++ ++static void iee_alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) ++{ ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); ++ ++ /* ++ * Check for initial section mappings in the pgd/pud. ++ */ ++ IEE_CHECK(pud_sect(pud)); ++ if (pud_none(pud)) { ++ pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; ++ phys_addr_t pmd_phys; ++ ++ if (flags & NO_EXEC_MAPPINGS) ++ pudval |= PUD_TABLE_PXN; ++ IEE_CHECK(!pgtable_alloc); ++ pmd_phys = pgtable_alloc(PMD_SHIFT); ++ __pud_populate(pudp, pmd_phys, pudval); ++ pud = READ_ONCE(*pudp); ++ } ++ IEE_CHECK(pud_bad(pud)); ++ ++ do { ++ pgprot_t __prot = prot; + - #include "nfsd.h" - #include "state.h" - #include "vfs.h" -@@ -78,8 +82,13 @@ nfs4_save_creds(const struct cred **original_creds) - if (!new) - return -ENOMEM; - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,GLOBAL_ROOT_UID); -+ iee_set_cred_fsgid(new,GLOBAL_ROOT_GID); -+ #else - new->fsuid = GLOBAL_ROOT_UID; - new->fsgid = GLOBAL_ROOT_GID; -+ #endif - *original_creds = override_creds(new); - put_cred(new); - return 0; -diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c -index c2495d98c189..779471a0aa66 100644 ---- a/fs/nfsd/nfsfh.c -+++ b/fs/nfsd/nfsfh.c -@@ -16,6 +16,10 @@ - #include "auth.h" - #include "trace.h" - -+#ifdef CONFIG_CREDP -+#include -+#endif ++ next = pmd_cont_addr_end(addr, end); + - #define NFSDDBG_FACILITY NFSDDBG_FH - - -@@ -223,9 +227,14 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) - error = nfserrno(-ENOMEM); - goto out; - } -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective, -+ new->cap_permitted)); -+ #else - new->cap_effective = - cap_raise_nfsd_set(new->cap_effective, - new->cap_permitted); -+ #endif - put_cred(override_creds(new)); - put_cred(new); - } else { -diff --git a/fs/open.c b/fs/open.c -index f9ac703ec1b2..5561a7ece18d 100644 ---- a/fs/open.c -+++ b/fs/open.c -@@ -35,6 +35,11 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + ++ iee_init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); + - #include "internal.h" - - int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry, -@@ -414,17 +419,34 @@ static const struct cred *access_override_creds(void) - * routine. - */ - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(override_cred,override_cred->uid); -+ iee_set_cred_fsgid(override_cred,override_cred->gid); -+ #else - override_cred->fsuid = override_cred->uid; - override_cred->fsgid = override_cred->gid; -+ #endif - - if (!issecure(SECURE_NO_SETUID_FIXUP)) { - /* Clear the capabilities if we switch to a non-root user */ - kuid_t root_uid = make_kuid(override_cred->user_ns, 0); - if (!uid_eq(override_cred->uid, root_uid)) -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = override_cred->cap_effective; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_effective(override_cred, tmp_cap); -+ } while (0); -+ #else - cap_clear(override_cred->cap_effective); -+ #endif - else -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(override_cred,override_cred->cap_permitted); -+ #else - override_cred->cap_effective = - override_cred->cap_permitted; -+ #endif - } - - /* -@@ -444,7 +466,11 @@ static const struct cred *access_override_creds(void) - * expecting RCU freeing. But normal thread-synchronous - * cred accesses will keep things non-RCY. - */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_non_rcu(override_cred,1); -+ #else - override_cred->non_rcu = 1; -+ #endif - - old_cred = override_creds(override_cred); - -diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c -index 54602f0bed8b..58534612dc64 100644 ---- a/fs/overlayfs/dir.c -+++ b/fs/overlayfs/dir.c -@@ -16,6 +16,10 @@ - #include - #include "overlayfs.h" - -+#ifdef CONFIG_CREDP -+#include -+#endif ++ phys += next - addr; ++ } while (addr = next, addr != end); ++} + - static unsigned short ovl_redirect_max = 256; - module_param_named(redirect_max, ovl_redirect_max, ushort, 0644); - MODULE_PARM_DESC(redirect_max, -@@ -590,8 +594,13 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode, - * create a new inode, so just use the ovl mounter's - * fs{u,g}id. - */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(override_cred,inode->i_uid); -+ iee_set_cred_fsgid(override_cred,inode->i_gid); -+ #else - override_cred->fsuid = inode->i_uid; - override_cred->fsgid = inode->i_gid; -+ #endif - err = security_dentry_create_files_as(dentry, - attr->mode, &dentry->d_name, old_cred, - override_cred); -diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c -index 2c056d737c27..9ede99ddb04b 100644 ---- a/fs/overlayfs/super.c -+++ b/fs/overlayfs/super.c -@@ -21,6 +21,10 @@ - #include "overlayfs.h" - #include "params.h" - -+#ifdef CONFIG_CREDP -+#include -+#endif ++static void iee_alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long next; ++ pud_t *pudp; ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); + - MODULE_AUTHOR("Miklos Szeredi "); - MODULE_DESCRIPTION("Overlay filesystem"); - MODULE_LICENSE("GPL"); -@@ -1485,7 +1489,15 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc) - sb->s_export_op = &ovl_export_fid_operations; - - /* Never override disk quota limits or use reserved space */ -+ #ifdef CONFIG_CREDP -+ { -+ kernel_cap_t tmp = cred->cap_effective; -+ cap_lower(tmp, CAP_SYS_RESOURCE); -+ iee_set_cred_cap_effective(cred, tmp); ++ if (p4d_none(p4d)) { ++ p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; ++ phys_addr_t pud_phys; ++ ++ if (flags & NO_EXEC_MAPPINGS) ++ p4dval |= P4D_TABLE_PXN; ++ IEE_CHECK(!pgtable_alloc); ++ pud_phys = pgtable_alloc(PUD_SHIFT); ++ __p4d_populate(p4dp, pud_phys, p4dval); ++ p4d = READ_ONCE(*p4dp); + } -+ #else - cap_lower(cred->cap_effective, CAP_SYS_RESOURCE); -+ #endif - - sb->s_magic = OVERLAYFS_SUPER_MAGIC; - sb->s_xattr = ofs->config.userxattr ? ovl_user_xattr_handlers : -diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c -index af7849e5974f..880e7386bfff 100644 ---- a/fs/smb/client/cifs_spnego.c -+++ b/fs/smb/client/cifs_spnego.c -@@ -18,6 +18,13 @@ - #include "cifs_spnego.h" - #include "cifs_debug.h" - #include "cifsproto.h" -+#ifdef CONFIG_CREDP -+#include -+#endif -+#ifdef CONFIG_KEYP -+#include -+#endif ++ IEE_CHECK(p4d_bad(p4d)); + - static const struct cred *spnego_cred; - - /* create a new cifs key */ -@@ -33,7 +40,12 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) - goto error; - - /* attach the data */ -+ #ifdef CONFIG_KEYP -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[0] = payload; -+ #else - key->payload.data[0] = payload; -+ #endif - ret = 0; - - error: -@@ -43,7 +55,11 @@ cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) - static void - cifs_spnego_key_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ kfree(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - kfree(key->payload.data[0]); -+ #endif - } - - -@@ -163,7 +179,11 @@ cifs_get_spnego_key(struct cifs_ses *sesInfo, - - #ifdef CONFIG_CIFS_DEBUG2 - if (cifsFYI && !IS_ERR(spnego_key)) { -+ #ifdef CONFIG_KEYP -+ struct cifs_spnego_msg *msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; -+ #else - struct cifs_spnego_msg *msg = spnego_key->payload.data[0]; -+ #endif - cifs_dump_mem("SPNEGO reply blob:", msg->data, min(1024U, - msg->secblob_len + msg->sesskey_len)); - } -@@ -211,9 +231,18 @@ init_cifs_spnego(void) - * instruct request_key() to use this special keyring as a cache for - * the results it looks up - */ -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(cred,keyring); -+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); -+ #else - cred->thread_keyring = keyring; - cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; -+ #endif - spnego_cred = cred; - - cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); -diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c -index f5b6df82e857..a51a8b474d4d 100644 ---- a/fs/smb/client/cifsacl.c -+++ b/fs/smb/client/cifsacl.c -@@ -26,6 +26,14 @@ - #include "cifs_fs_sb.h" - #include "cifs_unicode.h" - -+#ifdef CONFIG_CREDP -+#include -+#endif -+#ifdef CONFIG_KEYP -+#include -+#include ++ pudp = pud_set_fixmap_offset(p4dp, addr); ++ do { ++ pud_t old_pud = READ_ONCE(*pudp); ++ ++ next = pud_addr_end(addr, end); ++ ++ iee_alloc_init_cont_pmd(pudp, addr, next, phys, prot, ++ pgtable_alloc, flags); ++ ++ IEE_CHECK(pud_val(old_pud) != 0 && ++ pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); ++ phys += next - addr; ++ } while (pudp++, addr = next, addr != end); ++ ++ pud_clear_fixmap(); ++} ++ ++static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); ++ ++ /* ++ * If the virtual and physical address don't have the same offset ++ * within a page, we cannot map the region as the caller expects. ++ */ ++ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) ++ return; ++ ++ phys &= PAGE_MASK; ++ addr = virt & PAGE_MASK; ++ end = PAGE_ALIGN(virt + size); ++ ++ do { ++ next = pgd_addr_end(addr, end); ++ iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ phys += next - addr; ++ } while (pgdp++, addr = next, addr != end); ++} ++ ++void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ mutex_lock(&fixmap_lock); ++ __create_pgd_mapping_locked(pgdir, phys, virt, size, prot, ++ pgtable_alloc, flags); ++ mutex_unlock(&fixmap_lock); ++} ++ ++#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++extern __alias(__create_pgd_mapping_locked) ++void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, ++ phys_addr_t size, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags); +#endif + - /* security id for everyone/world system group */ - static const struct cifs_sid sid_everyone = { - 1, 1, {0, 0, 0, 0, 0, 1}, {0} }; -@@ -78,16 +86,31 @@ cifs_idmap_key_instantiate(struct key *key, struct key_preparsed_payload *prep) - * dereference payload.data! - */ - if (prep->datalen <= sizeof(key->payload)) { -+ #ifdef CONFIG_KEYP -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[0] = NULL; -+ memcpy(key_payload, prep->data, prep->datalen); -+ #else - key->payload.data[0] = NULL; - memcpy(&key->payload, prep->data, prep->datalen); -+ #endif - } else { - payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL); - if (!payload) - return -ENOMEM; -+ #ifdef CONFIG_KEYP -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[0] = payload; -+ #else - key->payload.data[0] = payload; -+ #endif - } - -+ #ifdef CONFIG_KEYP -+ iee_set_key_datalen(key, prep->datalen); -+ #else - key->datalen = prep->datalen; -+ #endif - return 0; - } - -@@ -95,7 +118,11 @@ static inline void - cifs_idmap_key_destroy(struct key *key) - { - if (key->datalen > sizeof(key->payload)) -+ #ifdef CONFIG_KEYP -+ kfree(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - kfree(key->payload.data[0]); -+ #endif - } - - static struct key_type cifs_idmap_key_type = { -@@ -311,9 +338,15 @@ id_to_sid(unsigned int cid, uint sidtype, struct cifs_sid *ssid) - * there are no subauthorities and the host has 8-byte pointers, then - * it could be. - */ -+ #ifdef CONFIG_KEYP -+ ksid = sidkey->datalen <= sizeof(sidkey->payload) ? -+ (struct cifs_sid *)(sidkey->name_link.next) : -+ (struct cifs_sid *)((union key_payload *)(sidkey->name_link.next))->data[0]; -+ #else - ksid = sidkey->datalen <= sizeof(sidkey->payload) ? - (struct cifs_sid *)&sidkey->payload : - (struct cifs_sid *)sidkey->payload.data[0]; -+ #endif - - ksid_size = CIFS_SID_BASE_SIZE + (ksid->num_subauth * sizeof(__le32)); - if (ksid_size > sidkey->datalen) { -@@ -422,14 +455,22 @@ sid_to_id(struct cifs_sb_info *cifs_sb, struct cifs_sid *psid, - if (sidtype == SIDOWNER) { - kuid_t uid; - uid_t id; -+ #ifdef CONFIG_KEYP -+ memcpy(&id, &((union key_payload *)(sidkey->name_link.next))->data[0], sizeof(uid_t)); -+ #else - memcpy(&id, &sidkey->payload.data[0], sizeof(uid_t)); -+ #endif - uid = make_kuid(&init_user_ns, id); - if (uid_valid(uid)) - fuid = uid; - } else { - kgid_t gid; - gid_t id; -+ #ifdef CONFIG_KEYP -+ memcpy(&id, &((union key_payload *)(sidkey->name_link.next))->data[0], sizeof(gid_t)); -+ #else - memcpy(&id, &sidkey->payload.data[0], sizeof(gid_t)); -+ #endif - gid = make_kgid(&init_user_ns, id); - if (gid_valid(gid)) - fgid = gid; -@@ -490,9 +531,18 @@ init_cifs_idmap(void) - - /* instruct request_key() to use this special keyring as a cache for - * the results it looks up */ -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(cred,keyring); -+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); -+ #else - cred->thread_keyring = keyring; - cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; -+ #endif - root_cred = cred; - - cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); -diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c -index 1df0a6edcc21..c67e1b13ed0c 100644 ---- a/fs/smb/client/connect.c -+++ b/fs/smb/client/connect.c -@@ -2149,7 +2149,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) - is_domain = 1; - } - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - upayload = user_key_payload_locked(key); - if (IS_ERR_OR_NULL(upayload)) { - rc = upayload ? PTR_ERR(upayload) : -EINVAL; -@@ -2227,7 +2231,11 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) - strscpy(ctx->workstation_name, ses->workstation_name, sizeof(ctx->workstation_name)); - - out_key_put: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - key_put(key); - out_err: - kfree(desc); -diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c -index 3216f786908f..3160e70b820f 100644 ---- a/fs/smb/client/sess.c -+++ b/fs/smb/client/sess.c -@@ -1591,7 +1591,11 @@ sess_auth_kerberos(struct sess_data *sess_data) - goto out; - } - -+ #ifdef CONFIG_KEYP -+ msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; -+ #else - msg = spnego_key->payload.data[0]; -+ #endif - /* - * check version field to make sure that cifs.upcall is - * sending us a response in an expected form -diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c -index a86a3fbfb5a4..dbbf0f2e9780 100644 ---- a/fs/smb/client/smb2pdu.c -+++ b/fs/smb/client/smb2pdu.c -@@ -1629,7 +1629,11 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) - goto out; - } - -+ #ifdef CONFIG_KEYP -+ msg = ((union key_payload *)(spnego_key->name_link.next))->data[0]; -+ #else - msg = spnego_key->payload.data[0]; -+ #endif - /* - * check version field to make sure that cifs.upcall is - * sending us a response in an expected form -diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c -index e564d5ff8781..2d3ca5951fa2 100644 ---- a/fs/ubifs/auth.c -+++ b/fs/ubifs/auth.c -@@ -284,7 +284,11 @@ int ubifs_init_authentication(struct ubifs_info *c) - return PTR_ERR(keyring_key); - } - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(keyring_key)); -+ #else - down_read(&keyring_key->sem); -+ #endif - - if (keyring_key->type != &key_type_logon) { - ubifs_err(c, "key type must be logon"); -@@ -351,7 +355,11 @@ int ubifs_init_authentication(struct ubifs_info *c) - if (err) - crypto_free_shash(c->hash_tfm); - out: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(keyring_key)); -+ #else - up_read(&keyring_key->sem); -+ #endif - key_put(keyring_key); - - return err; -diff --git a/fs/verity/signature.c b/fs/verity/signature.c -index 90c07573dd77..e076cb6fbc84 100644 ---- a/fs/verity/signature.c -+++ b/fs/verity/signature.c -@@ -62,7 +62,11 @@ int fsverity_verify_signature(const struct fsverity_info *vi, - return 0; - } - -+ #ifdef CONFIG_KEYP -+ if (((struct key_struct *)(fsverity_keyring->name_link.prev))->keys.nr_leaves_on_tree == 0) { ++phys_addr_t __pgd_pgtable_alloc(int shift) ++{ ++ #ifdef CONFIG_PTP ++ void *ptr = ptp_pg_alloc(&pg_cache, GFP_PGTABLE_KERNEL); + #else - if (fsverity_keyring->keys.nr_leaves_on_tree == 0) { ++ void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); + #endif - /* - * The ".fs-verity" keyring is empty, due to builtin signatures - * being supported by the kernel but not actually being used. -diff --git a/include/asm-generic/early_ioremap.h b/include/asm-generic/early_ioremap.h -index 9d0479f50f97..f501e0f965f8 100644 ---- a/include/asm-generic/early_ioremap.h -+++ b/include/asm-generic/early_ioremap.h -@@ -17,6 +17,9 @@ extern void *early_memremap_ro(resource_size_t phys_addr, - extern void *early_memremap_prot(resource_size_t phys_addr, - unsigned long size, unsigned long prot_val); - extern void early_iounmap(void __iomem *addr, unsigned long size); -+#ifdef CONFIG_PTP -+extern void early_iounmap_after_init(void __iomem *addr, unsigned long size); -+#endif - extern void early_memunmap(void *addr, unsigned long size); - - #if defined(CONFIG_GENERIC_EARLY_IOREMAP) && defined(CONFIG_MMU) -diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h -index 8cc7b09c1bc7..83158589a545 100644 ---- a/include/asm-generic/fixmap.h -+++ b/include/asm-generic/fixmap.h -@@ -70,6 +70,24 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) - __set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR) - #endif - -+#ifdef CONFIG_PTP -+#ifndef clear_fixmap_init -+#define clear_fixmap_init(idx) \ -+ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR) -+#endif ++ IEE_CHECK(!ptr); ++ ++ /* Ensure the zeroed page is visible to the page table walker */ ++ dsb(ishst); ++ return __pa(ptr); ++} ++ ++phys_addr_t pgd_pgtable_alloc(int shift) ++{ ++ phys_addr_t pa = __pgd_pgtable_alloc(shift); ++ struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); ++ ++ /* ++ * Call proper page table ctor in case later we need to ++ * call core mm functions like apply_to_page_range() on ++ * this pre-allocated page table. ++ * ++ * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is ++ * folded, and if so pagetable_pte_ctor() becomes nop. ++ */ ++ if (shift == PAGE_SHIFT) ++ IEE_CHECK(!pagetable_pte_ctor(ptdesc)); ++ else if (shift == PMD_SHIFT) ++ IEE_CHECK(!pagetable_pmd_ctor(ptdesc)); ++ ++ return pa; ++} ++ ++static void __init __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); ++ p4d_t *p4dp; ++ p4d_t p4d; + -+#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \ -+({ \ -+ unsigned long ________addr; \ -+ __iee_set_fixmap_pre_init(idx, phys, flags); \ -+ ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \ -+ ________addr; \ -+}) ++ /* ++ * If the virtual and physical address don't have the same offset ++ * within a page, we cannot map the region as the caller expects. ++ */ ++ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) ++ return; + -+#define iee_set_fixmap_offset_pre_init(idx, phys) \ -+ __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL) -+#endif ++ phys &= PAGE_MASK; ++ addr = virt & PAGE_MASK; ++ end = PAGE_ALIGN(virt + size); + - /* Return a pointer with offset calculated */ - #define __set_fixmap_offset(idx, phys, flags) \ - ({ \ -diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h -index 6796abe1900e..725cae1dbd02 100644 ---- a/include/asm-generic/memory_model.h -+++ b/include/asm-generic/memory_model.h -@@ -64,6 +64,26 @@ static inline int pfn_valid(unsigned long pfn) - #define page_to_pfn __page_to_pfn - #define pfn_to_page __pfn_to_page - -+#ifdef CONFIG_PTP -+#ifdef CONFIG_ARM64 -+#define IEE_PTDESC_MAP_START VMEMMAP_END -+#define iee_ptdesc_base ((struct ptdesc**)IEE_PTDESC_MAP_START - (memstart_addr >> PAGE_SHIFT)) ++ do { ++ next = pgd_addr_end(addr, end); ++ #ifdef CONFIG_PTP ++ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ #else ++ iee_alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ #endif + -+#define __pfn_to_ptdesc(pfn) (iee_ptdesc_base + (pfn)) -+#define __ptdesc_to_pfn(ptdesc) (unsigned long)((ptdesc) - iee_ptdesc_base) -+#else -+#define __pfn_to_ptdesc(pfn) (((struct ptdesc_t **)iee_ptdesc_base + (pfn))) -+#define __ptdesc_to_pfn(ptdesc) (unsigned long)((ptdesc) - (struct ptdesc_t **)iee_ptdesc_base) -+#endif ++ /* Set APTable RO on pgd entries of IEE mappings to prevent kernel access ++ * when TCR.HPD1 == 0. ++ */ ++ p4dp = p4d_offset(pgdp, addr); ++ p4d = READ_ONCE(*p4dp); ++ #ifdef CONFIG_PTP ++ iee_set_pgtable_pre_init((unsigned long *)p4dp, ++ (unsigned long)(__phys_to_p4d_val(__p4d_to_phys(p4d)) ++ | (PGD_APTABLE_RO | PGD_PXNTABLE | PGD_UXNTABLE | PUD_TYPE_TABLE))); ++ #else ++ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APTABLE_RO | PGD_PXNTABLE | ++ PGD_UXNTABLE | PUD_TYPE_TABLE)); ++ #endif + -+#define ptdesc_to_pfn __ptdesc_to_pfn -+#define pfn_to_ptdesc __pfn_to_ptdesc ++ phys += next - addr; ++ } while (pgdp++, addr = next, addr != end); ++} + -+#define iee_ptdesc(ptdesc) (*(pfn_to_ptdesc(page_to_pfn((struct page*)ptdesc)))) -+#define page_to_iee_ptdesc(page) (*(pfn_to_ptdesc(page_to_pfn(page)))) -+#define iee_ptdesc_to_page(ptdesc) (((struct ptdesc_t *)ptdesc)->page) -+#endif ++static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ mutex_lock(&fixmap_lock); ++ __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot, ++ pgtable_alloc, flags); ++ mutex_unlock(&fixmap_lock); ++} + - #endif /* __ASSEMBLY__ */ - - #endif -diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h -index c75d4a753849..1d86942b242c 100644 ---- a/include/asm-generic/pgalloc.h -+++ b/include/asm-generic/pgalloc.h -@@ -7,6 +7,10 @@ - #define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO) - #define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT) - -+#ifdef CONFIG_PTP -+#include -+#endif ++static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start, ++ phys_addr_t end, pgprot_t prot, int flags) ++{ ++ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, ++ prot, early_pgtable_alloc, flags); ++} + - /** - * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table - * @mm: the mm_struct of the current context -@@ -23,6 +27,7 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) - - if (!ptdesc) - return NULL; ++/* ++ * First function in IEE initialization. Create IEE linear mappings inside ++ * kernel address space to access the protected objects. ++ */ ++void __init iee_init_mappings(pgd_t *pgdp) ++{ ++ phys_addr_t start, end; ++ int flags = NO_EXEC_MAPPINGS; ++ u64 i; + - return ptdesc_address(ptdesc); - } - -@@ -106,6 +111,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) - struct ptdesc *ptdesc = page_ptdesc(pte_page); - - pagetable_pte_dtor(ptdesc); ++ /* Check if hardware supports IEE. */ ++ if (!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), ++ ID_AA64MMFR1_EL1_HPDS_SHIFT)) ++ panic("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n"); ++ else ++ pr_info("HAOC: ARM64 hardware support detected."); + - pagetable_free(ptdesc); - } - -@@ -149,6 +155,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) - - BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); - pagetable_pmd_dtor(ptdesc); ++ /* ++ * Not allowing block or continuous mappings on IEE for faster page ++ * attribution modification. ++ */ ++ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + - pagetable_free(ptdesc); - } - #endif -diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h -index 03b7dae47dd4..ef774115704f 100644 ---- a/include/asm-generic/pgtable-nop4d.h -+++ b/include/asm-generic/pgtable-nop4d.h -@@ -26,6 +26,11 @@ static inline void pgd_clear(pgd_t *pgd) { } - - #define pgd_populate(mm, pgd, p4d) do { } while (0) - #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) ++ /* map all the memory banks non-executable but invalid on iee addresses. */ ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; ++ __map_memblock_for_iee(pgdp, start, end, SET_NG(SET_INVALID(PAGE_KERNEL)), ++ flags); ++ } + -+#ifdef CONFIG_PTP -+#define iee_pgd_populate_pre_init(mm, pgd, p4d) do { } while (0) -+#define iee_pgd_populate_safe_pre_init(mm, pgd, p4d) do { } while (0) -+#endif - /* - * (p4ds are folded into pgds so this doesn't get actually called, - * but the define is needed for a generic inline function.) -diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h -index 22384baee10e..fd0a18fe1748 100644 ---- a/include/asm-generic/tlb.h -+++ b/include/asm-generic/tlb.h -@@ -209,6 +209,9 @@ struct mmu_table_batch { - ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) - - extern void tlb_remove_table(struct mmu_gather *tlb, void *table); -+#ifdef CONFIG_PTP -+extern void iee_tlb_remove_table(struct mmu_gather *tlb, void *table); -+#endif - - #else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */ - -@@ -361,7 +364,9 @@ struct mmu_gather { - }; - - void tlb_flush_mmu(struct mmu_gather *tlb); -- -+#ifdef CONFIG_PTP -+void iee_tlb_flush_mmu(struct mmu_gather *tlb); -+#endif - static inline void __tlb_adjust_range(struct mmu_gather *tlb, - unsigned long address, - unsigned int range_size) -@@ -476,6 +481,15 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb, - tlb_flush_mmu(tlb); - } - -+#ifdef CONFIG_PTP -+static inline void iee_tlb_remove_page_size(struct mmu_gather *tlb, -+ struct page *page, int page_size) -+{ -+ if (__tlb_remove_page_size(tlb, page, false, page_size)) -+ iee_tlb_flush_mmu(tlb); ++ iee_init_tcr(); ++ iee_setup_bootcpu_stack(); ++ #ifdef CONFIG_PTP ++ pr_info("HAOC: CONFIG_PTP enabled."); ++ #endif +} -+#endif + - static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb, - struct page *page, bool delay_rmap) - { -@@ -491,9 +505,19 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) - return tlb_remove_page_size(tlb, page, PAGE_SIZE); - } - -+#ifdef CONFIG_PTP -+static inline void iee_tlb_remove_page(struct mmu_gather *tlb, struct page *page) ++static void setup_iee_data_cache_bitmap(struct iee_early_alloc *cache, ++ enum HAOC_BITMAP_TYPE type) +{ -+ return iee_tlb_remove_page_size(tlb, page, PAGE_SIZE); ++ int block_nr = cache->curr_block_nr + 1; ++ ++ for (int j = 0; j < block_nr; j++) { ++ iee_set_bitmap_type((unsigned long)__va(cache->blocks[j].start), ++ 1 << cache->blocks[j].order, type); ++ } ++ #ifdef DEBUG ++ pr_info("IEE: Mark bitmap of %s block nr %d", cache->name, block_nr); ++ #endif +} -+#endif - static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) - { ++ ++void __init setup_iee_early_data_bitmap(void) ++{ ++ setup_iee_data_cache_bitmap(&iee_data, IEE_DATA); ++ setup_iee_data_cache_bitmap(&iee_stack, IEE_DATA); + #ifdef CONFIG_PTP -+ iee_tlb_remove_table(tlb, pt); -+ #else - tlb_remove_table(tlb, pt); ++ setup_iee_data_cache_bitmap(&iee_pgtable, IEE_PGTABLE); + #endif - } - - /* Like tlb_remove_ptdesc, but for page-like page directories. */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index 174d865ce46e..0d945cdd78e0 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -379,6 +379,17 @@ - KEEP(*(.dtb.init.rodata)) \ - __dtb_end = .; - -+#ifdef CONFIG_KOI -+#define KOI_DATA() \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_data_start = .; \ -+ *(.data..koi) \ -+ . = ALIGN(PAGE_SIZE); \ -+ __koi_data_end = .; -+#else -+#define KOI_DATA() -+#endif ++} + - /* - * .data section - */ -@@ -403,8 +414,8 @@ - BRANCH_PROFILE() \ - TRACE_PRINTKS() \ - BPF_RAW_TP() \ -- TRACEPOINT_STR() -- -+ TRACEPOINT_STR() \ -+ KOI_DATA() - /* - * Data section helpers - */ -@@ -1126,6 +1137,23 @@ - * They will fit only a subset of the architectures - */ - -+#ifdef CONFIG_CREDP -+ #define CRED_DATA \ -+ . = ALIGN(PAGE_SIZE); \ -+ *(.iee.cred) \ -+ . = ALIGN(PAGE_SIZE); -+#else -+ #define CRED_DATA ++#ifdef CONFIG_PTP ++static void setup_iee_early_address(struct iee_early_alloc *cache) ++{ ++ for (int j = 0; j < cache->curr_block_nr+1; j++) { ++ for (int i = 0; i < (1 << cache->blocks[j].order); i++) { ++ set_iee_address_pre_init(__phys_to_iee(cache->blocks[j].start ++ + i * PAGE_SIZE), true); ++ } ++ } ++} +#endif + -+#ifdef CONFIG_IEE_SELINUX_P -+ #define IEE_SELINUX_DATA \ -+ . = ALIGN(PAGE_SIZE); \ -+ *(.iee.selinux) \ -+ . = ALIGN(PAGE_SIZE); -+#else -+ #define IEE_SELINUX_DATA -+#endif - - /* - * Writeable data. -@@ -1143,6 +1171,8 @@ - . = ALIGN(PAGE_SIZE); \ - .data : AT(ADDR(.data) - LOAD_OFFSET) { \ - INIT_TASK_DATA(inittask) \ -+ CRED_DATA \ -+ IEE_SELINUX_DATA \ - NOSAVE_DATA \ - PAGE_ALIGNED_DATA(pagealigned) \ - CACHELINE_ALIGNED_DATA(cacheline) \ -diff --git a/include/keys/asymmetric-subtype.h b/include/keys/asymmetric-subtype.h -index d55171f640a0..1293c5a1c624 100644 ---- a/include/keys/asymmetric-subtype.h -+++ b/include/keys/asymmetric-subtype.h -@@ -54,7 +54,11 @@ struct asymmetric_key_subtype { - static inline - struct asymmetric_key_subtype *asymmetric_key_subtype(const struct key *key) - { -+ #ifdef CONFIG_KEYP -+ return ((union key_payload *)(key->name_link.next))->data[asym_subtype]; -+ #else - return key->payload.data[asym_subtype]; -+ #endif - } - - #endif /* _KEYS_ASYMMETRIC_SUBTYPE_H */ -diff --git a/include/keys/asymmetric-type.h b/include/keys/asymmetric-type.h -index 69a13e1e5b2e..6cd556bfb216 100644 ---- a/include/keys/asymmetric-type.h -+++ b/include/keys/asymmetric-type.h -@@ -69,13 +69,21 @@ extern struct asymmetric_key_id *asymmetric_key_generate_id(const void *val_1, - static inline - const struct asymmetric_key_ids *asymmetric_key_ids(const struct key *key) - { -+ #ifdef CONFIG_KEYP -+ return ((union key_payload *)(key->name_link.next))->data[asym_key_ids]; -+ #else - return key->payload.data[asym_key_ids]; -+ #endif - } - - static inline - const struct public_key *asymmetric_key_public_key(const struct key *key) - { -+ #ifdef CONFIG_KEYP -+ return ((union key_payload *)(key->name_link.next))->data[asym_crypto]; -+ #else - return key->payload.data[asym_crypto]; ++static void prot_iee_early_data_cache(struct iee_early_alloc *cache) ++{ ++ int block_nr = cache->curr_block_nr + 1; ++ ++ for (int j = 0; j < block_nr; j++) { ++ put_pages_into_iee((unsigned long)__va(cache->blocks[j].start), ++ cache->blocks[j].order); ++ } ++ #ifdef DEBUG ++ pr_info("IEE: Prot %s block nr %d", cache->name, cache->curr_block_nr+1); + #endif - } - - extern struct key *find_asymmetric_key(struct key *keyring, -diff --git a/include/keys/request_key_auth-type.h b/include/keys/request_key_auth-type.h -index 36b89a933310..63d5d9f66cb4 100644 ---- a/include/keys/request_key_auth-type.h -+++ b/include/keys/request_key_auth-type.h -@@ -26,7 +26,11 @@ struct request_key_auth { - - static inline struct request_key_auth *get_request_key_auth(const struct key *key) - { -+ #ifdef CONFIG_KEYP -+ return ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - return key->payload.data[0]; ++} ++ ++/* Put early allocated pages into IEE. */ ++void __init init_early_iee_data(void) ++{ ++ u64 i; ++ ++ #ifdef CONFIG_PTP ++ /* Setup iee mappings of early allocated IEE objects to enable IEE. */ ++ for (i = 0; ((unsigned long)idmap_pg_dir + i * PAGE_SIZE) < ++ (unsigned long)iee_init_data_end; i++) { ++ set_iee_address_pre_init(__phys_to_iee(__pa_symbol((unsigned long)idmap_pg_dir ++ + i * PAGE_SIZE)), true); ++ } ++ setup_iee_early_address(&iee_pgtable); ++ setup_iee_early_address(&iee_data); ++ ++ prot_iee_early_data_cache(&iee_pgtable); + #endif - } - - -diff --git a/include/linux/bpf.h b/include/linux/bpf.h -index 73ef8dda7ada..29b9ced9443a 100644 ---- a/include/linux/bpf.h -+++ b/include/linux/bpf.h -@@ -304,6 +304,17 @@ struct bpf_map { - bool free_after_mult_rcu_gp; - KABI_FILL_HOLE(bool free_after_rcu_gp) - s64 __percpu *elem_count; -+#ifdef CONFIG_HIVE -+ /* fbpf add for aggregation */ -+ bool is_aggregated; -+ int used_page_cnt; -+ union { -+ struct bpf_used_page *used_pages; -+ void *value; -+ }; -+ void *shadow_data; -+ struct mutex used_page_mutex; -+#endif - - KABI_USE(1, atomic64_t sleepable_refcnt) - KABI_USE(2, const struct btf_type *attach_func_proto) -@@ -1573,6 +1584,13 @@ struct bpf_prog { - const struct bpf_insn *insn); - struct bpf_prog_aux *aux; /* Auxiliary fields */ - struct sock_fprog_kern *orig_prog; /* Original BPF program */ -+#ifdef CONFIG_HIVE -+ void *shadow_stack_addr, *shadow_region_addr, *shadow_top_addr, -+ *shadow_skb_addr, *low_guard_region_addr, *high_guard_region_addr, -+ *map_data_addr; -+ int shadow_stack_page_cnt, total_page_cnt, shadow_skb_page_cnt, -+ guard_region_page_cnt, map_page_cnt; -+#endif - /* Instructions for interpreter */ - union { - DECLARE_FLEX_ARRAY(struct sock_filter, insns); -diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h -index 1447d41474f5..ad3922626b0e 100644 ---- a/include/linux/bpf_verifier.h -+++ b/include/linux/bpf_verifier.h -@@ -10,6 +10,10 @@ - #include - #include - -+#ifdef CONFIG_HIVE -+#include -+#endif -+ - /* Maximum variable offset umax_value permitted when resolving memory accesses. - * In practice this is far bigger than any realistic pointer offset; this limit - * ensures that umax_value + (int)off + (int)size cannot overflow a u64. -@@ -226,6 +230,11 @@ struct bpf_reg_state { - enum bpf_reg_liveness live; - /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; -+#ifdef CONFIG_HIVE -+ /* used to record whether the reg is checked */ -+ // bool sfi_checked; -+ enum bpf_sfi_check_state sfi_check_state; -+#endif - - KABI_RESERVE(1) - KABI_RESERVE(2) -@@ -565,6 +574,12 @@ struct bpf_insn_aux_data { - bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */ - bool is_iter_next; /* bpf_iter__next() kfunc call */ - u8 alu_state; /* used in combination with alu_limit */ -+#ifdef CONFIG_HIVE -+ /* fbpf record for patch isnns */ -+ enum bpf_reg_type arg_reg_type[5]; -+ u64 extra_map_ptr; -+ // struct bpf_sfi_check_unit *sfi_check_list; -+#endif - - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ -@@ -693,6 +708,10 @@ struct bpf_verifier_env { - bool bypass_spec_v4; - bool seen_direct_write; - struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ -+#ifdef CONFIG_HIVE -+ struct bpf_sfi_check_unit **insn_check_lists; /* array of per-insn check_list */ -+ struct mutex insn_check_list_mutex; -+#endif - const struct bpf_line_info *prev_linfo; - struct bpf_verifier_log log; - struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; -diff --git a/include/linux/cred.h b/include/linux/cred.h -index e01c6d094a30..75db7ea97ab6 100644 ---- a/include/linux/cred.h -+++ b/include/linux/cred.h -@@ -18,6 +18,10 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif + - struct cred; - struct inode; - -@@ -153,6 +157,22 @@ struct cred { - KABI_RESERVE(4) - } __randomize_layout; - -+#ifdef CONFIG_CREDP -+extern unsigned long long iee_rw_gate(int flag, ...); -+static void iee_set_cred_non_rcu(struct cred *cred, int non_rcu) ++ /* Now we can use IEE APIs as IEE mappings is valid. */ ++ prot_iee_early_data_cache(&iee_data); ++ prot_iee_early_data_cache(&iee_stack); ++} ++ ++#ifdef CONFIG_PTP ++void __init fixmap_copy(pgd_t *pgdir) +{ -+ iee_rw_gate(IEE_OP_SET_CRED_NON_RCU,cred,non_rcu); -+ *(int *)(&(((struct rcu_head *)(cred->rcu.func))->next)) = non_rcu; ++ if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdir, FIXADDR_TOT_START)))) { ++ /* ++ * The fixmap falls in a separate pgd to the kernel, and doesn't ++ * live in the carveout for the swapper_pg_dir. We can simply ++ * re-use the existing dir for the fixmap. ++ */ ++ iee_set_pgd_pre_init(pgd_offset_pgd(pgdir, FIXADDR_TOT_START), ++ READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START))); ++ } else if (CONFIG_PGTABLE_LEVELS > 3) { ++ pgd_t *bm_pgdp; ++ p4d_t *bm_p4dp; ++ pud_t *bm_pudp; ++ pudval_t pudval; ++ /* ++ * The fixmap shares its top level pgd entry with the kernel ++ * mapping. This can really only occur when we are running ++ * with 16k/4 levels, so we can simply reuse the pud level ++ * entry instead. ++ */ ++ IEE_CHECK(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); ++ bm_pgdp = pgd_offset_pgd(pgdir, FIXADDR_TOT_START); ++ bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_TOT_START); ++ bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_TOT_START); ++ pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; ++ pudval |= PUD_TABLE_UXN; ++ iee_set_pgtable_pre_init((unsigned long *)bm_pudp, ++ (unsigned long)(__phys_to_pud_val(__pa_symbol(bm_pmd)) | pudval)); ++ pud_clear_fixmap_pre_init(); ++ } else { ++ BUG(); ++ } +} + -+static bool iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr) ++int __pmdp_set_access_flags(struct vm_area_struct *vma, ++ unsigned long address, pmd_t *pmdp, ++ pmd_t entry, int dirty) +{ -+ bool ret; -+ ret = iee_rw_gate(IEE_OP_SET_CRED_ATOP_USAGE,cred,flag,nr); -+ return ret; ++ pmdval_t old_pmdval, pmdval; ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ if (pmd_same(pmd, entry)) ++ return 0; ++ ++ /* only preserve the access flags and write permission */ ++ pmd_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; ++ ++ /* ++ * Setting the flags must be done atomically to avoid racing with the ++ * hardware update of the access/dirty state. The PTE_RDONLY bit must ++ * be set to the most permissive (lowest value) of *ptep and entry ++ * (calculated as: a & b == ~(~a | ~b)). ++ */ ++ pmd_val(entry) ^= PTE_RDONLY; ++ pmdval = pmd_val(pmd); ++ do { ++ old_pmdval = pmdval; ++ pmdval ^= PTE_RDONLY; ++ pmdval |= pmd_val(entry); ++ pmdval ^= PTE_RDONLY; ++ pmdval = iee_set_pmd_cmpxchg_relaxed(pmdp, old_pmdval, pmdval); ++ } while (pmdval != old_pmdval); ++ ++ /* Invalidate a stale read-only entry */ ++ if (dirty) ++ flush_tlb_page(vma, address); ++ return 1; +} -+#endif + - extern void __put_cred(struct cred *); - extern void exit_creds(struct task_struct *); - extern int copy_creds(struct task_struct *, unsigned long); -@@ -189,7 +209,11 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred) - */ - static inline struct cred *get_new_cred_many(struct cred *cred, int nr) - { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_atomic_op_usage(cred, AT_ADD, nr); -+ #else - atomic_long_add(nr, &cred->usage); -+ #endif - return cred; - } - -@@ -202,7 +226,7 @@ static inline struct cred *get_new_cred_many(struct cred *cred, int nr) - */ - static inline struct cred *get_new_cred(struct cred *cred) - { -- return get_new_cred_many(cred, 1); -+ return get_new_cred_many(cred, 1); // XXXzgc atomic_inc -> get_new_cred_many - } - - /** -@@ -224,7 +248,11 @@ static inline const struct cred *get_cred_many(const struct cred *cred, int nr) - struct cred *nonconst_cred = (struct cred *) cred; - if (!cred) - return cred; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_non_rcu(nonconst_cred,0); -+ #else - nonconst_cred->non_rcu = 0; -+ #endif - return get_new_cred_many(nonconst_cred, nr); - } - -@@ -247,9 +275,19 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred) - struct cred *nonconst_cred = (struct cred *) cred; - if (!cred) - return NULL; -+ #ifdef CONFIG_CREDP -+ if (!iee_set_cred_atomic_op_usage(nonconst_cred,AT_INC_NOT_ZERO,0)) -+ return NULL; -+ #else - if (!atomic_long_inc_not_zero(&nonconst_cred->usage)) - return NULL; -+ #endif ++void * __ref __ptp_vmemmap_alloc_block(unsigned long size, int node) ++{ ++ int order = get_order(size); ++ ++ /* If the main allocator is up use that, fallback to bootmem. */ ++ if (slab_is_available()) ++ return ptp_pg_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); + -+ #ifdef CONFIG_CREDP -+ iee_set_cred_non_rcu(nonconst_cred,0); -+ #else - nonconst_cred->non_rcu = 0; -+ #endif - return cred; - } - -@@ -270,8 +308,13 @@ static inline void put_cred_many(const struct cred *_cred, int nr) - struct cred *cred = (struct cred *) _cred; - - if (cred) { -+ #ifdef CONFIG_CREDP -+ if (iee_set_cred_atomic_op_usage(cred,AT_SUB_AND_TEST,nr)) -+ __put_cred(cred); -+ #else - if (atomic_long_sub_and_test(nr, &cred->usage)) - __put_cred(cred); -+ #endif - } - } - -diff --git a/include/linux/efi.h b/include/linux/efi.h -index 9ed79128458c..970cc4f7068b 100644 ---- a/include/linux/efi.h -+++ b/include/linux/efi.h -@@ -740,6 +740,15 @@ extern int __init __efi_memmap_init(struct efi_memory_map_data *data); - extern int __init efi_memmap_init_early(struct efi_memory_map_data *data); - extern int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size); - extern void __init efi_memmap_unmap(void); -+#ifdef CONFIG_PTP -+extern void __init efi_memmap_unmap_after_init(void); ++ if (order != 0) ++ panic("PTP: Unsupport vmemmap alloc."); ++ return __va(early_iee_pgtable_alloc(0)); ++} +#endif -+extern int __init efi_memmap_install(struct efi_memory_map_data *data); -+extern int __init efi_memmap_split_count(efi_memory_desc_t *md, -+ struct range *range); -+extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap, -+ void *buf, struct efi_mem_range *mem); -+extern void __init efi_print_memmap(void); - - #ifdef CONFIG_EFI_ESRT - extern void __init efi_esrt_init(void); -diff --git a/include/linux/filter.h b/include/linux/filter.h -index c358bad6cf8f..051838ea4c34 100644 ---- a/include/linux/filter.h -+++ b/include/linux/filter.h -@@ -50,6 +50,9 @@ struct ctl_table_header; - #define BPF_REG_ARG5 BPF_REG_5 - #define BPF_REG_CTX BPF_REG_6 - #define BPF_REG_FP BPF_REG_10 -+#ifdef CONFIG_HIVE -+#define BPF_REG_BASE BPF_REG_11 -+#endif - - /* Additional register mappings for converted user programs. */ - #define BPF_REG_A BPF_REG_0 -diff --git a/include/linux/iee-func.h b/include/linux/iee-func.h +diff --git a/arch/arm64/kernel/haoc/iee/iee-si-gate.S b/arch/arm64/kernel/haoc/iee/iee-si-gate.S new file mode 100644 -index 000000000000..1958c89f5aed +index 000000000000..b92f4b64a3d2 --- /dev/null -+++ b/include/linux/iee-func.h -@@ -0,0 +1,31 @@ -+#ifndef _LINUX_IEE_FUNC_H -+#define _LINUX_IEE_FUNC_H -+#define HUGE_PMD_ORDER 9 -+#define TASK_ORDER 4 ++++ b/arch/arm64/kernel/haoc/iee/iee-si-gate.S +@@ -0,0 +1,81 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#include ++#include ++#include ++#include + -+#ifndef page_to_phys -+#define page_to_phys(page) ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT) -+#endif ++SYM_FUNC_START(iee_rwx_gate) ++ /* save daif, close irq */ ++ mrs x13, daif ++ msr daifset, #0xf ++ isb + -+extern unsigned long init_iee_stack_begin[]; -+extern unsigned long init_iee_stack_end[]; -+extern void *init_token_page_vaddr; ++ stp x29, x30, [sp, #-16]! ++ bl iee_rwx_gate_tramp ++ ldp x29, x30, [sp], #16 + -+extern unsigned long long iee_rw_gate(int flag, ...); -+extern u32 get_cpu_asid_bits(void); -+extern unsigned long arm64_mm_context_get(struct mm_struct *mm); -+extern void set_iee_page(unsigned long addr, int order); -+extern void unset_iee_page(unsigned long addr, int order); -+extern void set_iee_page_valid(unsigned long addr); -+extern void iee_set_logical_mem_ro(unsigned long addr); -+extern void __init iee_mark_all_lm_pgtable_ro(void); -+extern unsigned long iee_read_token_stack(struct task_struct *tsk); -+extern void iee_set_token_page_valid(void *token, void *new, unsigned int order); -+extern void iee_set_token_page_invalid(void *token_addr, void *token_page, unsigned long order); -+extern void unset_iee_stack_page(unsigned long addr, int order); -+extern void set_iee_stack_page(unsigned long addr, int order); -+extern void *iee_read_tmp_page(struct task_struct *tsk); -+extern void *iee_read_freeptr(unsigned long ptr); -+extern void iee_set_stack_freeptr(unsigned long addr, void *free_ptr); -+extern void iee_rest_init(void); -+#endif -diff --git a/include/linux/key.h b/include/linux/key.h -index 938d7ecfb495..057252aacc13 100644 ---- a/include/linux/key.h -+++ b/include/linux/key.h -@@ -280,6 +280,38 @@ struct key { - struct key_restriction *restrict_link; - }; - -+#ifdef CONFIG_KEYP -+struct key_union { -+ union { -+ struct list_head graveyard_link; -+ struct rb_node serial_node; -+ }; -+ struct rw_semaphore sem; -+ struct key *key; -+}; ++ /* restore irq */ ++ msr daif, x13 ++ ret ++SYM_FUNC_END(iee_rwx_gate) + -+struct key_struct { -+ struct { -+ /* Keyring bits */ -+ struct list_head name_link; -+ struct assoc_array keys; -+ }; -+ struct key *key; -+}; ++/* We can confim the following code are uninterruptible by ELR_EL1 checking ++ * on kernel entry. ++ */ ++ .pushsection ".iee.text", "ax" ++ ++SYM_FUNC_START(iee_rwx_gate_tramp) ++ /* entry gate */ ++ mrs x12, tcr_el1 ++ orr x12, x12, #TCR_HPD1 ++ orr x12, x12, #TCR_A1 ++ msr tcr_el1, x12 ++ isb ++ /* Check TCR */ ++ ldr x9, iee_tcr ++ cbz x9, 1f /* Skip tcr check and stack switch on early init */ ++ ldr x10, iee_tcr_mask ++ and x12, x12, x10 ++ cmp x9, x12 ++ b.ne 3f ++ ++ /* Switch to iee stack by per cpu ptr. */ ++ ldr_this_cpu x9, iee_cpu_stack_ptr, x10 ++ mov x14, sp ++ mov sp, x9 ++1: ++ stp x13, x14, [sp, #-32]! ++ stp x29, x30, [sp, #16] ++ /* call iee func */ ++ bl iee_si_handler ++ ldp x29, x30, [sp, #16] ++ ldp x13, x14, [sp], #32 ++ ++ /* exit gate */ ++ mrs x12, tcr_el1 ++ bic x12, x12, #TCR_HPD1 ++ bic x12, x12, #TCR_A1 ++ msr tcr_el1, x12 ++ isb ++ /* Check TCR */ ++ ldr x9, kernel_tcr ++ cbz x9, 2f ++ ldr x10, iee_tcr_mask ++ and x12, x12, x10 ++ cmp x9, x12 ++ b.ne 3f ++ ++ /* Switch to kernel stack */ ++ mov sp, x14 ++2: ++ ret ++3: ++ mov x0, sp ++ mov x1, #BAD_TCR_EL1 ++ mrs x2, esr_el1 ++ mrs x3, tcr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++SYM_FUNC_END(iee_rwx_gate_tramp) ++ ++ .popsection +diff --git a/arch/arm64/kernel/haoc/iee/iee-si.c b/arch/arm64/kernel/haoc/iee/iee-si.c +new file mode 100644 +index 000000000000..5ff1ce2d0d31 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee-si.c +@@ -0,0 +1,105 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_IEE_PTRP ++#include +#endif + -+#ifdef CONFIG_KEYP -+#define KEY_SEM(KEY) (((struct key_union *)(KEY->graveyard_link.next))->sem) -+#include -+extern unsigned long long iee_rw_gate(int flag, ...); -+static bool iee_set_key_usage(struct key *key, int n, int flag) ++u64 __ro_after_init iee_si_reserved_pg_dir; ++ ++static inline unsigned long iee_si_mask(unsigned long mask, ++ unsigned long new_val, unsigned long old_val) +{ -+ bool ret; -+ ret = iee_rw_gate(IEE_OP_SET_KEY_USAGE, key, n, flag); -+ return ret; ++ return (new_val & mask) | (old_val & ~mask); +} -+#endif + - extern struct key *key_alloc(struct key_type *type, - const char *desc, - kuid_t uid, kgid_t gid, -@@ -305,7 +337,11 @@ extern void key_remove_domain(struct key_tag *domain_tag); - - static inline struct key *__key_get(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ iee_set_key_usage(key, 0, REFCOUNT_INC); -+ #else - refcount_inc(&key->usage); -+ #endif - return key; - } - -@@ -478,17 +514,67 @@ static inline bool key_is_negative(const struct key *key) - return key_read_state(key) < 0; - } - -+#ifdef CONFIG_KEYP -+static inline void iee_write_key_payload_rcu_data0(struct key *key, void *rcu_data0) ++unsigned long __iee_si_code iee_si_handler(int flag, ...) +{ -+ union key_payload *key_payload = (union key_payload *)(key->name_link.next); -+ WRITE_ONCE(key_payload->rcu_data0, rcu_data0); ++ va_list pArgs; ++ unsigned long old_val, new_val; ++ ++ va_start(pArgs, flag); ++ switch (flag) { ++ case IEE_SI_TEST: ++ break; ++ case IEE_SI_SET_SCTLR_EL1: ++ old_val = read_sysreg(sctlr_el1); ++ new_val = va_arg(pArgs, u64); ++ new_val = iee_si_mask(IEE_SI_SCTLR_MASK, new_val, old_val); ++ write_sysreg(new_val, sctlr_el1); ++ break; ++ case IEE_SI_SET_TTBR0: ++ case IEE_SI_CONTEXT_SWITCH: ++ old_val = read_sysreg(ttbr0_el1); ++ new_val = va_arg(pArgs, u64); ++ /* Skip checking before init IEE. */ ++ if (iee_tcr != 0) { ++ u64 new_asid; ++ /* ASID shall not be the one reserved for IEE. */ ++ new_asid = new_val >> 48; ++ if (new_asid == IEE_ASID) ++ panic("IEE SI: Using reserved IEE ASID in TTRB0."); ++ #ifdef CONFIG_IEE_PTRP ++ u64 old_phys; ++ struct task_token *token = (struct task_token *)__addr_to_iee(current); ++ /* Phys in TTBR0 shall be the same with current->mm->pgd. */ ++ old_phys = (old_val & PAGE_MASK) & ~TTBR_ASID_MASK; ++ if (!(current == &init_task) && token->pgd ++ && old_phys != iee_si_reserved_pg_dir) { ++ u64 token_phys = __pa(token->pgd); ++ ++ if (old_phys != token_phys) ++ panic("IEE: Pgd set error. old ttbr0:%llx, token ttbr0:%llx", ++ old_phys, token_phys); ++ } ++ #endif ++ } ++ write_sysreg(new_val, ttbr0_el1); ++ break; ++ case IEE_SI_SET_VBAR: ++ new_val = va_arg(pArgs, u64); ++ write_sysreg(new_val, vbar_el1); ++ break; ++ case IEE_SI_SET_TCR_EL1: ++ new_val = va_arg(pArgs, u64); ++ old_val = read_sysreg(tcr_el1); ++ new_val = iee_si_mask(IEE_SI_TCR_MASK, new_val, old_val); ++ write_sysreg(new_val, tcr_el1); ++ break; ++ default: ++ break; ++ } ++ va_end(pArgs); ++ return 0; +} + -+#define dereference_key_rcu(KEY) \ -+ (rcu_dereference(((union key_payload *)(KEY->name_link.next))->rcu_data0)) ++/* Map iee si code PXNTable=1 on pmd page tables. */ ++static int __init iee_si_init_code(void) ++{ ++ u64 addr = (u64)__iee_si_text_start; ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ pmd_t pmd = READ_ONCE(*pmdp); + -+#ifdef CONFIG_KEYP -+#define dereference_key_locked(KEY) \ -+ (rcu_dereference_protected(((union key_payload *)(KEY->name_link.next))->rcu_data0, \ -+ rwsem_is_locked(&KEY_SEM(((struct key *)(KEY)))))) -+#else -+#define dereference_key_locked(KEY) \ -+ (rcu_dereference_protected(((union key_payload *)(KEY->name_link.next))->rcu_data0, \ -+ rwsem_is_locked(&((struct key *)(KEY))->sem))) -+#endif ++ if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) { ++ pr_err("IEE SI: addr 0x%llx is pmd block. SI init failed.", addr); ++ return 0; ++ } + -+#define iee_smp_store_release(p, v, KEY) \ -+do { \ -+ compiletime_assert_atomic_type(*p); \ -+ barrier(); \ -+ iee_write_key_payload_rcu_data0(KEY, v); \ -+} while (0) ++ pmd = __pmd(pmd_val(pmd) | PGD_PXNTABLE); ++ set_pmd(pmdp, pmd); ++ flush_tlb_all(); ++ return 1; ++} + -+#define iee_rcu_assign_pointer(p, v, KEY) \ -+do { \ -+ uintptr_t _r_a_p__v = (uintptr_t)(v); \ -+ rcu_check_sparse(p, __rcu); \ -+ \ -+ if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \ -+ iee_write_key_payload_rcu_data0(KEY, (typeof(p))(_r_a_p__v)); \ -+ else \ -+ iee_smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v), KEY); \ -+} while (0) ++void __init iee_si_init(void) ++{ ++ if (iee_si_init_code()) ++ pr_info("IEE: Sensitive instruction protection is ready."); ++} +diff --git a/arch/arm64/kernel/haoc/iee/iee-token.c b/arch/arm64/kernel/haoc/iee/iee-token.c +new file mode 100644 +index 000000000000..3b9a432b3a19 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee-token.c +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include "slab.h" + -+#define rcu_assign_keypointer(KEY, PAYLOAD) \ -+do { \ -+ iee_rcu_assign_pointer(((union key_payload *)(KEY->name_link.next))->rcu_data0, (PAYLOAD), KEY); \ -+} while (0) -+#else - #define dereference_key_rcu(KEY) \ - (rcu_dereference((KEY)->payload.rcu_data0)) - -+#ifdef CONFIG_KEYP -+#define dereference_key_locked(KEY) \ -+ (rcu_dereference_protected((KEY)->payload.rcu_data0, \ -+ rwsem_is_locked(&KEY_SEM(((struct key *)(KEY)))))) -+#else - #define dereference_key_locked(KEY) \ - (rcu_dereference_protected((KEY)->payload.rcu_data0, \ - rwsem_is_locked(&((struct key *)(KEY))->sem))) -+#endif - - #define rcu_assign_keypointer(KEY, PAYLOAD) \ - do { \ - rcu_assign_pointer((KEY)->payload.rcu_data0, (PAYLOAD)); \ - } while (0) -+#endif - - /* - * the userspace interface -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 2e6ef9532fc3..3ecd0d77a1dc 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -32,6 +32,11 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#include -+#endif ++void __init iee_prepare_init_task_token(void) ++{ ++ struct task_token *init_token = (struct task_token *)__kimg_to_iee(&init_task); ++ u64 init_token_addr = (u64)init_token; ++ phys_addr_t init_token_page; ++ int order = 0; + - struct mempolicy; - struct anon_vma; - struct anon_vma_chain; -@@ -2944,7 +2949,19 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) - */ - static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) - { -+ #ifdef CONFIG_PTP -+ #ifdef CONFIG_X86_64 -+ struct page *page; -+ if (iee_pgt_jar_init) -+ page = virt_to_page(get_iee_pgtable_page(gfp | __GFP_COMP)); -+ else -+ page = alloc_pages(gfp | __GFP_COMP, order); -+ #else -+ struct page *page = virt_to_page(get_iee_pgtable_page(gfp | __GFP_COMP)); -+ #endif -+ #else - struct page *page = alloc_pages(gfp | __GFP_COMP, order); -+ #endif - - return page_ptdesc(page); - } -@@ -2960,7 +2977,11 @@ static inline void pagetable_free(struct ptdesc *pt) - { - struct page *page = ptdesc_page(pt); - -+ #ifdef CONFIG_PTP -+ free_iee_pgtable_page((void *)page_address(page)); -+ #else - __free_pages(page, compound_order(page)); -+ #endif - } - - #if USE_SPLIT_PTE_PTLOCKS -@@ -2971,7 +2992,18 @@ void ptlock_free(struct ptdesc *ptdesc); - - static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) - { -+ #ifdef CONFIG_PTP -+ #ifdef CONFIG_X86_64 -+ if (iee_pgt_jar_init) -+ return (iee_ptdesc(ptdesc))->ptl; -+ else -+ return ptdesc->ptl; -+ #else -+ return (iee_ptdesc(ptdesc))->ptl; -+ #endif -+ #else - return ptdesc->ptl; -+ #endif - } - #else /* ALLOC_SPLIT_PTLOCKS */ - static inline void ptlock_cache_init(void) -@@ -2989,7 +3021,18 @@ static inline void ptlock_free(struct ptdesc *ptdesc) - - static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) - { -+ #ifdef CONFIG_PTP -+ #ifdef CONFIG_X86_64 -+ if (iee_pgt_jar_init) -+ return &(iee_ptdesc(ptdesc))->ptl; -+ else -+ return &ptdesc->ptl; -+ #else -+ return &(iee_ptdesc(ptdesc))->ptl; -+ #endif -+ #else - return &ptdesc->ptl; -+ #endif - } - #endif /* ALLOC_SPLIT_PTLOCKS */ - -@@ -3007,7 +3050,11 @@ static inline bool ptlock_init(struct ptdesc *ptdesc) - * It can happen if arch try to use slab for page table allocation: - * slab code uses page->slab_cache, which share storage with page->ptl. - */ -+ #ifdef CONFIG_PTP -+ VM_BUG_ON_PAGE(*(unsigned long *)&(iee_ptdesc(ptdesc)->ptl), ptdesc_page(ptdesc)); -+ #else - VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); -+ #endif - if (!ptlock_alloc(ptdesc)) - return false; - spin_lock_init(ptlock_ptr(ptdesc)); -@@ -3106,7 +3153,11 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) - static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) - { - #ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ #ifdef CONFIG_PTP -+ ((iee_ptdesc(ptdesc))->pmd_huge_pte) = NULL; -+ #else - ptdesc->pmd_huge_pte = NULL; -+ #endif - #endif - return ptlock_init(ptdesc); - } -@@ -3114,12 +3165,20 @@ static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) - static inline void pmd_ptlock_free(struct ptdesc *ptdesc) - { - #ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ #ifdef CONFIG_PTP -+ VM_BUG_ON_PAGE(((iee_ptdesc(ptdesc))->pmd_huge_pte), ptdesc_page(ptdesc)); -+ #else - VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc)); -+ #endif - #endif - ptlock_free(ptdesc); - } - -+#ifdef CONFIG_PTP -+#define pmd_huge_pte(mm, pmd) ((iee_ptdesc(pmd_ptdesc(pmd)))->pmd_huge_pte) -+#else - #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) -+#endif - - #else - -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 4b9a8723d3eb..07da48fd7d88 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -483,6 +483,42 @@ struct ptdesc { - #endif - }; - -+#ifdef CONFIG_PTP -+struct ptdesc_t { -+ unsigned long __page_flags; ++ /* ++ * if the init token covers the page boundary, the token page shall ++ * be allocated more. ++ */ ++ if (ALIGN((init_token_addr + sizeof(struct task_token)), PAGE_SIZE) ++ != ALIGN(init_token_addr + 1, PAGE_SIZE)) ++ order = 1; + -+ union { -+ struct rcu_head pt_rcu_head; -+ struct list_head pt_list; -+ struct { -+ unsigned long _pt_pad_1; -+ pgtable_t pmd_huge_pte; -+ }; -+ }; -+ unsigned long __page_mapping; ++ /* Allocate from IEE data pool. */ ++ init_token_page = early_iee_data_alloc(order); + -+ union { -+ struct mm_struct *pt_mm; -+ atomic_t pt_frag_refcount; -+ }; ++ /* Map the IEE token address of init_task. */ ++ for (int i = 0; i < 1UL << order; i++) { ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, init_token_addr); ++ p4d_t *p4dp = p4d_offset(pgdp, init_token_addr); ++ pud_t *pudp = pud_offset(p4dp, init_token_addr); ++ pmd_t *pmdp = pmd_offset(pudp, init_token_addr); ++ pte_t *ptep = pte_offset_kernel(pmdp, init_token_addr); ++ pte_t pte = READ_ONCE(*ptep); + -+ union { -+ unsigned long _pt_pad_2; -+#if ALLOC_SPLIT_PTLOCKS -+ spinlock_t *ptl; -+#else -+ spinlock_t ptl; -+#endif -+ }; -+ unsigned int __page_type; -+ atomic_t _refcount; -+#ifdef CONFIG_MEMCG -+ unsigned long pt_memcg_data; -+#endif -+ struct page *page; -+}; -+#endif ++ pte = __pte(((pte_val(pte) | PTE_VALID) & ~PTE_ADDR_MASK) ++ | __phys_to_pte_val(init_token_page)); ++ /* Manaully go through IEE gates to bypass PTP checks. */ ++ #ifdef CONFIG_PTP ++ write_sysreg(read_sysreg(TCR_EL1) | TCR_HPD1 | TCR_A1, tcr_el1); ++ isb(); ++ WRITE_ONCE(*__ptr_to_iee(ptep), pte); ++ write_sysreg(read_sysreg(TCR_EL1) & ~(TCR_HPD1 | TCR_A1), tcr_el1); ++ isb(); ++ #else ++ set_pte(ptep, pte); ++ #endif + - #define TABLE_MATCH(pg, pt) \ - static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) - TABLE_MATCH(flags, __page_flags); -@@ -1181,6 +1217,9 @@ struct mmu_gather; - extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); - extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); - extern void tlb_finish_mmu(struct mmu_gather *tlb); -+#ifdef CONFIG_PTP -+extern void iee_tlb_finish_mmu(struct mmu_gather *tlb); -+#endif - - struct vm_fault; - -diff --git a/include/linux/module.h b/include/linux/module.h -index 990f9d66d2f1..169021fc8501 100644 ---- a/include/linux/module.h -+++ b/include/linux/module.h -@@ -609,6 +609,7 @@ struct module { - KABI_RESERVE(2) - KABI_RESERVE(3) - KABI_RESERVE(4) -+ - } ____cacheline_aligned __randomize_layout; - #ifndef MODULE_ARCH_INIT - #define MODULE_ARCH_INIT {} -diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h -index 98db43432ee9..dca565ffb64d 100644 ---- a/include/linux/pgtable.h -+++ b/include/linux/pgtable.h -@@ -1095,6 +1095,38 @@ static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) - set_pgd(pgdp, pgd); \ - }) - -+#ifdef CONFIG_PTP -+#define iee_set_pte_safe_pre_init(ptep, pte) \ -+({ \ -+ WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ -+ iee_set_pte_pre_init(ptep, pte); \ -+}) ++ init_token_addr += PAGE_SIZE; ++ } ++ ++ /* Operate on the init token since it's already valid. */ ++ iee_validate_token(&init_task); ++ pr_info("IEE: CONFIG_PTRP enabled."); ++} ++ ++/* To map or unmap token pages when allocate or free task_struct. */ ++static inline void iee_set_token(unsigned long token_addr, unsigned long token_pages, ++ unsigned int order, bool prot) ++{ ++ phys_addr_t token_phys = __pa(token_pages); ++ unsigned long end_addr = token_addr + (PAGE_SIZE << order); ++ u64 curr_addr = token_addr; ++ /* ++ * IEE mappings must be 4-level so we just need to find out the pte of start ++ * address. ++ */ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, token_addr); ++ p4d_t *p4dp = p4d_offset(pgdp, token_addr); ++ pud_t *pudp = pud_offset(p4dp, token_addr); ++ pmd_t *pmdp = pmd_offset(pudp, token_addr); ++ pte_t *ptep = pte_offset_kernel(pmdp, token_addr); + -+#define iee_set_pmd_safe_pre_init(pmdp, pmd) \ -+({ \ -+ WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ -+ iee_set_pmd_pre_init(pmdp, pmd); \ -+}) ++ /* ++ * We assume that input address range would never cross the boundary ++ * of a pmd block, so this function could simply use ptep++ to iterate ++ * inside a pte table. ++ */ ++ if (token_addr != ALIGN(token_addr, PAGE_SIZE) ++ || end_addr > ALIGN(token_addr + 1, PMD_SIZE)) ++ panic("%s: invalid input address range 0x%lx-0x%lx.", __func__, ++ token_addr, end_addr); + -+#define iee_set_pud_safe_pre_init(pudp, pud) \ -+({ \ -+ WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ -+ iee_set_pud_pre_init(pudp, pud); \ -+}) ++ /* map new pages to IEE addresses one by one or clear them. */ ++ for (int i = 0; i < (1UL << order); i++) { ++ pte_t pte = READ_ONCE(*ptep); + -+#define iee_set_p4d_safe_pre_init(p4dp, p4d) \ -+({ \ -+ WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ -+ iee_set_p4d_pre_init(p4dp, p4d); \ -+}) ++ if (prot) { ++ /* rewrite physical address on pte. */ ++ pte = __pte(((pte_val(pte) | PTE_VALID) & ~PTE_ADDR_MASK) ++ | __phys_to_pte_val(token_phys)); ++ token_phys += PAGE_SIZE; ++ } else { ++ /* Restore the 1:1 mapping with physical addresses. */ ++ pte = __pte(((pte_val(pte) & ~PTE_VALID) & ~PTE_ADDR_MASK) ++ | __phys_to_pte_val(__iee_to_phys(curr_addr))); ++ curr_addr += PAGE_SIZE; ++ } + -+#define iee_set_pgd_safe_pre_init(pgdp, pgd) \ -+({ \ -+ WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ -+ iee_set_pgd_pre_init(pgdp, pgd); \ -+}) -+#endif ++ set_pte(ptep, pte); ++ ptep++; ++ } + - #ifndef __HAVE_ARCH_DO_SWAP_PAGE - static inline void arch_do_swap_page_nr(struct mm_struct *mm, - struct vm_area_struct *vma, -diff --git a/include/linux/sched.h b/include/linux/sched.h -index a694cc11dea5..5b89c9c5485d 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -773,6 +773,24 @@ struct task_struct_resvd { - struct task_struct *task; - }; - -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+struct task_token { -+#ifdef CONFIG_IEE -+ pgd_t *pgd; /* Logical VA */ -+ void *iee_stack; /* VA */ -+ void *tmp_page; -+ bool valid; -+ void *kernel_stack; /* VA */ -+#endif -+#ifdef CONFIG_KOI -+ void *koi_kernel_stack; /* VA */ -+ void *koi_stack; /* VA */ -+ void *koi_stack_base; /* VA */ -+ unsigned long current_ttbr1; -+#endif -+}; -+#endif ++ /* Apply or remove RO protection on linear mappings. */ ++ iee_set_logical_mem(token_pages, order, prot); + - struct task_struct { - #ifdef CONFIG_THREAD_INFO_IN_TASK - /* -@@ -795,6 +813,7 @@ struct task_struct { - randomized_struct_fields_start - - void *stack; ++ flush_tlb_kernel_range(token_addr, end_addr); ++ flush_tlb_kernel_range(token_pages, (token_pages + (PAGE_SIZE << order))); ++} + - refcount_t usage; - /* Per task flags (PF_*), defined further below: */ - unsigned int flags; -diff --git a/include/linux/sfi_bpf.h b/include/linux/sfi_bpf.h -new file mode 100644 -index 000000000000..23c5d9409c6e ---- /dev/null -+++ b/include/linux/sfi_bpf.h -@@ -0,0 +1,86 @@ -+#ifndef LINUX_SFI_BPF_H -+#define LINUX_SFI_BPF_H ++/* ++ * iee_set_token_page_valid() - After allocated task token pages, map them to the ++ * corresponding IEE addresses, and enforce RO protection on their linear mappings. ++ * ++ * @token_addr: the start IEE address of task tokens. ++ * @token_pages: virtual address of allocated token pages. ++ * @order: size for address range. ++ */ ++void iee_set_token_page_valid(unsigned long token_addr, unsigned long token_pages, ++ unsigned int order) ++{ ++ #ifdef DEBUG ++ pr_info("IEE: Set token addr 0x%lx order %d for page 0x%lx", token_addr, ++ order, token_pages); ++ #endif ++ iee_set_token(token_addr, token_pages, order, true); ++} + -+#include -+#include ++void iee_set_token_page_invalid(unsigned long token_addr, unsigned long token_pages, ++ unsigned int order) ++{ ++ #ifdef DEBUG ++ pr_info("IEE: Unset token addr 0x%lx order %d for page 0x%lx", token_addr, ++ order, token_pages); ++ #endif ++ iee_set_token(token_addr, token_pages, order, false); ++} + -+/* fbpf log */ -+#define FBPF_LOG_ENABLE -+#define fbpf_log(fmt, ...) do_fbpf_log(__FUNCTION__, fmt, ##__VA_ARGS__) -+void do_fbpf_log(const char *func_name, const char *fmt, ...); ++struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, ++ struct slab *slab, unsigned int order) ++{ ++ if (!slab || s != task_struct_cachep) ++ return slab; + -+/* fbpf used for output */ -+extern const char *bpf_map_type_strings[]; ++ struct folio *folio = slab_folio(slab); ++ unsigned long token_addr = __slab_to_iee(slab); ++ unsigned long alloc_token = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + -+/* fbpf map vaddr to struct page* */ -+struct page *kv_virt_to_page(const void *vaddr); ++ /* Allocation of task_struct and token pages must be done at the same time. */ ++ if (!alloc_token) { ++ /* Failed on allocation of token page. Free the allocated ones, ++ * return and try smaller order. ++ */ ++ __slab_clear_pfmemalloc(slab); ++ folio->mapping = NULL; ++ /* Make the mapping reset visible before clearing the flag */ ++ smp_wmb(); ++ __folio_clear_slab(folio); ++ __free_pages((struct page *)folio, order); ++ return NULL; ++ } + -+/* fbpf aggregate */ -+int bpf_sfi_aggregate_memory(struct bpf_verifier_env *env); ++ /* Map allocated token pages to token addresses. */ ++ iee_set_token_page_valid(token_addr, alloc_token, order); ++ return slab; ++} + -+/* fbpf record percpu map used pages */ -+struct bpf_used_page { -+ struct page *physic_page; -+ struct list_head list_head; -+ u64 shadow_page; -+}; ++void __iee_code _iee_init_token(unsigned long __unused, struct task_struct *tsk) ++{ ++ /* Do nothing for now. Wait for later update. */ ++} + -+/* fbpf: relevant api for bpf_used_pages */ -+int bpf_record_used_pages(u64 start_addr, u64 end_addr, -+ struct mutex *write_mutex, struct list_head *head); ++void __iee_code _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, ++ pgd_t *pgd) ++{ ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + -+/* fbpf: htab's value addr is built-in element, use api to substitue */ -+void substitute_hash_value_ptr(struct bpf_map *map); ++ token->pgd = pgd; ++} + -+inline void *alloc_aligned_memory(u64 size); ++void __iee_code _iee_validate_token(unsigned long __unused, struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + -+inline int map_physic_pages(struct page **pages, void *virt_addr, int page_cnt); ++ if (token->valid) ++ pr_err("IEE: validate token for multiple times."); ++ token->valid = true; ++ #ifdef CONFIG_CREDP ++ token->curr_cred = tsk->cred; ++ #endif ++} + -+struct bpf_sfi_check_unit *bpf_sfi_get_check_list( -+ struct bpf_verifier_env *env, struct bpf_insn *insn, -+ struct bpf_reg_state *regs); ++void __iee_code _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + -+/* fbpf: check type for a reg in insn */ -+enum bpf_sfi_check_type { -+ BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR, -+ BPF_SFI_CHECK_LDST_AS_SAFE_PTR, -+ BPF_SFI_CHECK_LDST_AS_OFFSET, -+ BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR, -+ BPF_SFI_CHECK_HELPER_AS_SAFE_PTR, -+ BPF_SFI_CHECK_HELPER_AS_OFFSET, -+}; ++ token->pgd = NULL; ++ token->valid = false; ++ #ifdef CONFIG_CREDP ++ token->curr_cred = NULL; ++ token->new_cred = NULL; ++ #endif ++} +diff --git a/arch/arm64/kernel/haoc/iee/iee.c b/arch/arm64/kernel/haoc/iee/iee.c +new file mode 100644 +index 000000000000..6257434e3fbd +--- /dev/null ++++ b/arch/arm64/kernel/haoc/iee/iee.c +@@ -0,0 +1,37 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include + -+/* fbpf: check state for a reg */ -+enum bpf_sfi_check_state { -+ BPF_SFI_UNSAFE_PTR, -+ BPF_SFI_SAFE_PTR, -+ BPF_SFI_OFFSET, -+}; ++#pragma GCC push_options ++#pragma GCC optimize("O0") ++void __iee_code _iee_memset(unsigned long __unused, void *ptr, int data, size_t n) ++{ ++ char *_ptr; + -+// TODO: allocate and free this link list for each insn_aux_data -+/* fbpf: check unit (link list) for a insn */ -+struct bpf_sfi_check_unit { -+ struct list_head list_head; -+ int reg_no; -+ enum bpf_sfi_check_type check_type; -+ bool reg_as_offset; -+ enum bpf_sfi_check_state proposed_state; -+}; ++ iee_verify_not_normal(ptr, ptr + n); ++ _ptr = __ptr_to_iee((char *)ptr); ++ // _ptr = (char *)ptr; + -+/* fbpf dump insns of a BPF Program */ -+void dump_insns(struct bpf_prog *prog, int start, int len, const char *prompt); ++ while (n--) ++ *_ptr++ = data; ++} + -+int bpf_sfi_tmp_check(struct bpf_verifier_env *env); ++void __iee_code _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) ++{ ++ char *_dst, *_src; + -+/* fbpf patch to check ld/st insns */ -+int bpf_sfi_check_ldst(struct bpf_verifier_env *env); ++ iee_verify_not_normal(dst, dst + n); ++ _dst = __ptr_to_iee(dst); ++ _src = (char *)src; + -+/* fbpf patch to check helper's args */ -+int bpf_sfi_check_helper_args(struct bpf_verifier_env *env); ++ while (n--) ++ *_dst++ = *_src++; ++} ++#pragma GCC pop_options + -+struct bpf_prog *_bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, -+ const struct bpf_insn *patch, u32 len); ++void __iee_code _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr) ++{ ++ iee_verify_not_normal(pptr, pptr + sizeof(void *)); ++ pptr = __ptr_to_iee(pptr); ++ *pptr = ptr; ++} +diff --git a/arch/arm64/kernel/haoc/keyp/Makefile b/arch/arm64/kernel/haoc/keyp/Makefile +new file mode 100644 +index 000000000000..307cebd57ff7 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/keyp/Makefile +@@ -0,0 +1 @@ ++obj-y += keyp.o +\ No newline at end of file +diff --git a/arch/arm64/kernel/haoc/keyp/keyp.c b/arch/arm64/kernel/haoc/keyp/keyp.c +new file mode 100644 +index 000000000000..167837098b28 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/keyp/keyp.c +@@ -0,0 +1,264 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include + -+bool bpf_sfi_can_instrument(struct bpf_verifier_env *env); ++struct watch_list; + -+#endif -\ No newline at end of file -diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h -index 1597a5f9b5b8..828660c74337 100644 ---- a/include/linux/skbuff.h -+++ b/include/linux/skbuff.h -@@ -1065,6 +1065,9 @@ struct sk_buff { - sk_buff_data_t end; - unsigned char *head, - *data; -+#ifdef CONFIG_HIVE -+ unsigned char *sfi_data, *sfi_data_meta, *sfi_data_end; -+#endif - unsigned int truesize; - refcount_t users; - -diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h -index 482647774bf5..4eb077e21915 100644 ---- a/include/uapi/linux/bpf.h -+++ b/include/uapi/linux/bpf.h -@@ -63,6 +63,7 @@ enum { - BPF_REG_8, - BPF_REG_9, - BPF_REG_10, -+ BPF_REG_11, - __MAX_BPF_REG, - }; - -diff --git a/include/uapi/linux/bpf_common.h b/include/uapi/linux/bpf_common.h -index ee97668bdadb..104caf07a85f 100644 ---- a/include/uapi/linux/bpf_common.h -+++ b/include/uapi/linux/bpf_common.h -@@ -26,6 +26,7 @@ - #define BPF_MEM 0x60 - #define BPF_LEN 0x80 - #define BPF_MSH 0xa0 -+#define BPF_REG 0xe0 - - /* alu/jmp fields */ - #define BPF_OP(code) ((code) & 0xf0) -diff --git a/init/main.c b/init/main.c -index 8fdfa69dba0f..feca51bc2ef9 100644 ---- a/init/main.c -+++ b/init/main.c -@@ -102,6 +102,16 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#include -+#include -+#include -+#endif -+#ifdef CONFIG_PTP -+#include -+#endif ++static inline void iee_verify_key_type(const struct key *key) ++{ ++ iee_verify_type(key, IEE_KEY, "key"); ++} + - #include - #include - #include -@@ -933,6 +943,12 @@ void start_kernel(void) - sort_main_extable(); - trap_init(); - mm_core_init(); -+ #ifdef CONFIG_IEE -+ iee_stack_init(); -+ #endif -+ #ifdef CONFIG_PTP -+ iee_pgtable_init(); -+ #endif - poking_init(); - ftrace_init(); - -@@ -1073,6 +1089,20 @@ void start_kernel(void) - arch_post_acpi_subsys_init(); - kcsan_init(); - -+ // Later IEE settings. -+ #ifdef CONFIG_IEE -+ iee_rest_init(); -+ set_iee_stack_page((unsigned long)__va(__pa_symbol(init_iee_stack_begin)), 2); -+ for (int i = 0; i < 4; i++) { -+ iee_set_logical_mem_ro((unsigned long)init_iee_stack_begin + PAGE_SIZE * i); ++unsigned long __iee_code _iee_set_key_flag_bit(unsigned long __unused, struct key *key, ++ long nr, int flag) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ switch (flag) { ++ case SET_BIT_OP: { ++ set_bit(nr, &key->flags); ++ break; + } -+ set_iee_page((unsigned long)init_token_page_vaddr, 0); -+ // Set the logical va of existing pgtable readonly. -+ #ifdef CONFIG_PTP -+ iee_mark_all_lm_pgtable_ro(); -+ #endif -+ #endif ++ case TEST_AND_CLEAR_BIT: { ++ return test_and_clear_bit(nr, &key->flags); ++ } ++ case TEST_AND_SET_BIT: { ++ return test_and_set_bit(nr, &key->flags); ++ } ++ } ++ return 0; ++} + - /* Do the rest non-__init'ed, we're now alive */ - arch_call_rest_init(); - -@@ -1460,7 +1490,7 @@ static int __ref kernel_init(void *unused) - * to finalize PTI. - */ - pti_finalize(); -- -+ - system_state = SYSTEM_RUNNING; - numa_default_policy(); - -diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig -index 19f6ab882ab1..8d0dea7f0f51 100644 ---- a/kernel/bpf/Kconfig -+++ b/kernel/bpf/Kconfig -@@ -113,4 +113,15 @@ config BPF_SCHED - - If you are unsure how to answer this question, answer N. - -+config HIVE -+ bool "Reserved for eBPF security enhancement" -+ default n -+ depends on BPF_SYSCALL -+ depends on DEBUG_INFO_DWARF4 -+ depends on DEBUG_INFO_BTF -+ select NET_ACT_BPF -+ select BPFILTER -+ select NET_CLS_BPF -+ select NET_SCH_INGRESS -+ - endmenu # "BPF subsystem" -diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile -index f526b7573e97..316c693b8d35 100644 ---- a/kernel/bpf/Makefile -+++ b/kernel/bpf/Makefile -@@ -46,3 +46,5 @@ obj-$(CONFIG_BPF_PRELOAD) += preload/ - obj-$(CONFIG_BPF_SYSCALL) += relo_core.o - $(obj)/relo_core.o: $(srctree)/tools/lib/bpf/relo_core.c FORCE - $(call if_changed_rule,cc_o_c) -+ -+obj-$(CONFIG_HIVE) += sfi_bpf.o -diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c -index 1811efcfbd6e..2a59acc27e6e 100644 ---- a/kernel/bpf/arraymap.c -+++ b/kernel/bpf/arraymap.c -@@ -12,6 +12,9 @@ - #include - #include - #include -+#ifdef CONFIG_HIVE -+#include -+#endif - - #include "map_in_map.h" - -@@ -21,12 +24,38 @@ - - static void bpf_array_free_percpu(struct bpf_array *array) - { -+#ifdef CONFIG_HIVE -+ if (unlikely(!array->map.is_aggregated)) { -+ int i; -+ for (i = 0; i < array->map.max_entries; i++) { -+ free_percpu(array->pptrs[i]); -+ cond_resched(); -+ } -+ } else { -+ struct bpf_used_page *used_page_entry, *used_page_next; -+ struct list_head *used_pages_list_head = &array->map.used_pages->list_head; ++void __iee_code _iee_set_key_restrict_link(unsigned long __unused, ++ struct key *key, ++ struct key_restriction *restrict_link) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->restrict_link = restrict_link; ++} + -+ free_percpu(array->pptrs[0]); -+ cond_resched(); ++void __iee_code _iee_set_key_magic(unsigned long __unused, struct key *key, ++ unsigned int magic) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++#ifdef KEY_DEBUGGING ++ key->magic = magic; ++#endif ++} + -+ if (used_pages_list_head && !list_empty(used_pages_list_head)) { -+ list_for_each_entry_safe(used_page_entry, used_page_next, -+ used_pages_list_head, list_head){ -+ list_del(&used_page_entry->list_head); -+ vfree(used_page_entry); -+ } -+ } ++void __iee_code _iee_set_key_flags(unsigned long __unused, struct key *key, ++ unsigned long flags) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->flags = flags; ++} + -+ vfree(array->map.used_pages); -+ } -+#else - int i; - - for (i = 0; i < array->map.max_entries; i++) { - free_percpu(array->pptrs[i]); - cond_resched(); - } -+#endif - } - - static int bpf_array_alloc_percpu(struct bpf_array *array) -@@ -34,6 +63,55 @@ static int bpf_array_alloc_percpu(struct bpf_array *array) - void __percpu *ptr; - int i; - -+#ifdef CONFIG_HIVE -+ int page_cnt; -+ struct bpf_used_page *used_pages; -+ u64 start_address, end_address; -+ -+ if (unlikely(!array->map.is_aggregated)) { -+ for (i = 0; i < array->map.max_entries; i++) { -+ ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, -+ GFP_USER | __GFP_NOWARN); -+ if (!ptr) { -+ bpf_array_free_percpu(array); -+ return -ENOMEM; -+ } -+ array->pptrs[i] = ptr; -+ cond_resched(); -+ } -+ } else { -+ /* init used_pages of map */ -+ used_pages = vmalloc(sizeof(*used_pages)); -+ if (!used_pages) -+ return -ENOMEM; -+ INIT_LIST_HEAD(&used_pages->list_head); -+ mutex_init(&array->map.used_page_mutex); -+ array->map.used_pages = used_pages; -+ /* WARNING, align to PAGE_SIZE tends to alloc failed */ -+ ptr = __alloc_percpu_gfp(array->elem_size * array->map.max_entries, -+ PAGE_SIZE, GFP_USER | __GFP_NOWARN); -+ if (!ptr) { -+ bpf_array_free_percpu(array); -+ pr_err("alloc percpu failed"); -+ return -ENOMEM; -+ } -+ for (i = 0; i < array->map.max_entries; i++) { -+ array->pptrs[i] = (void *)((u64) ptr + i * array->elem_size); -+ cond_resched(); -+ } ++void __iee_code _iee_set_key_index_key(unsigned long __unused, ++ struct key *key, ++ struct keyring_index_key *index_key) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->index_key = *index_key; ++} + -+ /* record physic pages */ -+ start_address = round_down((u64)ptr, PAGE_SIZE); -+ end_address = round_up((u64)ptr + array->map.max_entries * -+ array->elem_size, PAGE_SIZE); -+ page_cnt = bpf_record_used_pages(start_address, end_address, -+ &array->map.used_page_mutex, &array->map.used_pages->list_head); -+ if (page_cnt >= 0) -+ array->map.used_page_cnt = page_cnt; -+ else -+ return page_cnt; -+ } -+#else - for (i = 0; i < array->map.max_entries; i++) { - ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, - GFP_USER | __GFP_NOWARN); -@@ -44,6 +122,7 @@ static int bpf_array_alloc_percpu(struct bpf_array *array) - array->pptrs[i] = ptr; - cond_resched(); - } -+#endif - - return 0; - } -@@ -89,6 +168,17 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) - u64 array_size, mask64; - struct bpf_array *array; - -+#ifdef CONFIG_HIVE -+ bool is_aggregated; ++void __iee_code _iee_set_key_hash(unsigned long __unused, struct key *key, ++ unsigned long hash) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->hash = hash; ++} + -+ /* bpf_sfi_on && map_need_aggregate => is_aggregated */ -+ if (attr->map_type == BPF_MAP_TYPE_ARRAY || -+ attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) -+ is_aggregated = true; -+ else -+ is_aggregated = false; -+#endif ++void __iee_code _iee_set_key_len_desc(unsigned long __unused, struct key *key, ++ unsigned long len_desc) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->len_desc = len_desc; ++} + - elem_size = round_up(attr->value_size, 8); - - max_entries = attr->max_entries; -@@ -114,12 +204,21 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) - - array_size = sizeof(*array); - if (percpu) { -+ #ifdef CONFIG_HIVE -+ array_size += PAGE_ALIGN((u64) max_entries * sizeof(void *)); -+ #else - array_size += (u64) max_entries * sizeof(void *); -+ #endif - } else { - /* rely on vmalloc() to return page-aligned memory and - * ensure array->value is exactly page-aligned - */ -- if (attr->map_flags & BPF_F_MMAPABLE) { -+ #ifdef CONFIG_HIVE -+ if ((attr->map_flags & BPF_F_MMAPABLE) || is_aggregated) -+ #else -+ if (attr->map_flags & BPF_F_MMAPABLE) -+ #endif -+ { - array_size = PAGE_ALIGN(array_size); - array_size += PAGE_ALIGN((u64) max_entries * elem_size); - } else { -@@ -128,7 +227,13 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) - } - - /* allocate all map elements and zero-initialize them */ -- if (attr->map_flags & BPF_F_MMAPABLE) { -+#ifdef CONFIG_HIVE -+ if ((attr->map_flags & BPF_F_MMAPABLE) || -+ (is_aggregated && !percpu)) -+#else -+ if (attr->map_flags & BPF_F_MMAPABLE) -+#endif -+ { - void *data; - - /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */ -@@ -145,6 +250,16 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) - array->index_mask = index_mask; - array->map.bypass_spec_v1 = bypass_spec_v1; - -+#ifdef CONFIG_HIVE -+ array->map.is_aggregated = is_aggregated; -+ array->map.value = array->value; -+ -+ /* record map_page_cnt for aggregation */ -+ if (!percpu && is_aggregated) -+ array->map.used_page_cnt = PAGE_ALIGN((u64) max_entries * elem_size) -+ >> PAGE_SHIFT; -+#endif -+ - /* copy mandatory map attributes */ - bpf_map_init_from_attr(&array->map, attr); - array->elem_size = elem_size; -@@ -171,7 +286,17 @@ static void *array_map_lookup_elem(struct bpf_map *map, void *key) - if (unlikely(index >= array->map.max_entries)) - return NULL; - -+#ifdef CONFIG_HIVE -+ /* here array->shadow_data equals map->shadow_data */ -+ if (map->is_aggregated && map->shadow_data) -+ return map->shadow_data + array->elem_size * -+ (index & array->index_mask); -+ else { -+ return array->value + array->elem_size * (index & array->index_mask); -+ } -+#else - return array->value + (u64)array->elem_size * (index & array->index_mask); -+#endif - } - - static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, -@@ -184,7 +309,15 @@ static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, - if (off >= map->value_size) - return -EINVAL; - ++void __iee_code _iee_set_key_type(unsigned long __unused, struct key *key, ++ struct key_type *type) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->type = type; ++} ++ ++void __iee_code _iee_set_key_domain_tag(unsigned long __unused, ++ struct key *key, ++ struct key_tag *domain_tag) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->domain_tag = domain_tag; ++} ++ ++void __iee_code _iee_set_key_description(unsigned long __unused, ++ struct key *key, char *description) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->description = description; ++} ++ ++void __iee_code _iee_set_key_uid(unsigned long __unused, struct key *key, ++ kuid_t uid) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->uid = uid; ++} ++ ++void __iee_code _iee_set_key_gid(unsigned long __unused, struct key *key, ++ kgid_t gid) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->gid = gid; ++} + -+#ifdef CONFIG_HIVE -+ if (map->is_aggregated && map->shadow_data) -+ *imm = (unsigned long)map->shadow_data; -+ else -+ *imm = (unsigned long)array->value; -+#else - *imm = (unsigned long)array->value; -+#endif - return 0; - } - -@@ -217,7 +350,21 @@ static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) - if (map->map_flags & BPF_F_INNER_MAP) - return -EOPNOTSUPP; - -+#ifdef CONFIG_HIVE -+ fbpf_log("shadow_data = %016llx\n", map->shadow_data); -+ if (map->is_aggregated && map->shadow_data) { -+ /* substitute array->value with map->shadow_data */ -+ /* assume map is the first attribute of struct array, -+ so that array == array->map -+ array=ffff8000844efe80, &array->map=ffff8000844efe80 */ -+ *insn++ = BPF_LDX_MEM(BPF_DW, map_ptr, map_ptr, offsetof(struct bpf_map, shadow_data)); -+ } -+ else -+ *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, -+ offsetof(struct bpf_array, value)); -+#else - *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); -+#endif - *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); - if (!map->bypass_spec_v1) { - *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); -@@ -237,6 +384,35 @@ static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) - return insn - insn_buf; - } - -+#ifdef CONFIG_HIVE -+extern pte_t *bpf_sfi_get_ptep(u64 addr); -+ -+static void *__percpu_array_map_lookup_elem(struct bpf_array *array, u64 value_ptr) -+{ -+ // struct page *phy_page; -+ // pteval_t phy_pte, vir_pte; -+ struct list_head *used_pages_head; -+ struct bpf_used_page *entry; -+ struct bpf_used_page *next; -+ u64 in_page_offset; -+ -+ in_page_offset = value_ptr - round_down(value_ptr, PAGE_SIZE); -+ // phy_page = kv_virt_to_page((void *)value_ptr); -+ // phy_pte = pte_val(*bpf_sfi_get_ptep(value_ptr)); -+ -+ used_pages_head = &array->map.used_pages->list_head; -+ list_for_each_entry_safe(entry, next, used_pages_head, list_head) { -+ if ((u64)entry->physic_page == round_down(value_ptr, PAGE_SIZE)) { -+ return (void *)entry->shadow_page + in_page_offset; -+ } -+ } ++void __iee_code _iee_set_key_perm(unsigned long __unused, struct key *key, ++ key_perm_t perm) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->perm = perm; ++} + -+ pr_err("fail to find shadow_data of percpu array %016llx\n", -+ (u64)(&array->map)); -+ return NULL; ++void __iee_code _iee_set_key_quotalen(unsigned long __unused, struct key *key, ++ unsigned short quotalen) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->quotalen = quotalen; +} -+#endif + - /* Called from eBPF program */ - static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) - { -@@ -246,7 +422,17 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) - if (unlikely(index >= array->map.max_entries)) - return NULL; - -+#ifdef CONFIG_HIVE -+ if (!map->is_aggregated) -+ return this_cpu_ptr(array->pptrs[index & array->index_mask]); -+ else { -+ u64 value_ptr = (u64)this_cpu_ptr( -+ array->pptrs[index & array->index_mask]); -+ return __percpu_array_map_lookup_elem(array, value_ptr); -+ } -+#else - return this_cpu_ptr(array->pptrs[index & array->index_mask]); -+#endif - } - - static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) -@@ -438,7 +624,11 @@ static void array_map_free(struct bpf_map *map) - if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) - bpf_array_free_percpu(array); - -+#ifdef CONFIG_HIVE -+ if ((array->map.map_flags & BPF_F_MMAPABLE) || array->map.is_aggregated) -+#else - if (array->map.map_flags & BPF_F_MMAPABLE) -+#endif - bpf_map_area_free(array_map_vmalloc_addr(array)); - else - bpf_map_area_free(array); -diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c -index 7c64ad4f3732..f65f1104557d 100644 ---- a/kernel/bpf/hashtab.c -+++ b/kernel/bpf/hashtab.c -@@ -15,6 +15,9 @@ - #include "bpf_lru_list.h" - #include "map_in_map.h" - #include -+#ifdef CONFIG_HIVE -+#include -+#endif - - #define HTAB_CREATE_FLAG_MASK \ - (BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \ -@@ -205,6 +208,19 @@ static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size - return *(void __percpu **)(l->key + key_size); - } - -+#ifdef CONFIG_HIVE -+static inline void htab_normal_elem_set_ptr(struct htab_elem *l, u32 key_size, -+ void *pptr) ++void __iee_code _iee_set_key_datalen(unsigned long __unused, struct key *key, ++ unsigned short datalen) +{ -+ *(void **)(l->key + key_size) = pptr; ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->datalen = datalen; +} + -+static inline void *htab_normal_elem_get_ptr(struct htab_elem *l, u32 key_size) ++void __iee_code _iee_set_key_state(unsigned long __unused, struct key *key, ++ short state) +{ -+ return *(void **)(l->key + key_size); ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ WRITE_ONCE(key->state, state); +} -+#endif + - static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) - { - return *(void **)(l->key + roundup(map->key_size, 8)); -@@ -275,6 +291,35 @@ static void htab_free_elems(struct bpf_htab *htab) - if (!htab_is_percpu(htab)) - goto free_elems; - -+#ifdef CONFIG_HIVE -+ if (unlikely(!htab->map.is_aggregated)) { -+ for (i = 0; i < htab->map.max_entries; i++) { -+ void __percpu *pptr; -+ pptr = htab_elem_get_ptr(get_htab_elem(htab, i), -+ htab->map.key_size); -+ free_percpu(pptr); -+ cond_resched(); -+ } -+ } else { -+ void __percpu *pptr; -+ struct bpf_used_page *used_page_entry, *used_page_next; -+ struct list_head *used_pages_list_head = &htab->map.used_pages->list_head; -+ -+ pptr = htab_elem_get_ptr(get_htab_elem(htab, 0), htab->map.key_size); -+ free_percpu(pptr); -+ cond_resched(); -+ -+ /* free used_pages list */ -+ if (used_pages_list_head && !list_empty(used_pages_list_head)) { -+ list_for_each_entry_safe(used_page_entry, used_page_next, -+ used_pages_list_head, list_head) { -+ list_del(&used_page_entry->list_head); -+ vfree(used_page_entry); -+ } -+ } -+ vfree(htab->map.used_pages); -+ } -+#else - for (i = 0; i < htab->map.max_entries; i++) { - void __percpu *pptr; - -@@ -283,8 +328,13 @@ static void htab_free_elems(struct bpf_htab *htab) - free_percpu(pptr); - cond_resched(); - } -+#endif - free_elems: - bpf_map_area_free(htab->elems); -+#ifdef CONFIG_HIVE -+ if (!htab_is_percpu(htab)) -+ vfree(htab->map.value); -+#endif - } - - /* The LRU list has a lock (lru_lock). Each htab bucket has a lock -@@ -318,18 +368,82 @@ static int prealloc_init(struct bpf_htab *htab) - { - u32 num_entries = htab->map.max_entries; - int err = -ENOMEM, i; -+#ifdef CONFIG_HIVE -+ void *data; -+ u64 value_size, alloc_size; -+#endif - - if (htab_has_extra_elems(htab)) - num_entries += num_possible_cpus(); - - htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries, - htab->map.numa_node); -- if (!htab->elems) -+ if (!htab->elems) { -+ pr_err("htab->elems = %llx\n", (u64)htab->elems); - return -ENOMEM; -+ } - - if (!htab_is_percpu(htab)) - goto skip_percpu_elems; - -+#ifdef CONFIG_HIVE -+realloc_elems: -+ if (!htab->map.is_aggregated) { -+ for (i = 0; i < num_entries; i++) { -+ u32 size = round_up(htab->map.value_size, 8); -+ void __percpu *pptr; -+ -+ pptr = bpf_map_alloc_percpu(&htab->map, size, 8, -+ GFP_USER | __GFP_NOWARN); -+ if (IS_ERR(pptr)) { -+ err = PTR_ERR(pptr); -+ goto free_elems; -+ } -+ htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, -+ pptr); -+ cond_resched(); -+ } -+ } else { -+ u32 size = round_up(htab->map.value_size, 8); -+ void __percpu *pptr; -+ struct bpf_used_page *used_pages; -+ u64 start_address, end_address; -+ int page_cnt; -+ pptr = __alloc_percpu_gfp(size * num_entries, PAGE_SIZE, -+ GFP_USER | __GFP_NOWARN); -+ if (!pptr) { -+ pr_err("pptr = %llx\n", (u64)pptr); -+ htab->map.is_aggregated = false; -+ goto realloc_elems; -+ } ++void __iee_code _iee_set_key_user(unsigned long __unused, struct key *key, ++ struct key_user *user) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->user = user; ++} + -+ for (i = 0; i < num_entries; i++) -+ htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, -+ pptr + size * i); ++void __iee_code _iee_set_key_security(unsigned long __unused, struct key *key, ++ void *security) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->security = security; ++} + -+ /* fBPF init user_pages */ -+ used_pages = vmalloc(sizeof(*used_pages)); -+ if (IS_ERR(used_pages)) { -+ pr_err("used_pages = %llx\n", (u64)used_pages); -+ goto free_elems; -+ } -+ htab->map.used_pages = used_pages; -+ INIT_LIST_HEAD(&used_pages->list_head); -+ mutex_init(&htab->map.used_page_mutex); -+ -+ /* fBPF record percpu physic pages */ -+ start_address = round_down((u64)pptr, PAGE_SIZE); -+ end_address = round_up((u64)pptr + htab->map.max_entries * size, PAGE_SIZE); -+ page_cnt = bpf_record_used_pages(start_address, end_address, -+ &htab->map.used_page_mutex, &htab->map.used_pages->list_head); -+ if (page_cnt < 0) { -+ err = page_cnt; -+ pr_err("page_cnt = %d\n", page_cnt); -+ goto free_elems; -+ } -+ htab->map.used_page_cnt = page_cnt; -+ } -+#else - for (i = 0; i < num_entries; i++) { - u32 size = round_up(htab->map.value_size, 8); - void __percpu *pptr; -@@ -342,8 +456,30 @@ static int prealloc_init(struct bpf_htab *htab) - pptr); - cond_resched(); - } -+#endif - - skip_percpu_elems: -+#ifdef CONFIG_HIVE -+ if (htab_is_percpu(htab) || !htab->map.is_aggregated) -+ goto skip_elems; ++void __iee_code _iee_set_key_expiry(unsigned long __unused, struct key *key, ++ time64_t expiry) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->expiry = expiry; ++} + -+ value_size = round_up(htab->map.value_size, 8); -+ alloc_size = PAGE_ALIGN(value_size * num_entries); -+ // data = bpf_map_area_alloc(alloc_size, htab->map.numa_node); -+ data = vmalloc(alloc_size); -+ if (!data) { -+ pr_err("data = %llx\n", (u64)data); -+ err = -ENOMEM; -+ goto free_elems; -+ } -+ htab->map.value = data; -+ htab->map.used_page_cnt = alloc_size >> PAGE_SHIFT; -+ for (i = 0; i < num_entries; i++) -+ htab_normal_elem_set_ptr(get_htab_elem(htab, i), round_up(htab->map.key_size, 8), -+ data + value_size * i); -+ -+skip_elems: -+#endif - if (htab_is_lru(htab)) - err = bpf_lru_init(&htab->lru, - htab->map.map_flags & BPF_F_NO_COMMON_LRU, -@@ -373,6 +509,29 @@ static int prealloc_init(struct bpf_htab *htab) - return err; - } - -+#ifdef CONFIG_HIVE -+void substitute_hash_value_ptr(struct bpf_map *map) ++void __iee_code _iee_set_key_revoked_at(unsigned long __unused, ++ struct key *key, time64_t revoked_at) +{ -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+ int i; -+ u32 value_size = round_up(map->value_size, 8); -+ int num_entries = map->max_entries; ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->revoked_at = revoked_at; ++} + -+ if (!htab_is_percpu(htab) && !htab_is_lru(htab)) -+ num_entries += num_possible_cpus(); ++void __iee_code _iee_set_key_last_used_at(unsigned long __unused, ++ struct key *key, ++ time64_t last_used_at) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->last_used_at = last_used_at; ++} + -+ for (i = 0; i < num_entries; i++) { -+ htab_normal_elem_set_ptr(get_htab_elem(htab, i), -+ round_up(htab->map.key_size, 8), -+ map->shadow_data + value_size * i); ++unsigned long __iee_code _iee_set_key_usage(unsigned long __unused, struct key *key, ++ int n, int flag) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ switch (flag) { ++ case REFCOUNT_INC: { ++ refcount_inc(&key->usage); ++ break; ++ } ++ case REFCOUNT_SET: { ++ refcount_set(&key->usage, n); ++ break; ++ } ++ case REFCOUNT_DEC_AND_TEST: { ++ return refcount_dec_and_test(&key->usage); ++ } ++ case REFCOUNT_INC_NOT_ZERO: { ++ return refcount_inc_not_zero(&key->usage); + } ++ } ++ return 0; ++} + -+ pr_info("substitute htab %d data from %016llx to %016llx-%016llx\n", -+ map->id, (u64)htab->map.value, (u64)map->shadow_data, -+ (u64)map->shadow_data + value_size * num_entries); ++void __iee_code _iee_set_key_serial(unsigned long __unused, struct key *key, ++ key_serial_t serial) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->serial = serial; +} -+#endif + - static void prealloc_destroy(struct bpf_htab *htab) - { - htab_free_elems(htab); -@@ -489,6 +648,19 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) - - bpf_map_init_from_attr(&htab->map, attr); - -+#ifdef CONFIG_HIVE -+ if (htab->map.map_type == BPF_MAP_TYPE_HASH || -+ htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || -+ htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || -+ htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { -+ htab->map.is_aggregated = true; -+ htab->map.used_pages = NULL; -+ htab->map.map_flags &= ~BPF_F_NO_PREALLOC; -+ // pr_warn("BPF_F_NO_PREALLOC flag is deprecated\n"); -+ prealloc = true; -+ } ++void __iee_code _iee_set_key_watchers(unsigned long __unused, struct key *key, ++ struct watch_list *watchers) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++#ifdef CONFIG_KEY_NOTIFICATIONS ++ key->watchers = watchers; +#endif ++} + - if (percpu_lru) { - /* ensure each CPU's lru list has >=1 elements. - * since we are at it, make each lru list has the same -@@ -512,7 +684,12 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) - - htab->elem_size = sizeof(struct htab_elem) + - round_up(htab->map.key_size, 8); -+#ifdef CONFIG_HIVE -+ /* elem size = htab_elem + key + value */ -+ if (percpu || htab->map.is_aggregated) -+#else - if (percpu) -+#endif - htab->elem_size += sizeof(void *); - else - htab->elem_size += round_up(htab->map.value_size, 8); -@@ -699,8 +876,17 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key) - { - struct htab_elem *l = __htab_map_lookup_elem(map, key); - -+#ifdef CONFIG_HIVE -+ if (l) { -+ if (map->is_aggregated) -+ return htab_normal_elem_get_ptr(l, round_up(map->key_size, 8)); -+ else -+ return htab_normal_elem_get_ptr(l, round_up(map->key_size, 8)); -+ } -+#else - if (l) - return l->key + round_up(map->key_size, 8); -+#endif - - return NULL; - } -@@ -725,9 +911,21 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) - (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); - *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); -+#ifdef CONFIG_HIVE -+ // if (map->is_aggregated) -+ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, -+ offsetof(struct htab_elem, key) + -+ round_up(map->key_size, 8)); -+ // else { -+ // *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, -+ // offsetof(struct htab_elem, key) + -+ // roundup(map->key_size, 8)); -+ // } -+#else - *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + - round_up(map->key_size, 8)); -+#endif - return insn - insn_buf; - } - -@@ -739,7 +937,14 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, - if (l) { - if (mark) - bpf_lru_node_set_ref(&l->lru_node); -+ #ifdef CONFIG_HIVE -+ if (map->is_aggregated) -+ return htab_normal_elem_get_ptr(l, round_up(map->key_size, 8)); -+ else -+ return l->key + round_up(map->key_size, 8); -+ #else - return l->key + round_up(map->key_size, 8); -+ #endif - } - - return NULL; -@@ -774,9 +979,20 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref), - 1); -+#ifdef CONFIG_HIVE -+ if (map->is_aggregated) -+ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, -+ offsetof(struct htab_elem, key) + -+ round_up(map->key_size, 8)); -+ else -+ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, -+ offsetof(struct htab_elem, key) + -+ round_up(map->key_size, 8)); -+#else - *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + - round_up(map->key_size, 8)); -+#endif - return insn - insn_buf; - } - -@@ -1002,7 +1218,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, - bool percpu, bool onallcpus, - struct htab_elem *old_elem) - { -+#ifdef CONFIG_HIVE -+ u32 size = round_up(htab->map.value_size, 8); -+#else - u32 size = htab->map.value_size; -+#endif - bool prealloc = htab_is_prealloc(htab); - struct htab_elem *l_new, **pl_new; - void __percpu *pptr; -@@ -1064,11 +1284,30 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, - htab_elem_set_ptr(l_new, key_size, pptr); - } else if (fd_htab_map_needs_adjust(htab)) { - size = round_up(size, 8); -+ #ifdef CONFIG_HIVE -+ if (htab->map.is_aggregated) { -+ memcpy(htab_normal_elem_get_ptr(l_new, round_up(key_size, 8)), -+ value, size); -+ } else -+ memcpy(l_new->key + round_up(key_size, 8), value, size); -+ #else - memcpy(l_new->key + round_up(key_size, 8), value, size); -+ #endif - } else { -+ #ifdef CONFIG_HIVE -+ if (htab->map.is_aggregated) -+ copy_map_value(&htab->map, -+ htab_normal_elem_get_ptr(l_new, round_up(key_size, 8)), -+ value); -+ else -+ copy_map_value(&htab->map, -+ l_new->key + round_up(key_size, 8), -+ value); -+ #else - copy_map_value(&htab->map, - l_new->key + round_up(key_size, 8), - value); -+ #endif - } - - l_new->hash = hash; -@@ -1129,9 +1368,20 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, - return ret; - if (l_old) { - /* grab the element lock and update value in place */ -+ #ifdef CONFIG_HIVE -+ if (map->is_aggregated) -+ copy_map_value_locked(map, -+ htab_normal_elem_get_ptr(l_old, round_up(key_size, 8)), -+ value, false); -+ else -+ copy_map_value_locked(map, -+ l_old->key + round_up(key_size, 8), -+ value, false); -+ #else - copy_map_value_locked(map, - l_old->key + round_up(key_size, 8), - value, false); -+ #endif - return 0; - } - /* fall through, grab the bucket lock and lookup again. -@@ -1157,9 +1407,20 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, - * grab the element lock in addition to the bucket lock - * and update element in place - */ -+ #ifdef CONFIG_HIVE -+ if (map->is_aggregated) -+ copy_map_value_locked(map, -+ htab_normal_elem_get_ptr(l_old, round_up(key_size, 8)), -+ value, false); -+ else -+ copy_map_value_locked(map, -+ l_old->key + round_up(key_size, 8), -+ value, false); -+ #else - copy_map_value_locked(map, - l_old->key + round_up(key_size, 8), - value, false); -+ #endif - ret = 0; - goto err; - } -@@ -1229,8 +1490,17 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value - l_new = prealloc_lru_pop(htab, key, hash); - if (!l_new) - return -ENOMEM; -+#ifdef CONFIG_HIVE -+ if (map->is_aggregated) -+ copy_map_value(&htab->map, -+ htab_normal_elem_get_ptr(l_new, round_up(map->key_size, 8)), value); -+ else -+ copy_map_value(&htab->map, -+ l_new->key + round_up(map->key_size, 8), value); -+#else - copy_map_value(&htab->map, - l_new->key + round_up(map->key_size, 8), value); -+#endif - - ret = htab_lock_bucket(htab, b, hash, &flags); - if (ret) -@@ -2298,13 +2568,52 @@ const struct bpf_map_ops htab_lru_map_ops = { - .iter_seq_info = &iter_seq_info, - }; - -+#ifdef CONFIG_HIVE -+static void *percpu_htab_aggregated_addr(struct bpf_htab *htab, const void *value_ptr1) ++void __iee_code _iee_set_key_union(unsigned long __unused, struct key *key, ++ struct key_union *key_union) +{ -+ // struct page *phy_ptr; -+ struct list_head *used_pages_head; -+ struct bpf_used_page *entry; -+ struct bpf_used_page *next; -+ u64 in_page_offset; ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->graveyard_link.next = (struct list_head *)key_union; ++} + -+ // phy_ptr = kv_virt_to_page(value_ptr1); -+ used_pages_head = &htab->map.used_pages->list_head; -+ in_page_offset = (u64)value_ptr1 - round_down((u64)value_ptr1, PAGE_SIZE); ++void __iee_code _iee_set_key_struct(unsigned long __unused, struct key *key, ++ struct key_struct *key_struct) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->name_link.prev = (struct list_head *)key_struct; ++} + -+ if (!used_pages_head) { -+ pr_err("used_pages_head is empty = %llx\n", (u64)used_pages_head); -+ return ERR_PTR(-ENOMEM); -+ } ++void __iee_code _iee_set_key_payload(unsigned long __unused, struct key *key, ++ union key_payload *key_payload) ++{ ++ iee_verify_key_type(key); ++ key = __ptr_to_iee(key); ++ key->name_link.next = (struct list_head *)key_payload; ++} +diff --git a/arch/arm64/kernel/haoc/ptp/Makefile b/arch/arm64/kernel/haoc/ptp/Makefile +new file mode 100644 +index 000000000000..c2887cf5c18c +--- /dev/null ++++ b/arch/arm64/kernel/haoc/ptp/Makefile +@@ -0,0 +1 @@ ++obj-y += ptp.o iee-ptp-init.o +diff --git a/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c +new file mode 100644 +index 000000000000..e0dcbb59dddd +--- /dev/null ++++ b/arch/arm64/kernel/haoc/ptp/iee-ptp-init.c +@@ -0,0 +1,406 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+ list_for_each_entry_safe(entry, next, used_pages_head, list_head) { -+ if ((u64)entry->physic_page == round_down((u64)value_ptr1, PAGE_SIZE)) { -+ return (void *)entry->shadow_page + in_page_offset; -+ } -+ } ++#define PTP_CHECK(condition) do { \ ++ if (unlikely(condition)) \ ++ panic("PTP check failed on %s.", __func__); \ ++} while (0) + -+ return 0; ++#if CONFIG_PGTABLE_LEVELS > 3 ++void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) ++{ ++ iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); +} -+#endif -+ - /* Called from eBPF program */ - static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) - { - struct htab_elem *l = __htab_map_lookup_elem(map, key); - -+#ifdef CONFIG_HIVE -+ if (l) { -+ if (map->is_aggregated && map->shadow_data) { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+ void *value_ptr = this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -+ return percpu_htab_aggregated_addr(htab, value_ptr); -+ } else -+ return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -+ } -+#else - if (l) - return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -+#endif - else - return NULL; - } -@@ -2317,8 +2626,20 @@ static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, - return NULL; - - l = __htab_map_lookup_elem(map, key); -+#ifdef CONFIG_HIVE -+ if (l) { -+ if (map->is_aggregated) { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+ void *value_ptr = this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -+ return percpu_htab_aggregated_addr(htab, value_ptr); -+ } -+ else -+ return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); -+ } +#else - if (l) - return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); ++void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) ++{ ++ BUILD_BUG(); ++} +#endif - else - return NULL; - } -diff --git a/kernel/bpf/sfi_bpf.c b/kernel/bpf/sfi_bpf.c -new file mode 100644 -index 000000000000..93d8c4977820 ---- /dev/null -+++ b/kernel/bpf/sfi_bpf.c -@@ -0,0 +1,1387 @@ -+#include -+#include -+#include "disasm.h" -+#include -+ -+extern pte_t *bpf_sfi_get_ptep(u64 addr); -+ -+const enum bpf_map_type can_lookup_map_types[] = { -+ BPF_MAP_TYPE_ARRAY, -+ BPF_MAP_TYPE_PERCPU_ARRAY, -+ BPF_MAP_TYPE_HASH, -+ BPF_MAP_TYPE_LRU_HASH, -+ BPF_MAP_TYPE_PERCPU_HASH, -+ BPF_MAP_TYPE_LRU_PERCPU_HASH, -+ BPF_MAP_TYPE_CPUMAP, -+ BPF_MAP_TYPE_CGROUP_STORAGE, -+ BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, -+ BPF_MAP_TYPE_QUEUE, -+ BPF_MAP_TYPE_STACK, -+ BPF_MAP_TYPE_SK_STORAGE, -+ BPF_MAP_TYPE_INODE_STORAGE, -+ BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, -+ BPF_MAP_TYPE_LPM_TRIE, -+ BPF_MAP_TYPE_SOCKMAP, -+ BPF_MAP_TYPE_SOCKHASH, -+}; + -+void do_fbpf_log(const char *func_name, const char *fmt, ...) ++void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) +{ -+ #ifdef FBPF_LOG_ENABLE -+ va_list args; -+ char buffer[256]; -+ -+ va_start(args, fmt); -+ vsnprintf(buffer, sizeof(buffer), fmt, args); -+ va_end(args); -+ -+ pr_info("%s: %s", func_name, buffer); -+ #endif ++ iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); +} + -+const char *bpf_map_type_strings[] = { -+ "BPF_MAP_TYPE_UNSPEC", -+ "BPF_MAP_TYPE_HASH", -+ "BPF_MAP_TYPE_ARRAY", -+ "BPF_MAP_TYPE_PROG_ARRAY", -+ "BPF_MAP_TYPE_PERF_EVENT_ARRAY", -+ "BPF_MAP_TYPE_PERCPU_HASH", -+ "BPF_MAP_TYPE_PERCPU_ARRAY", -+ "BPF_MAP_TYPE_STACK_TRACE", -+ "BPF_MAP_TYPE_CGROUP_ARRAY", -+ "BPF_MAP_TYPE_LRU_HASH", -+ "BPF_MAP_TYPE_LRU_PERCPU_HASH", -+ "BPF_MAP_TYPE_LPM_TRIE", -+ "BPF_MAP_TYPE_ARRAY_OF_MAPS", -+ "BPF_MAP_TYPE_HASH_OF_MAPS", -+ "BPF_MAP_TYPE_DEVMAP", -+ "BPF_MAP_TYPE_SOCKMAP", -+ "BPF_MAP_TYPE_CPUMAP", -+ "BPF_MAP_TYPE_XSKMAP", -+ "BPF_MAP_TYPE_SOCKHASH", -+ "BPF_MAP_TYPE_CGROUP_STORAGE", -+ "BPF_MAP_TYPE_REUSEPORT_SOCKARRAY", -+ "BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE", -+ "BPF_MAP_TYPE_QUEUE", -+ "BPF_MAP_TYPE_STACK", -+ "BPF_MAP_TYPE_SK_STORAGE", -+ "BPF_MAP_TYPE_DEVMAP_HASH", -+ "BPF_MAP_TYPE_STRUCT_OPS", -+ "BPF_MAP_TYPE_RINGBUF", -+ "BPF_MAP_TYPE_INODE_STORAGE", -+}; ++void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, ++ pmdval_t prot) ++{ ++ iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); ++} + -+int bpf_record_used_pages(u64 start_addr, u64 end_addr, -+ struct mutex *write_mutex, struct list_head *head) ++int iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot) +{ -+ int cpu; -+ u64 page_addr = start_addr; -+ int mmapable_page_cnt = 0; -+ -+ while (page_addr < end_addr) { -+ for_each_possible_cpu(cpu) { -+ // struct page *physic_page; -+ void *ptr; -+ struct bpf_used_page* new_page; -+ -+ ptr = per_cpu_ptr((void *)page_addr, cpu); -+ // if (is_vmalloc_addr(ptr)) { -+ // physic_page = vmalloc_to_page(ptr); -+ // } else { -+ // physic_page = virt_to_page(ptr); -+ // } -+ // if (!physic_page) { -+ // pr_err("cannot find physic page for %016llx\n", page_addr); -+ // return -ENOENT; -+ // } -+ new_page = vmalloc(sizeof(*new_page)); -+ new_page->physic_page = ptr; -+ -+ mutex_lock(write_mutex); -+ // printk("virt addr=%016llx\n", (u64)ptr); -+ list_add(&new_page->list_head, head); -+ mmapable_page_cnt++; -+ mutex_unlock(write_mutex); -+ } -+ page_addr += PAGE_SIZE; -+ } ++ pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); + -+ return mmapable_page_cnt; -+} ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), ++ pud_val(new_pud))) ++ return 0; + -+struct page *kv_virt_to_page(const void *vaddr) -+{ -+ if (is_vmalloc_addr(vaddr)) -+ return vmalloc_to_page(vaddr); -+ else -+ return virt_to_page(vaddr); ++ WARN_ON_ONCE(phys & ~PUD_MASK); ++ iee_set_pud_pre_init(pudp, new_pud); ++ return 1; +} + -+static bool map_can_lookup(struct bpf_map *map) ++int iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) +{ -+ int i; -+ enum bpf_map_type type = map->map_type; ++ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); + -+ for (i = 0; i < ARRAY_SIZE(can_lookup_map_types); i++) -+ if (type == can_lookup_map_types[i]) -+ return true; ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), ++ pmd_val(new_pmd))) ++ return 0; + -+ return false; ++ WARN_ON_ONCE(phys & ~PMD_MASK); ++ iee_set_pmd_pre_init(pmdp, new_pmd); ++ return 1; +} + -+static int get_env_map_cnt(struct bpf_verifier_env *env) ++static inline pte_t *fixmap_pte(unsigned long addr) +{ -+ int i; -+ struct bpf_map *map; -+ int page_cnt = 0; -+ -+ // calculate maps page count -+ for (i = 0; i < env->used_map_cnt; i++) { -+ map = env->used_maps[i]; -+ if (map->is_aggregated && map->used_page_cnt) { -+ page_cnt += map->used_page_cnt; -+ fbpf_log("map %d %s page_cnt = %d\n", map->id, -+ bpf_map_type_strings[map->map_type], map->used_page_cnt); -+ } else if (map_can_lookup(map)) { -+ /* if do not support this map, just return to normal bpf load */ -+ pr_err("unsupport map_type %s\n", bpf_map_type_strings[map->map_type]); -+ return -EPERM; -+ // continue; -+ } -+ } -+ -+ return page_cnt; ++ return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; +} + -+static inline u64 count_page(struct page **pages, u64 start_addr, int *page_index) ++void __iee_set_fixmap_pre_init(enum fixed_addresses idx, ++ phys_addr_t phys, pgprot_t flags) +{ -+ pages[*page_index] = (struct page *)start_addr; ++ unsigned long addr = __fix_to_virt(idx); ++ pte_t *ptep; + -+ // pr_info("addr of page %d = %llx\n", *page_index, (u64)start_addr); ++ PTP_CHECK(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + -+ start_addr += PAGE_SIZE; -+ (*page_index)++; ++ ptep = fixmap_pte(addr); + -+ return start_addr; ++ if (pgprot_val(flags)) { ++ iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ } else { ++ iee_set_pte_pre_init(ptep, __pte(0)); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ } +} + -+static int count_percpu_pages(struct bpf_map *map, struct page **pages, int *page_index) ++void iee_set_pgtable_pre_init(unsigned long *addr, unsigned long content) +{ -+ struct list_head *used_pages_head = NULL; -+ struct bpf_used_page *entry; -+ struct bpf_used_page *next; -+ int i = 0; -+ -+ used_pages_head = &map->used_pages->list_head; -+ -+ /* should not list_del and free here, later other progs may reuse this map */ -+ list_for_each_entry_safe(entry, next, used_pages_head, list_head) { -+ // printk("count page %016llx\n", entry->physic_page); -+ // pages[*page_index] = entry->physic_page; -+ // pages[*page_index] = page_address(entry->physic_page); -+ // entry->shadow_page = *page_index; -+ // if (!pages[*page_index]) { -+ // pr_err("illegal address=%016llx\n", (u64)pages[*page_index]); -+ // return -EFAULT; -+ // } -+ // pr_info("addr of page %d = %llx\n", *page_index, (u64)page_address(entry->physic_page)); -+ // (*page_index)++; -+ count_page(pages, (u64)entry->physic_page, page_index); -+ entry->shadow_page = i; -+ i++; -+ } -+ -+ return 0; ++ WRITE_ONCE(*addr, content); +} + -+static int count_all_pages(struct bpf_verifier_env *env, struct page **pages, int *page_index) ++void set_iee_address_pre_init(unsigned long addr, bool valid) +{ -+ struct bpf_map *map; -+ int i, j; -+ u64 start_addr; -+ int err; -+ int page_index_before; -+ -+ /* 1. coutn pages in low guard region */ -+ // start_addr = PAGE_ALIGN((u64)env->prog->low_guard_region_addr); -+ // for (i = 0; i < env->prog->guard_region_page_cnt / 2; i++) { -+ // // pages[*page_index] = kv_virt_to_page((void *)start_addr); -+ // start_addr = count_page(pages, start_addr, page_index); -+ -+ // /* DEBUG check addr */ -+ // // pr_info("check page %d addr 0x%llx\n", i, start_addr); -+ // // *(u64 *)start_addr = 0; -+ // // pr_info("write into 0x%llx page pass\n", start_addr); ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + -+ // /* DEBUG check get_ptep */ -+ // // pte_t *ptep = bpf_sfi_get_ptep(start_addr); -+ // // if (IS_ERR(ptep)) -+ // // pr_err("get ptep of 0x%llx failed\n", start_addr); -+ // // else -+ // // pr_info("get ptep of 0x%llx pass\n", start_addr); -+ // } ++ p4d_t *p4dp = p4d_offset(pgdp, addr); + -+ /* 2. count pages in map's data area */ -+ for (i = 0; i < env->used_map_cnt; i++) { -+ page_index_before = *page_index; -+ map = env->used_maps[i]; ++ pud_t *pudp = pud_offset(p4dp, addr); + -+ if (!map->is_aggregated) -+ continue; ++ pmd_t *pmdp = pmd_offset(pudp, addr); + -+ if (map->map_type == BPF_MAP_TYPE_ARRAY || -+ map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) { -+ start_addr = (u64)map->value; -+ } else if (map->map_type == BPF_MAP_TYPE_HASH || -+ map->map_type == BPF_MAP_TYPE_LRU_HASH) { -+ start_addr = (u64)map->value; -+ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || -+ map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -+ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { -+ err = count_percpu_pages(map, pages, page_index); -+ if (err) -+ return err; -+ goto hook_end; -+ } else if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { -+ pr_err("WARNING: ignore map %s\n", bpf_map_type_strings[map->map_type]); -+ goto hook_end; -+ } else { -+ pr_err("ERROR: miss map %s in count_all_pages()\n", -+ bpf_map_type_strings[map->map_type]); -+ return -ENOENT; -+ } ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); + -+ /* assert start_addr is page_aligned */ -+ if (unlikely(!PAGE_ALIGNED(start_addr))) { -+ pr_err("ERROR: fbpf self, map_data should be PAGE_ALIGNED\n"); -+ pr_err("map %s, addr=%016llx", -+ bpf_map_type_strings[map->map_type], (u64)map); -+ return -EFAULT; -+ } ++ if (valid) ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ else ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ iee_set_pgtable_pre_init((unsigned long *)ptep, (unsigned long)pte.pte); ++} + -+ /* for normal map with base addr */ -+ for (j = 0; j < map->used_page_cnt; j++) { -+ // pages[*page_index] = kv_virt_to_page((void *)start_addr); -+ start_addr = count_page(pages, start_addr, page_index); -+ } ++static void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot) ++{ ++ pte_t *ptep; + -+hook_end: -+ fbpf_log("hook aggregated area for map %d: %s, %016llx, \ -+page_cnt: %d, should be: %d\n", -+ map->id, bpf_map_type_strings[map->map_type], (u64)map, -+ *page_index - page_index_before, map->used_page_cnt); -+ } ++ ptep = pte_set_fixmap_offset_pre_init(pmdp, addr); ++ do { ++ pte_t old_pte = __ptep_get(ptep); + -+ /* 3. count pages in skb area */ -+ start_addr = PAGE_ALIGN((u64)env->prog->shadow_skb_addr); -+ for (i = 0; i < env->prog->shadow_skb_page_cnt; i++) { -+ // pages[*page_index] = kv_virt_to_page((void *)start_addr); -+ start_addr = count_page(pages, start_addr, page_index); -+ } ++ iee_set_pgtable_pre_init((unsigned long *)ptep, ++ (unsigned long)(pfn_pte(__phys_to_pfn(phys), prot).pte)); + -+ /* 4. count pages in stack area */ -+ start_addr = PAGE_ALIGN((u64)env->prog->shadow_stack_addr); -+ for (i = 0; i < env->prog->shadow_stack_page_cnt; i++) { -+ // pages[*page_index] = kv_virt_to_page((void *)start_addr); -+ start_addr = count_page(pages, start_addr, page_index); -+ } ++ /* ++ * After the PTE entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ PTP_CHECK(!pgattr_change_is_safe(pte_val(old_pte), ++ pte_val(__ptep_get(ptep)))); + -+ /* 5. coutn pages in high guard region */ -+ // start_addr = PAGE_ALIGN((u64)env->prog->high_guard_region_addr); -+ // for (i = 0; i < env->prog->guard_region_page_cnt / 2; i++) { -+ // start_addr = count_page(pages, start_addr, page_index); -+ // } ++ phys += PAGE_SIZE; ++ } while (ptep++, addr += PAGE_SIZE, addr != end); + -+ return 0; ++ pte_clear_fixmap_pre_init(); +} + -+static int record_percpu_map_shadow_page(struct bpf_map *map) ++static void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) +{ -+ struct list_head *used_pages_head = NULL; -+ struct bpf_used_page *entry; -+ struct bpf_used_page *next; -+ void *shadow_base_addr = map->shadow_data; ++ unsigned long next; ++ pmd_t pmd = READ_ONCE(*pmdp); + -+ used_pages_head = &map->used_pages->list_head; ++ PTP_CHECK(pmd_sect(pmd)); ++ if (pmd_none(pmd)) { ++ pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF; ++ phys_addr_t pte_phys; + -+ /* fbpf should not list_del and free here, -+ later other progs may reuse this map. -+ instead, map_free will do this job */ -+ list_for_each_entry_safe(entry, next, used_pages_head, list_head) { -+ entry->shadow_page = (u64)shadow_base_addr + PAGE_SIZE * -+ entry->shadow_page; /* shadow_page is the index recorded in count_all_pages() */ -+ pteval_t phy_pte, vir_pte; -+ phy_pte = pte_val(*bpf_sfi_get_ptep((u64)entry->physic_page)); -+ vir_pte = pte_val(*bpf_sfi_get_ptep((u64)entry->shadow_page)); -+ if (unlikely(phy_pte != vir_pte)) { -+ pr_err("ERROR: recorded shadow_page unequal actual shadow_page\n"); -+ return -EFAULT; -+ } ++ if (flags & NO_EXEC_MAPPINGS) ++ pmdval |= PMD_TABLE_PXN; ++ PTP_CHECK(!pgtable_alloc); ++ pte_phys = pgtable_alloc(PAGE_SHIFT); ++ iee_set_pgtable_pre_init((unsigned long *)pmdp, ++ (unsigned long)(__phys_to_pmd_val(pte_phys) | pmdval)); ++ pmd = READ_ONCE(*pmdp); + } ++ PTP_CHECK(pmd_bad(pmd)); + -+ return 0; -+} ++ do { ++ pgprot_t __prot = prot; + -+// static inline int set_as_guard_page(unsigned long addr) -+// { -+// pte_t pte, *ptep; -+ -+// ptep = bpf_sfi_get_ptep(addr); -+// if (IS_ERR(ptep)) -+// return PTR_ERR(ptep); ++ next = pte_cont_addr_end(addr, end); + -+// pte = __pte(pte_val(*ptep) & ~PTE_VALID); -+// set_pte(ptep, __pte((pte_val(pte) | PTE_BPF_SFI_GP))); ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + -+// flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -+// isb(); ++ iee_init_pte_pre_init(pmdp, addr, next, phys, __prot); + -+// return 0; -+// } ++ phys += next - addr; ++ } while (addr = next, addr != end); ++} + -+inline void *alloc_aligned_memory(u64 size) ++static void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) +{ -+ // unsigned long order = get_order(size); -+ // void *ptr = (void *)__get_free_pages(GFP_KERNEL, order); -+ int page_cnt = size >> PAGE_SHIFT; -+ void *base = vmalloc((2 * page_cnt - 1) * PAGE_SIZE); -+ if (IS_ERR(base)) { -+ printk(KERN_ERR "Failed to allocate memory\n"); -+ return ERR_PTR(-ENOMEM); -+ } ++ unsigned long next; ++ pmd_t *pmdp; + -+ void *ptr = (void *)(((unsigned long)base + page_cnt * PAGE_SIZE) & ~(size - 1)); ++ pmdp = pmd_set_fixmap_offset_pre_init(pudp, addr); ++ do { ++ pmd_t old_pmd = READ_ONCE(*pmdp); + -+ // check alignment -+ if ((unsigned long)ptr & (size - 1)) { -+ // free_pages((unsigned long)ptr, order); -+ vfree(base); -+ pr_err("Memory not properly aligned\n"); -+ return ERR_PTR(-ENOMEM); -+ } ++ next = pmd_addr_end(addr, end); + -+ return ptr; -+} ++ /* try section mapping first */ ++ if (((addr | next | phys) & ~PMD_MASK) == 0 && ++ (flags & NO_BLOCK_MAPPINGS) == 0) { ++ iee_set_pgtable_pre_init((unsigned long *)pmdp, ++ (unsigned long)(pfn_pmd(__phys_to_pfn(phys), ++ mk_pmd_sect_prot(prot)).pmd)); + -+inline int map_physic_pages(struct page **pages, void *virt_addr, int page_cnt) -+{ -+ int i; ++ /* ++ * After the PMD entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ PTP_CHECK(!pgattr_change_is_safe(pmd_val(old_pmd), ++ READ_ONCE(pmd_val(*pmdp)))); ++ } else { ++ iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot, ++ pgtable_alloc, flags); + -+ for (i = 0; i < page_cnt; i++) { -+ // u64 origin_addr = page_to_pfn(pages[i]) << PAGE_SHIFT; -+ // u64 origin_addr = (u64)page_address(pages[i]); -+ u64 origin_addr = (u64)pages[i]; -+ // pr_info("get origin ptep of page %d, addr = %llx\n", i, origin_addr); -+ pte_t *origin_ptep = bpf_sfi_get_ptep(origin_addr); -+ if (IS_ERR(origin_ptep)) { -+ // pr_err("get origin ptep of 0x%llx failed\n", origin_addr); -+ return PTR_ERR(origin_ptep); -+ } -+ -+ // pr_info("get cur ptep of page %d\n", i); -+ u64 cur_addr = (u64)virt_addr + i * PAGE_SIZE; -+ pte_t *ptep = bpf_sfi_get_ptep(cur_addr); -+ if (IS_ERR(ptep)) { -+ // pr_err("get sfi ptep of 0x%llx failed\n", cur_addr); -+ return PTR_ERR(ptep); ++ PTP_CHECK(pmd_val(old_pmd) != 0 && ++ pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + } -+ -+ #ifdef CONFIG_PTP -+ set_pte(ptep, *origin_ptep); -+ #else -+ *ptep = *origin_ptep; -+ #endif -+ } ++ phys += next - addr; ++ } while (pmdp++, addr = next, addr != end); + -+ return 0; ++ pmd_clear_fixmap_pre_init(); +} + -+/** -+ * sfi_bpf space: -+ * guard page 8 pages -+ * map's data 0xffff80008457f000 - 0xffff80008457f000 -+ * skb 0xffff80008457f000 - 0xffff80008457f000 -+ * stack 0xffff80008457f000 - 0xffff800084580000 -+ * guard page 8 pages -+ */ -+int bpf_sfi_aggregate_memory(struct bpf_verifier_env *env) -+{ -+ struct page **pages = NULL; -+ struct bpf_map *map; -+ u64 start_addr; -+ void *shadow_base_addr = NULL, *shadow_high_addr = NULL; -+ int i, page_index = 0; -+ int total_page_cnt, map_page_cnt, stack_page_cnt, skb_page_cnt/*, guard_page_cnt*/; -+ int err; -+ void *shadow_stack_addr, *shadow_skb_addr; -+ -+ /* calculate page_cnt */ -+ env->prog->map_page_cnt = map_page_cnt = get_env_map_cnt(env); -+ stack_page_cnt = round_up(MAX_BPF_STACK, PAGE_SIZE) >> PAGE_SHIFT; -+ if (env->prog->type == BPF_PROG_TYPE_SCHED_CLS || -+ env->prog->type == BPF_PROG_TYPE_SCHED_ACT || -+ env->prog->type == BPF_PROG_TYPE_XDP || -+ env->prog->type == BPF_PROG_TYPE_LWT_XMIT || -+ env->prog->type == BPF_PROG_TYPE_SK_SKB || -+ env->prog->type == BPF_PROG_TYPE_SK_MSG) -+ skb_page_cnt = 16; -+ else -+ skb_page_cnt = 0; -+ // guard_page_cnt = 16; -+ -+ /* roundup totoal page number to power of two for SFI consideration */ -+ env->prog->shadow_skb_page_cnt = skb_page_cnt; -+ env->prog->total_page_cnt = total_page_cnt = roundup_pow_of_two(stack_page_cnt + map_page_cnt + skb_page_cnt); -+ env->prog->shadow_stack_page_cnt = stack_page_cnt = total_page_cnt - map_page_cnt - skb_page_cnt; -+ pr_info("page_cnt: map=%d, skb=%d, stack=%d\n", -+ map_page_cnt, skb_page_cnt, stack_page_cnt); -+ -+ /* map map's value area to pages */ -+ pages = kzalloc(sizeof(struct page *) * (total_page_cnt), GFP_KERNEL | __GFP_NOWARN); -+ if (IS_ERR(pages)) { -+ pr_err("ERROR: alloc tmp pages array failed\n"); -+ err = PTR_ERR(pages); -+ goto ret; -+ } -+ -+ /* alloc tmp skb page to fill in pages */ -+ if (skb_page_cnt) { -+ shadow_skb_addr = vmalloc(skb_page_cnt * PAGE_SIZE); -+ if (IS_ERR(shadow_skb_addr)) { -+ pr_err("alloc skb area failed\n"); -+ err = PTR_ERR(shadow_skb_addr); -+ goto ret; -+ } -+ env->prog->shadow_skb_addr = shadow_skb_addr; -+ } ++static void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) ++{ ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); + -+ /* alloc shadow stack value area */ -+ if (stack_page_cnt) { -+ shadow_stack_addr = vmalloc(stack_page_cnt * PAGE_SIZE); -+ if (IS_ERR(shadow_stack_addr)) { -+ pr_err("alloc stack area failed\n"); -+ err = PTR_ERR(shadow_stack_addr); -+ goto ret; -+ } -+ env->prog->shadow_stack_addr = shadow_stack_addr; -+ } else { -+ pr_err("no bpf stack is alloced!\n"); -+ err = -ENOMEM; -+ goto ret; -+ } -+ -+ /* alloc guard region */ -+ // if (guard_page_cnt) { -+ // void *guard_region_addr = vmalloc(guard_page_cnt * PAGE_SIZE); -+ // if (IS_ERR(guard_region_addr)) { -+ // pr_err("alloc guard region failed\n"); -+ // err = PTR_ERR(guard_region_addr); -+ // goto ret; -+ // } -+ // env->prog->low_guard_region_addr = guard_region_addr; -+ // env->prog->high_guard_region_addr = guard_region_addr + guard_page_cnt / 2 * PAGE_SIZE; -+ // } ++ /* ++ * Check for initial section mappings in the pgd/pud. ++ */ ++ PTP_CHECK(pud_sect(pud)); ++ if (pud_none(pud)) { ++ pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF; ++ phys_addr_t pmd_phys; + -+ /* fill the array pages */ -+ err = count_all_pages(env, pages, &page_index); -+ if (err) -+ goto ret; -+ -+ /* check every page is counted */ -+ fbpf_log("page index=%d, page cnt=%d (should equal)\n", page_index, total_page_cnt); -+ if(unlikely(page_index != total_page_cnt)) { -+ pr_err("ERROR: page_index != page_cnt\n"); -+ err = -EFAULT; -+ goto ret; ++ if (flags & NO_EXEC_MAPPINGS) ++ pudval |= PUD_TABLE_PXN; ++ PTP_CHECK(!pgtable_alloc); ++ pmd_phys = pgtable_alloc(PMD_SHIFT); ++ iee_set_pgtable_pre_init((unsigned long *)pudp, ++ (unsigned long)(__phys_to_pud_val(pmd_phys) | pudval)); ++ pud = READ_ONCE(*pudp); + } ++ PTP_CHECK(pud_bad(pud)); + -+ /* remap memory area to virtually contiguous space */ -+ // shadow_base_addr = vmap(pages, total_page_cnt, VM_MAP, PAGE_KERNEL); -+ shadow_base_addr = alloc_aligned_memory(total_page_cnt * PAGE_SIZE); -+ if (IS_ERR(shadow_base_addr)) { -+ pr_err("unable to vmap map's value area and stack area\n"); -+ err = PTR_ERR(shadow_base_addr); -+ goto ret; -+ } -+ shadow_high_addr = shadow_base_addr + total_page_cnt * PAGE_SIZE; -+ env->prog->shadow_region_addr = shadow_base_addr; -+ env->prog->shadow_top_addr = shadow_high_addr; ++ do { ++ pgprot_t __prot = prot; + -+ fbpf_log("shadow_base = 0x%llx, shadow_stack = 0x%llx\n", shadow_base_addr, shadow_high_addr); ++ next = pmd_cont_addr_end(addr, end); + -+ err = map_physic_pages(pages, shadow_base_addr, total_page_cnt); -+ if (err < 0) { -+ pr_err("remap failed, %d\n", err); -+ goto ret; -+ } ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + -+ /* substitute with SFI address space addrs */ -+ start_addr = (u64)shadow_base_addr; -+ -+ /* low guard region */ -+ // env->prog->low_guard_region_addr = (void *)start_addr; -+ // start_addr += guard_page_cnt / 2 * PAGE_SIZE; ++ iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags); + -+ /* IMPORTANT: update shadow addr for maps */ -+ env->prog->map_data_addr = (void *)start_addr; -+ for (i = 0; i < env->used_map_cnt; i++) { -+ map = env->used_maps[i]; ++ phys += next - addr; ++ } while (addr = next, addr != end); ++} + -+ if (!map->is_aggregated) -+ continue; ++void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long next; ++ pud_t *pudp; ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); + -+ /* record addr mapping info */ -+ map->shadow_data = (void *)start_addr; -+ -+ if (map->map_type == BPF_MAP_TYPE_HASH || -+ map->map_type == BPF_MAP_TYPE_LRU_HASH) { -+ substitute_hash_value_ptr(map); -+ } else if (map->map_type == BPF_MAP_TYPE_ARRAY || -+ map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) { -+ ; // only need to record map->shadow_data -+ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY || -+ map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -+ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { -+ err = record_percpu_map_shadow_page(map); -+ if (err) -+ goto ret; -+ } else { -+ ; // no need to record or substitute -+ } ++ if (p4d_none(p4d)) { ++ p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF; ++ phys_addr_t pud_phys; + -+ start_addr += map->used_page_cnt * PAGE_SIZE; ++ if (flags & NO_EXEC_MAPPINGS) ++ p4dval |= P4D_TABLE_PXN; ++ PTP_CHECK(!pgtable_alloc); ++ pud_phys = pgtable_alloc(PUD_SHIFT); ++ iee_set_pgtable_pre_init((unsigned long *)p4dp, ++ (unsigned long)(__phys_to_p4d_val(pud_phys) | p4dval)); ++ p4d = READ_ONCE(*p4dp); + } ++ PTP_CHECK(p4d_bad(p4d)); + -+ /* the space between map's data and stack is for skb_buff */ -+ env->prog->shadow_skb_addr = (void *)start_addr; -+ start_addr += skb_page_cnt * PAGE_SIZE; -+ -+ /* stack */ -+ /* NOTICE: stack's addr is high addr (sp), rather than low addr (fp) */ -+ start_addr += stack_page_cnt * PAGE_SIZE; -+ env->prog->shadow_stack_addr = (void *)start_addr; -+ -+ /* high guard region */ -+ // env->prog->high_guard_region_addr = (void *)start_addr; -+ // start_addr += guard_page_cnt / 2 * PAGE_SIZE; -+ -+ BUG_ON(unlikely(start_addr != (u64)(env->prog->shadow_top_addr))); -+ -+ // set guard region and skb region as guard page -+ // start_addr = (u64)env->prog->low_guard_region_addr; -+ // for (int i = 0; i < guard_page_cnt / 2; i++) { -+ // err = set_as_guard_page(start_addr); -+ // if (err < 0) { -+ // pr_err("set as guard page failed, %d\n", err); -+ // goto ret; -+ // } -+ // start_addr += PAGE_SIZE; -+ // } -+ // start_addr = (u64)env->prog->high_guard_region_addr; -+ // for (int i = 0; i < guard_page_cnt / 2; i++) { -+ // err = set_as_guard_page(start_addr); -+ // if (err < 0) { -+ // pr_err("set as guard page failed, %d\n", err); -+ // goto ret; -+ // } -+ // start_addr += PAGE_SIZE; -+ // } -+ // start_addr = (u64)env->prog->shadow_skb_addr; -+ // for (int i = 0; i < skb_page_cnt; i++) { -+ // err = set_as_guard_page(start_addr); -+ // if (err < 0) { -+ // pr_err("set as guard page failed, %d\n", err); -+ // goto ret; -+ // } -+ // start_addr += PAGE_SIZE; -+ // } -+ -+ // manually trigger page fault now -+ // pr_err("start probe now\n"); -+ // *(u64 *)(env->prog->low_guard_region_addr + 16) = 1; -+ // pr_err("oops, back from page fault!\n"); -+ -+ fbpf_log("sfi_bpf space:\n"); -+ // fbpf_log("low guard region\t0x%016llx - 0x%016llx\n", -+ // env->prog->low_guard_region_addr, env->prog->low_guard_region_addr + guard_page_cnt / 2 * PAGE_SIZE); -+ fbpf_log("map's data\t\t0x%016llx - 0x%016llx\n", -+ env->prog->map_data_addr, env->prog->map_data_addr + map_page_cnt * PAGE_SIZE); -+ fbpf_log("skb\t\t\t0x%016llx - 0x%016llx\n", -+ env->prog->shadow_skb_addr, env->prog->shadow_skb_addr + skb_page_cnt * PAGE_SIZE); -+ fbpf_log("stack\t\t\t0x%016llx - 0x%016llx\n", -+ env->prog->shadow_stack_addr - stack_page_cnt * PAGE_SIZE, env->prog->shadow_stack_addr); -+ // fbpf_log("high guard region\t0x%016llx - 0x%016llx\n", -+ // env->prog->high_guard_region_addr, env->prog->high_guard_region_addr + guard_page_cnt / 2 * PAGE_SIZE); ++ pudp = pud_set_fixmap_offset_pre_init(p4dp, addr); ++ do { ++ pud_t old_pud = READ_ONCE(*pudp); + -+ fbpf_log("prog [%s] func_id = %d: 0x%016llx - 0x%016llx, size = 0x%llx\n", -+ env->prog->aux->name, env->prog->aux->func_idx, -+ (u64)env->prog->shadow_region_addr, -+ (u64)env->prog->shadow_stack_addr, -+ (u64)(total_page_cnt * PAGE_SIZE)); ++ next = pud_addr_end(addr, end); + -+ret: -+ if (pages) -+ kfree(pages); -+ return err; -+} ++ iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot, ++ pgtable_alloc, flags); + -+static inline bool is_LDST_insn(struct bpf_insn *insn) -+{ -+ u8 class = BPF_CLASS(insn->code); ++ PTP_CHECK(pud_val(old_pud) != 0 && ++ pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); ++ phys += next - addr; ++ } while (pudp++, addr = next, addr != end); + -+ if (class == BPF_LD || class == BPF_LDX || class == BPF_ST || class == BPF_STX) -+ return true; -+ else -+ return false; ++ pud_clear_fixmap_pre_init(); +} + -+static inline bool is_HELPER_insn(struct bpf_insn *insn) ++static void __create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) +{ -+ u8 class = BPF_CLASS(insn->code); -+ u8 op = BPF_OP(insn->code); -+ -+ if ((class == BPF_JMP || class == BPF_JMP32) && -+ op == BPF_CALL && -+ insn->src_reg != BPF_PSEUDO_CALL) -+ return true; -+ else -+ return false; -+} -+ -+__attribute__((unused)) static char *bpf_sfi_check_type_strs[] = { -+ [BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR] = "BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR", -+ [BPF_SFI_CHECK_HELPER_AS_SAFE_PTR] = "BPF_SFI_CHECK_HELPER_AS_SAFE_PTR", -+ [BPF_SFI_CHECK_HELPER_AS_OFFSET] = "BPF_SFI_CHECK_HELPER_AS_OFFSET", -+ [BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR] = "BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR", -+ [BPF_SFI_CHECK_LDST_AS_SAFE_PTR] = "BPF_SFI_CHECK_LDST_AS_SAFE_PTR", -+ [BPF_SFI_CHECK_LDST_AS_OFFSET] = "BPF_SFI_CHECK_LDST_AS_OFFSET", -+}; -+ -+/** -+ * return a link list, each element mark a check of a reg need to be done. -+ * Ex: -+ * - { reg_0, CHECK_LDST } -> NULL -+ * - { reg_3, CHEKC_HELPER } -> { reg_2, CHECK_MODIFIED } -> { reg_1, CHECK_NONE } -+ * -+ */ -+struct bpf_sfi_check_unit *bpf_sfi_get_check_list(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *regs) -+{ -+ if (is_LDST_insn(insn)) { -+ enum bpf_reg_type ptr_type; -+ int reg_no; -+ struct bpf_sfi_check_unit *check_list, *new_unit; -+ bool reg_as_offset; -+ u8 mode; -+ -+ if (BPF_CLASS(insn->code) == BPF_LD || -+ BPF_CLASS(insn->code) == BPF_LDX) -+ reg_no = insn->src_reg; -+ else -+ reg_no = insn->dst_reg; -+ -+ if (reg_no == BPF_REG_FP) -+ return NULL; -+ -+ ptr_type = base_type(regs[reg_no].type); -+ if (ptr_type != PTR_TO_STACK && -+ ptr_type != PTR_TO_MAP_VALUE && -+ ptr_type != PTR_TO_PACKET) { -+ // fbpf_log("%d: ignore ptr=%d, r%d", i, ptr_type, reg); -+ return NULL; -+ } -+ -+ mode = BPF_MODE(insn->code); -+ switch (mode) { -+ case BPF_MEM: -+ reg_as_offset = true; -+ break; -+ case BPF_ATOMIC: -+ case BPF_IMM: -+ reg_as_offset = false; -+ break; -+ default: -+ pr_err("unprocessed mode %d\n", mode); -+ return ERR_PTR(ENOTSUPP); -+ } -+ -+ reg_as_offset = false; -+ -+ check_list = (struct bpf_sfi_check_unit *)kzalloc(sizeof(struct bpf_sfi_check_unit), GFP_KERNEL); -+ INIT_LIST_HEAD(&check_list->list_head); -+ -+ new_unit = (struct bpf_sfi_check_unit *)kzalloc(sizeof(struct bpf_sfi_check_unit), GFP_KERNEL); -+ INIT_LIST_HEAD(&new_unit->list_head); -+ new_unit->reg_no = reg_no; -+ new_unit->reg_as_offset = reg_as_offset; -+ -+ if (regs[reg_no].sfi_check_state == BPF_SFI_UNSAFE_PTR) { -+ new_unit->check_type = BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR; -+ } -+ else if (regs[reg_no].sfi_check_state == BPF_SFI_SAFE_PTR) { -+ new_unit->check_type = BPF_SFI_CHECK_LDST_AS_SAFE_PTR; -+ } -+ else { -+ new_unit->check_type = BPF_SFI_CHECK_LDST_AS_OFFSET; -+ } -+ -+ if (insn->off == 0) { -+ if (new_unit->check_type == BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR || -+ new_unit->check_type == BPF_SFI_CHECK_LDST_AS_OFFSET) { -+ if (reg_as_offset) -+ new_unit->proposed_state = BPF_SFI_OFFSET; -+ else -+ new_unit->proposed_state = BPF_SFI_SAFE_PTR; -+ } -+ else -+ new_unit->proposed_state = BPF_SFI_SAFE_PTR; -+ } else { -+ new_unit->proposed_state = BPF_SFI_SAFE_PTR; -+ } -+ -+ list_add(&new_unit->list_head, &check_list->list_head); -+ // fbpf_log("%d: add %s check point\n", insn - env->prog->insnsi, -+ // bpf_sfi_check_type_strs[new_unit->check_type]); -+ -+ return check_list; -+ } -+ else if (is_HELPER_insn(insn)) { -+ const struct bpf_func_proto *fn; -+ int arg, reg_no; -+ enum bpf_reg_type arg_reg_type; -+ struct bpf_sfi_check_unit *bpf_sfi_check_list; -+ -+ if (unlikely(!env->ops->get_func_proto)) { -+ pr_err("ERROR: unsupport helper\n"); -+ return ERR_PTR(ENOENT); -+ } -+ fn = env->ops->get_func_proto(insn->imm, env->prog); -+ if (!fn) { -+ pr_err("ERROR: unknown helper type\n"); -+ return ERR_PTR(ENOENT); -+ } -+ -+ bpf_sfi_check_list = (struct bpf_sfi_check_unit *)kzalloc(sizeof(struct bpf_sfi_check_unit), GFP_KERNEL); -+ INIT_LIST_HEAD(&bpf_sfi_check_list->list_head); -+ -+ for (arg = 0; arg < 5; arg++) { -+ reg_no = arg + 1; -+ /* here use mapped actual reg type, not formal arg type */ -+ arg_reg_type = base_type(regs[reg_no].type); -+ -+ // fbpf_log("%d: arg%d_type = %d\n", insn - env->prog->insnsi, reg_no, arg_reg_type); -+ -+ if (arg_reg_type == PTR_TO_STACK || -+ arg_reg_type == PTR_TO_MAP_VALUE || -+ arg_reg_type == PTR_TO_PACKET) { -+ struct bpf_sfi_check_unit *check_unit = (struct bpf_sfi_check_unit *)kzalloc(sizeof(struct bpf_sfi_check_unit), GFP_KERNEL); -+ INIT_LIST_HEAD(&check_unit->list_head); -+ check_unit->reg_no = reg_no; -+ -+ if (regs[reg_no].sfi_check_state == BPF_SFI_UNSAFE_PTR) { -+ check_unit->check_type = BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR; -+ } -+ else if (regs[reg_no].sfi_check_state == BPF_SFI_SAFE_PTR) { -+ check_unit->check_type = BPF_SFI_CHECK_HELPER_AS_SAFE_PTR; -+ } -+ else { -+ check_unit->check_type = BPF_SFI_CHECK_HELPER_AS_OFFSET; -+ } ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); + -+ check_unit->proposed_state = BPF_SFI_SAFE_PTR; -+ -+ list_add(&check_unit->list_head, &bpf_sfi_check_list->list_head); -+ // fbpf_log("%d: add %s check point\n", insn - env->prog->insnsi, -+ // bpf_sfi_check_type_strs[check_unit->check_type]); -+ } -+ } -+ -+ return bpf_sfi_check_list; -+ } -+ else { -+ return NULL; -+ } -+} ++ /* ++ * If the virtual and physical address don't have the same offset ++ * within a page, we cannot map the region as the caller expects. ++ */ ++ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) ++ return; + -+__printf(2, 3) static void fake_verbose(void *private_data, const char *fmt, ...) -+{ -+ va_list args; ++ phys &= PAGE_MASK; ++ addr = virt & PAGE_MASK; ++ end = PAGE_ALIGN(virt + size); + -+ va_start(args, fmt); -+ vprintk(fmt, args); -+ va_end(args); ++ do { ++ next = pgd_addr_end(addr, end); ++ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ phys += next - addr; ++ } while (pgdp++, addr = next, addr != end); +} + -+void dump_insns(struct bpf_prog *prog, int start, int len, const char *prompt) ++void __create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) +{ -+ #ifdef FBPF_LOG_ENABLE -+ int insn_cnt; -+ struct bpf_insn *insn; -+ int i; -+ struct bpf_insn_cbs cbs = { -+ .cb_print = fake_verbose, -+ .private_data = NULL, -+ }; -+ -+ if (unlikely(!prog)) { -+ fbpf_log("ERROR: prog is NULL\n"); -+ return; -+ } -+ -+ if (start + len > prog->len) { -+ fbpf_log("ERROR: print insn from %d to %d is out of bound\n", start, start + len - 1); -+ return; -+ } -+ insn_cnt = len; -+ insn = prog->insnsi + start; -+ -+ if (prompt) -+ printk("[dump insns] --- %s\n", prompt); -+ for (i = 0; i < insn_cnt; i++) { -+ print_bpf_insn(&cbs, insn, false); -+ insn++; -+ } -+ #endif ++ mutex_lock(&fixmap_lock); ++ __create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot, ++ pgtable_alloc, flags); ++ mutex_unlock(&fixmap_lock); +} + -+int bpf_sfi_tmp_check(struct bpf_verifier_env *env) ++static void __init early_iounmap_after_init(void __iomem *addr, unsigned long size) +{ -+ struct bpf_prog *prog; -+ struct bpf_insn *insn; -+ int insn_cnt; -+ struct bpf_insn_aux_data *aux; -+ int i; -+ struct bpf_sfi_check_unit *check_list; ++ unsigned long virt_addr; ++ unsigned long offset; ++ unsigned int nrpages; ++ enum fixed_addresses idx; ++ int i, slot; + -+ if (!env) { -+ fbpf_log("env is NULL\n"); -+ return -EFAULT; ++ slot = -1; ++ for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { ++ if (prev_map[i] == addr) { ++ slot = i; ++ break; ++ } + } -+ prog = env->prog; -+ aux = env->insn_aux_data; + -+ if (!prog) { -+ fbpf_log("prog is NULL\n"); -+ return -EFAULT; -+ } -+ insn_cnt = prog->len; ++ if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n", ++ addr, size)) ++ return; + -+ insn = prog->insnsi; -+ if (!insn) -+ return 0; ++ if (WARN(prev_size[slot] != size, ++ "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", ++ addr, size, slot, prev_size[slot])) ++ return; + -+ for (i = 0; i < insn_cnt; i++) { -+ insn = &prog->insnsi[i]; -+ aux = &env->insn_aux_data[i]; -+ check_list = env->insn_check_lists[i]; ++ WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n", ++ addr, size, slot); + -+ if (!aux) { -+ dump_insns(prog, i, 1, "empty aux"); -+ fbpf_log("aux is empty for insn %d", i); -+ return -EFAULT; -+ } ++ virt_addr = (unsigned long)addr; ++ if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN))) ++ return; + -+ /* check optimization */ -+ if (check_list) { -+ struct bpf_sfi_check_unit *p; -+ char check_type[128]; -+ -+ mutex_lock(&env->insn_check_list_mutex); -+ list_for_each_entry(p, &check_list->list_head, list_head) { -+ switch (p->check_type) { -+ case BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR: -+ if (insn->off == 0) -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR"); -+ else -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_OFFSET_AS_UNSAFE_PTR");; -+ break; -+ case BPF_SFI_CHECK_LDST_AS_SAFE_PTR: -+ if (insn->off == 0) -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_AS_SAFE_PTR"); -+ else -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_OFFSET_AS_SAFE_PTR");; -+ break; -+ case BPF_SFI_CHECK_LDST_AS_OFFSET: -+ if (insn->off == 0) -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_AS_OFFSET"); -+ else -+ strcpy(check_type, "BPF_SFI_CHECK_LDST_OFFSET_AS_OFFSET");; -+ break; -+ case BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR: -+ strcpy(check_type, "BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR"); -+ break; -+ case BPF_SFI_CHECK_HELPER_AS_SAFE_PTR: -+ strcpy(check_type, "BPF_SFI_CHECK_HELPER_AS_SAFE_PTR"); -+ break; -+ case BPF_SFI_CHECK_HELPER_AS_OFFSET: -+ strcpy(check_type, "BPF_SFI_CHECK_HELPER_AS_OFFSET"); -+ break; -+ } -+ fbpf_log("%d: reg %d, check type %s\n", i, p->reg_no, check_type); -+ } -+ mutex_unlock(&env->insn_check_list_mutex); -+ } ++ offset = offset_in_page(virt_addr); ++ nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; ++ ++ idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; ++ while (nrpages > 0) { ++ if (after_paging_init) ++ __late_clear_fixmap(idx); ++ else ++ __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); ++ --idx; ++ --nrpages; + } -+ -+ return 0; ++ prev_map[slot] = NULL; +} + -+/** -+ * return: -+ * > 0: patch cnt -+ * = 0: no need to patch -+ * < 0: errno -+ */ -+static inline int form_check_ldst_as_unsafe_ptr(struct bpf_insn *origin_insn, struct bpf_insn *patch, bool reg_as_offset, -+ u64 sfi_region_size, int reg, int *origin_insn_no) -+{ -+ int patch_len; -+ struct bpf_insn modified_origin_insn = *origin_insn; -+ u8 size = BPF_SIZE(origin_insn->code); -+ u8 class = BPF_CLASS(origin_insn->code); -+ -+ pr_info("reg_as_offset %d\n", reg_as_offset); -+ -+ /* lock *(u64 *)(r0 + 0) += 1 */ -+ /* lock *(u64 *)(r0 + 8) += 1 */ -+ if (!reg_as_offset) { -+ patch[0] = BPF_ALU64_IMM(BPF_AND, reg, sfi_region_size - 1); -+ patch[1] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[2] = *origin_insn; -+ -+ *origin_insn_no = 2; -+ patch_len = 3; -+ } else { -+ /* *(u64 *)(r0 + 0) = 0 */ -+ if (origin_insn->off == 0) { -+ patch[0] = BPF_ALU64_IMM(BPF_AND, reg, sfi_region_size - 1); -+ modified_origin_insn.code = class | BPF_REG | size; -+ if (class == BPF_LD || class == BPF_LDX) -+ modified_origin_insn.src_reg = BPF_REG_BASE; -+ if (class == BPF_ST || class == BPF_STX) -+ modified_origin_insn.dst_reg = BPF_REG_BASE; -+ modified_origin_insn.off = reg; -+ patch[1] = modified_origin_insn; -+ -+ *origin_insn_no = 1; -+ patch_len = 2; -+ } -+ /* *(u64 *)(r0 + 8) = 0 */ -+ else { -+ patch[0] = BPF_ALU64_IMM(BPF_AND, reg, sfi_region_size - 1); -+ patch[1] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[2] = *origin_insn; -+ -+ *origin_insn_no = 2; -+ patch_len = 3; -+ } -+ } ++void __init efi_memmap_unmap_after_init(void) ++{ ++ if (!efi_enabled(EFI_MEMMAP)) ++ return; + -+ return patch_len; -+} ++ if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) { ++ unsigned long size; + -+/** -+ * return: -+ * > 0: patch cnt -+ * = 0: no need to patch -+ * < 0: errno -+ */ -+static inline int form_check_ldst_as_offset(struct bpf_insn *origin_insn, struct bpf_insn *patch, bool reg_as_offset, -+ u64 sfi_region_size, int reg, int *origin_insn_no) -+{ -+ int patch_len; -+ struct bpf_insn modified_origin_insn = *origin_insn; -+ u8 size = BPF_SIZE(origin_insn->code); -+ u8 class = BPF_CLASS(origin_insn->code); -+ -+ if (!reg_as_offset) { -+ /* lock *(u64 *)(r0 + 0) += 1 */ -+ /* lock *(u64 *)(r0 + 8) += 1 */ -+ patch[0] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[1] = *origin_insn; -+ -+ *origin_insn_no = 1; -+ patch_len = 2; -+ ++ size = efi.memmap.desc_size * efi.memmap.nr_map; ++ early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size); + } else { -+ /* *(u64 *)(r0 + 0) = 0 */ -+ if (origin_insn->off == 0) { -+ modified_origin_insn.code = class | BPF_REG | size; -+ if (class == BPF_LD || class == BPF_LDX) -+ modified_origin_insn.src_reg = BPF_REG_BASE; -+ if (class == BPF_ST || class == BPF_STX) -+ modified_origin_insn.dst_reg = BPF_REG_BASE; -+ modified_origin_insn.off = reg; -+ patch[0] = modified_origin_insn; -+ -+ *origin_insn_no = 0; -+ patch_len = 1; -+ } -+ /* *(u64 *)(r0 + 8) = 0 */ -+ else { -+ patch[0] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[1] = *origin_insn; -+ -+ *origin_insn_no = 1; -+ patch_len = 2; -+ } ++ memunmap(efi.memmap.map); + } + -+ return patch_len; ++ efi.memmap.map = NULL; ++ clear_bit(EFI_MEMMAP, &efi.flags); +} +diff --git a/arch/arm64/kernel/haoc/ptp/ptp.c b/arch/arm64/kernel/haoc/ptp/ptp.c +new file mode 100644 +index 000000000000..973542291a92 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/ptp/ptp.c +@@ -0,0 +1,280 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include + -+/** -+ * form a patch according to the insn -+ * v3 means this is optimized twice -+ * @origin_insn: where the instrument point locates at -+ * @check_unit: the sfi check unit -+ * @patch_array: patch will be stored in this array if return code is 0 -+ * @origin_insn_no: the index of origin insn -+ * @sfi_region_size: the size of sfi region -+ * return: -+ * < 0: errno -+ * >= 0: patch_cnt -+ */ -+int form_sfi_check_patch_v3(struct bpf_insn *origin_insn, struct bpf_sfi_check_unit *check_unit, -+ struct bpf_insn *patch_array, int* origin_insn_no, u64 sfi_region_size) -+{ -+ struct bpf_insn patch[16]; -+ int patch_len = 0; -+ enum bpf_sfi_check_type type = check_unit->check_type; -+ int reg = check_unit->reg_no; -+ -+ /** normal case, -+ * 1. and with SFI_REGION_SIZE -+ * 2. if not reg_as_offset -+ * 2.1 or with SFI_REGION_BASE -+ * 2.2 if has off, add off -+ * 2.3 origin insn -+ * 2.4 if has off, sub off -+ * 3. if reg_as_offset -+ * 3.1 if has off, add off -+ * 3.2 replace BPF_LD/BPF_ST to BPF_LDR/BPF_STR -+ * 3.3 if has off, sub off -+ */ -+ if (type == BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR) { -+ bool reg_as_offset = check_unit->reg_as_offset; -+ patch_len = form_check_ldst_as_unsafe_ptr(origin_insn, patch, -+ reg_as_offset, sfi_region_size, reg, origin_insn_no); -+ } -+ /** second case -+ * 1. no need to and again -+ * 2. but need to process off -+ * 3. and replace insn to BPF_LDR/BPF_STR -+ */ -+ else if (type == BPF_SFI_CHECK_LDST_AS_OFFSET) { -+ bool reg_as_offset = check_unit->reg_as_offset; -+ patch_len = form_check_ldst_as_offset(origin_insn, patch, -+ reg_as_offset, sfi_region_size, reg, origin_insn_no); -+ } -+ /** after check helper, ptr is still ptr, but safe -+ * 1. and with SFI_REGION_SIZE -+ * 2. or with SFI_REGION_BASE -+ */ -+ else if (type == BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR) { -+ patch[0] = BPF_ALU64_IMM(BPF_AND, reg, sfi_region_size - 1); -+ patch[1] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[2] = *origin_insn; -+ -+ *origin_insn_no = 2; -+ patch_len = 3; -+ } -+ /** second case -+ * 1. no need to and -+ * 2. just or -+ */ -+ else if (type == BPF_SFI_CHECK_HELPER_AS_OFFSET) { -+ patch[0] = BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE); -+ patch[1] = *origin_insn; -+ -+ *origin_insn_no = 1; -+ patch_len = 2; -+ } -+ /* no need to check again */ -+ else if (type == BPF_SFI_CHECK_HELPER_AS_SAFE_PTR || -+ type == BPF_SFI_CHECK_LDST_AS_SAFE_PTR) { -+ *origin_insn_no = 0; -+ patch_len = 0; -+ } -+ else { -+ pr_err("unresolved case\n"); -+ BUG_ON(1); -+ } ++bool check_addr_in_iee_valid(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; + -+ if (patch_len > sizeof(patch_array)) -+ return -ENOMEM; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ pud_t *pudp; ++ pmd_t *pmdp; ++ pte_t *ptep; + -+ for (int i = 0; i < patch_len; i++) -+ patch_array[i] = patch[i]; -+ -+ return patch_len; -+} ++ // if (haoc_bitmap_ready) ++ // return iee_verify_not_normal(__iee_to_virt(addr), ++ // __iee_to_virt(addr+PAGE_SIZE)); + -+int bpf_sfi_check_ldst(struct bpf_verifier_env *env) -+{ -+ struct bpf_prog *prog; -+ struct bpf_insn *insn; -+ int insn_cnt; -+ struct bpf_insn_aux_data *aux; -+ struct bpf_prog *new_prog; -+ int i, cnt, delta = 0; -+ u8 reg; -+ int origin_insn_no; -+ enum bpf_reg_type ptr_type; -+ struct bpf_sfi_check_unit *check_list, *origin_check_list; ++ if (!(p4d_val(READ_ONCE(*p4dp)) & PTE_VALID)) ++ return false; + -+ if (!env) { -+ fbpf_log("env is NULL\n"); -+ return -EFAULT; -+ } -+ prog = env->prog; -+ aux = env->insn_aux_data; ++ pudp = pud_offset(p4dp, addr); + -+ if (!prog) { -+ fbpf_log("prog is NULL\n"); -+ return -EFAULT; -+ } -+ insn_cnt = prog->len; ++ if (!(pud_val(READ_ONCE(*pudp)) & PTE_VALID)) ++ return false; + -+ insn = prog->insnsi; -+ if (!insn) -+ return 0; ++ pmdp = pmd_offset(pudp, addr); + -+ /* use i + delta to traverse: insn, aux. */ -+ for (i = 0; i < insn_cnt; i++) { -+ struct bpf_insn_aux_data origin_insn_aux; ++ if (!(pmd_val(READ_ONCE(*pmdp)) & PTE_VALID)) ++ return false; + -+ insn = &prog->insnsi[i + delta]; -+ aux = &env->insn_aux_data[i + delta]; -+ check_list = env->insn_check_lists[i + delta]; ++ ptep = pte_offset_kernel(pmdp, addr); + -+ if (!aux) { -+ dump_insns(prog, i, 1, "empty aux"); -+ fbpf_log("aux is empty for insn %d", i); -+ return -EFAULT; -+ } -+ origin_insn_aux = *aux; -+ origin_check_list = env->insn_check_lists[i + delta]; -+ -+ /* check optimization */ -+ if (check_list) { -+ struct bpf_sfi_check_unit *p; -+ // u64 offset_bound_mask = (roundup_pow_of_two( -+ // env->prog->total_page_cnt) << PAGE_SHIFT) - 1; -+ u64 sfi_region_size = env->prog->total_page_cnt << PAGE_SHIFT; -+ -+ list_for_each_entry(p, &check_list->list_head, list_head) { -+ if (p->check_type != BPF_SFI_CHECK_LDST_AS_UNSAFE_PTR && -+ p->check_type != BPF_SFI_CHECK_LDST_AS_SAFE_PTR && -+ p->check_type != BPF_SFI_CHECK_LDST_AS_OFFSET) -+ continue; -+ -+ reg = p->reg_no; -+ ptr_type = base_type(aux->ptr_type); -+ -+ fbpf_log("%d: check ptr=%d, r%d", i, ptr_type, reg); -+ dump_insns(prog, i + delta, 1, "Ld/st insn to rewrite"); -+ -+ struct bpf_insn patch[16]; -+ cnt = form_sfi_check_patch_v3(insn, p, patch, &origin_insn_no, -+ sfi_region_size); -+ -+ /* NOTICE: use cnt to decide whether to patch */ -+ if (cnt == 0) -+ continue; -+ -+ new_prog = _bpf_patch_insn_data(env, i + delta, patch, cnt); -+ if (!new_prog) { -+ fbpf_log("patch failed\n"); -+ return -ENOMEM; -+ } ++ return (pte_val(READ_ONCE(*ptep)) & PTE_VALID); ++} + -+ dump_insns(new_prog, i + delta, cnt, "restore ldst"); ++// Return true if it is only changing prot of a pte. ++static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte) ++{ ++ if(((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0) ++ return true; ++ else ++ return false; ++} + -+ env->insn_aux_data[i + delta + origin_insn_no] = origin_insn_aux; -+ env->insn_check_lists[i + delta + origin_insn_no] = origin_check_list; -+ env->prog = prog = new_prog; -+ delta += cnt - 1; -+ } ++// Return true if the modify does not break DEP. ++static inline bool check_pte_dep(char *addr, pte_t pte) ++{ ++ // DEP for kernel code and readonly data ++ // _text: .text start addr, __init_begin: .rodata end addr ++ if (addr >= _stext && addr < _etext) { ++ if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable ++ || !(PTE_RDONLY & pte_val(pte))) { // DBM == 0 && AP[2] = 0 --> writable ++ panic("Can't make kernel's text/readonly page as writable!\n" ++ "addr = 0x%16llx, pte_val = 0x%16llx", ++ (u64)addr, pte_val(pte)); ++ } ++ } ++ return true; ++} + -+ // if (!old_off) { -+ // struct bpf_insn zero_off_patch[] = { -+ // BPF_ALU64_IMM(BPF_AND, reg, offset_bound_mask), -+ // BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE), -+ // *insn, -+ // }; -+ -+ // cnt = ARRAY_SIZE(zero_off_patch); -+ // origin_insn_no = find_origin_insn(zero_off_patch, cnt, insn); -+ -+ // new_prog = bpf_patch_insn_data(env, i + delta, zero_off_patch, cnt); -+ // if (!new_prog) { -+ // fbpf_log("patch failed\n"); -+ // return -ENOMEM; -+ // } -+ -+ // // dump_insns(new_prog, i + delta, cnt, "restore ldst"); -+ -+ // env->insn_aux_data[i + delta + origin_insn_no] = -+ // origin_insn_aux; -+ // env->prog = prog = new_prog; -+ // delta += cnt - 1; -+ // continue; -+ // } else { -+ // struct bpf_insn non_zero_off_patch[] = { -+ // BPF_ALU64_IMM(BPF_ADD, reg, old_off), -+ // BPF_ALU64_IMM(BPF_AND, reg, offset_bound_mask), -+ // BPF_ALU64_REG(BPF_OR, reg, BPF_REG_BASE), -+ // *insn, -+ // BPF_ALU64_IMM(BPF_SUB, reg, old_off), -+ // }; -+ -+ // cnt = ARRAY_SIZE(non_zero_off_patch); -+ // origin_insn_no = find_origin_insn(non_zero_off_patch, cnt, insn); -+ -+ // new_prog = bpf_patch_insn_data(env, i + delta, non_zero_off_patch, cnt); -+ // if (!new_prog) { -+ // fbpf_log("patch failed\n"); -+ // return -ENOMEM; -+ // } -+ -+ // // dump_insns(new_prog, i + delta, cnt, "restore ldst"); -+ -+ // env->insn_aux_data[i + delta + origin_insn_no] = -+ // origin_insn_aux; -+ // env->prog = prog = new_prog; -+ // delta += cnt - 1; -+ // continue; -+ // } ++// Return true if the modify does not break DEP. ++static inline bool check_pmd_dep(char *addr, pmd_t pmd) ++{ ++ // DEP for kernel code and readonly data ++ // _text: .text start addr, __init_begin: .rodata end addr ++ if (addr >= _stext && addr < _etext) { ++ if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable ++ !(PTE_RDONLY & pmd_val(pmd))) { // DBM == 0 && AP[2] = 0 --> writable ++ panic("Can't make kernel's text/readonly page as writable!\n" ++ "addr = 0x%16llx, pmd_val = 0x%16llx", ++ (u64)addr, pmd_val(pmd)); + } + } ++ return true; ++} + -+ // dump_insns(env->prog, 0, env->prog->len, "all after restore ldst"); ++void __iee_code _iee_set_static_pgd(int flag, pgd_t *pgdp, pgd_t pgd) ++{ ++ if ((pgd_val(pgd) & PMD_TABLE_BIT) && ++ !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) ++ panic("You can't use non-iee-pgtable\n"); + -+ return 0; ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); +} + -+int bpf_sfi_check_helper_args(struct bpf_verifier_env *env) ++void __iee_code _iee_set_bm_pte(int flag, pte_t *ptep, pte_t pte) +{ -+ struct bpf_prog *prog; -+ int insn_cnt; -+ struct bpf_prog *new_prog; -+ int i, cnt, delta = 0; -+ struct bpf_sfi_check_unit *check_list, *origin_check_list; ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa_symbol(ptep)))), pte); ++} + -+ if (!env) { -+ fbpf_log("ERROR: env is NULL\n"); -+ return -EFAULT; -+ } -+ prog = env->prog; ++void __iee_code _iee_set_pte(int flag, pte_t *ptep, pte_t pte) ++{ ++ char *addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); + -+ if (!prog) { -+ fbpf_log("ERROR: prog is NULL\n"); -+ return -EFAULT; ++ if (!(pte_val(pte) & PTE_VALID)) { ++ WRITE_ONCE(*((pte_t *)((unsigned long)ptep | IEE_OFFSET)), pte); ++ return; + } -+ insn_cnt = prog->len; -+ -+ // dump_insns(prog, 0, prog->len, "before restore_helper_args"); -+ -+ /** -+ * use i + delta to traverse: insn, aux. -+ * in inner loop, do not update the insn and aux. -+ */ -+ for (i = 0; i < insn_cnt; i++) { -+ struct bpf_insn_aux_data origin_insn_aux; -+ int sub_delta = 0; -+ int alpha = 0; -+ struct bpf_insn_aux_data *aux; -+ struct bpf_insn *insn; -+ -+ insn = &prog->insnsi[i + delta + alpha]; -+ aux = &env->insn_aux_data[i + delta + alpha]; -+ check_list = env->insn_check_lists[i + delta + alpha]; -+ if (!aux) { -+ dump_insns(prog, i, 1, "empty aux"); -+ fbpf_log("ERROR: aux is empty for insn %d\n", i); -+ return -EFAULT; -+ } -+ origin_insn_aux = *aux; -+ origin_check_list = env->insn_check_lists[i + delta + alpha]; -+ -+ /* check optimization */ -+ if (check_list) { -+ int func_id = insn->imm; -+ struct bpf_sfi_check_unit *p; -+ // u64 offset_bound_mask = (roundup_pow_of_two( -+ // env->prog->total_page_cnt) << PAGE_SHIFT) - 1; -+ u64 sfi_region_size = env->prog->total_page_cnt << PAGE_SHIFT; -+ -+ list_for_each_entry(p, &check_list->list_head, list_head) { -+ int origin_insn_no; -+ int reg_no = p->reg_no; -+ enum bpf_reg_type arg_type = aux->arg_reg_type[reg_no - 1]; -+ insn = &prog->insnsi[i + delta + alpha]; -+ aux = &env->insn_aux_data[i + delta + alpha]; -+ struct bpf_insn patch[16]; -+ -+ // struct bpf_insn patch[] = { -+ // BPF_ALU64_IMM(BPF_AND, reg_no, offset_bound_mask), -+ // BPF_ALU64_REG(BPF_OR, reg_no, BPF_REG_BASE), -+ // *insn, -+ // }; -+ -+ if (p->check_type != BPF_SFI_CHECK_HELPER_AS_UNSAFE_PTR && -+ p->check_type != BPF_SFI_CHECK_HELPER_AS_SAFE_PTR && -+ p->check_type != BPF_SFI_CHECK_HELPER_AS_OFFSET) -+ break; -+ -+ cnt = form_sfi_check_patch_v3(insn, p, patch, &origin_insn_no, -+ sfi_region_size); -+ -+ /* NOTICE: use cnt to decide whether to patch */ -+ if (cnt == 0) -+ continue; -+ -+ // cnt = ARRAY_SIZE(patch); -+ // origin_insn_no = find_origin_insn(patch, cnt, insn); -+ -+ fbpf_log("patch %d(%d): helper func=%d, \ -+restore arg r%d, type=%d, cnt=%d, alpha=%d\n", -+ i + delta + alpha, i, func_id, reg_no, -+ arg_type, cnt, alpha); -+ -+ new_prog = _bpf_patch_insn_data(env, i + delta + alpha, -+ patch, cnt); -+ if (!new_prog) { -+ fbpf_log("ERROR: patch failed\n"); -+ return -ENOMEM; -+ } + -+ dump_insns(new_prog, i + delta + alpha, -+ cnt, "restore stack ptr arg"); -+ -+ env->insn_aux_data[i + delta + alpha + origin_insn_no] = origin_insn_aux; -+ env->insn_check_lists[i + delta + alpha + origin_insn_no] = origin_check_list; -+ env->prog = prog = new_prog; ++ // Avoid mapping a new VA to IEE PA. ++ if(!is_changing_pte_prot(ptep, pte) && ++ check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) ++ panic("You are remmaping IEE page to other VA.\n"); + -+ sub_delta += cnt - 1; -+ alpha += origin_insn_no; -+ } -+ -+ delta += sub_delta; -+ } -+ } ++ // Avoid mapping a writable VA to kernel code PA. ++ if (!check_pte_dep(addr, pte)) ++ return; + -+ return 0; ++ WRITE_ONCE(*((pte_t *)((unsigned long)ptep | IEE_OFFSET)), pte); +} + -+bool bpf_sfi_can_instrument(struct bpf_verifier_env *env) ++void __iee_code _iee_set_pmd(int flag, pmd_t *pmdp, pmd_t pmd) +{ -+ for (int i = 0; i < env->used_map_cnt; i++) { -+ struct bpf_map *map = env->used_maps[i]; -+ if (map_can_lookup(map) && !map->is_aggregated) { -+ return false; -+ } -+ } -+ -+ return true; -+} -\ No newline at end of file -diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c -index 8fa7b37152c0..f305e2fdce82 100644 ---- a/kernel/bpf/verifier.c -+++ b/kernel/bpf/verifier.c -@@ -32,6 +32,10 @@ - #include - #endif - -+#ifdef CONFIG_HIVE -+#include -+#endif ++ char *addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); + - #include "disasm.h" - - static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { -@@ -10032,6 +10036,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn - int insn_idx = *insn_idx_p; - bool changes_data; - int i, err, func_id; -+#ifdef CONFIG_HIVE -+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; -+#endif - - /* find function prototype */ - func_id = insn->imm; -@@ -10122,6 +10129,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn - - regs = cur_regs(env); - -+#ifdef CONFIG_HIVE -+ /* fbpf record real arg type of helper call */ -+ for (i = 0; i < 5; i++) { -+ aux->arg_reg_type[i] = regs[i + 1].type; ++ if (!(pmd_val(pmd) & PMD_SECT_VALID)) { ++ WRITE_ONCE(*((pmd_t *)((unsigned long)pmdp | IEE_OFFSET)), pmd); ++ return; + } -+#endif -+ - if (meta.release_regno) { - err = -EINVAL; - /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot -@@ -16468,7 +16482,7 @@ static int propagate_liveness(struct bpf_verifier_env *env, - return -EFAULT; - } - /* Propagate read liveness of registers... */ -- BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); -+ BUILD_BUG_ON(BPF_REG_FP + 2 != MAX_BPF_REG); - for (frame = 0; frame <= vstate->curframe; frame++) { - parent = vparent->frame[frame]; - state = vstate->frame[frame]; -@@ -17076,15 +17090,30 @@ static int do_check(struct bpf_verifier_env *env) - bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); - struct bpf_verifier_state *state = env->cur_state; - struct bpf_insn *insns = env->prog->insnsi; -+#ifdef CONFIG_HIVE -+ struct bpf_reg_state *regs = NULL, *regs_before = NULL; -+ struct bpf_sfi_check_unit *check_list; -+ int i; -+#else - struct bpf_reg_state *regs; -+#endif - int insn_cnt = env->prog->len; - bool do_print_state = false; - int prev_insn_idx = -1; - -+#ifdef CONFIG_HIVE -+ regs_before = vmalloc(sizeof(struct bpf_reg_state) * MAX_BPF_REG); -+ if (IS_ERR(regs_before)) -+ return PTR_ERR(regs_before); -+#endif -+ - for (;;) { - struct bpf_insn *insn; - u8 class; - int err; -+ #ifdef CONFIG_HIVE -+ bool updated[MAX_BPF_REG] = {false}; -+ #endif - - /* reset current history entry on each new instruction */ - env->cur_hist_ent = NULL; -@@ -17177,6 +17206,16 @@ static int do_check(struct bpf_verifier_env *env) - sanitize_mark_insn_seen(env); - prev_insn_idx = env->insn_idx; - -+ #ifdef CONFIG_HIVE -+ /* fbpf: compare regs_before and regs to get which regs are changed */ -+ memcpy(regs_before, regs, sizeof(struct bpf_reg_state) * MAX_BPF_REG); -+ -+ /* fbpf: used to form check list of vulnerable insns */ -+ check_list = bpf_sfi_get_check_list(env, insn, regs); -+ if (IS_ERR(check_list)) -+ return PTR_ERR(check_list); -+ #endif -+ - if (class == BPF_ALU || class == BPF_ALU64) { - err = check_alu_op(env, insn); - if (err) -@@ -17218,8 +17257,17 @@ static int do_check(struct bpf_verifier_env *env) - err = check_atomic(env, env->insn_idx, insn); - if (err) - return err; -+ #ifdef CONFIG_HIVE -+ err = save_aux_ptr_type(env, regs[insn->dst_reg].type, true); -+ if (err) -+ return err; -+ #endif - env->insn_idx++; -+ #ifdef CONFIG_HIVE -+ goto next_insn; -+ #else - continue; -+ #endif - } - - if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { -@@ -17323,7 +17371,12 @@ static int do_check(struct bpf_verifier_env *env) - env->insn_idx += insn->off + 1; - else - env->insn_idx += insn->imm + 1; -+ -+ #ifdef CONFIG_HIVE -+ goto next_insn; -+ #else - continue; -+ #endif - - } else if (opcode == BPF_EXIT) { - if (BPF_SRC(insn->code) != BPF_K || -@@ -17363,7 +17416,11 @@ static int do_check(struct bpf_verifier_env *env) - if (err) - return err; - do_print_state = true; -+ #ifdef CONFIG_HIVE -+ goto next_insn; -+ #else - continue; -+ #endif - } - - err = check_return_code(env); -@@ -17380,7 +17437,11 @@ static int do_check(struct bpf_verifier_env *env) - break; - } else { - do_print_state = true; -+ #ifdef CONFIG_HIVE -+ goto next_insn; -+ #else - continue; -+ #endif - } - } else { - err = check_cond_jmp_op(env, insn, &env->insn_idx); -@@ -17406,12 +17467,52 @@ static int do_check(struct bpf_verifier_env *env) - verbose(env, "invalid BPF_LD mode\n"); - return -EINVAL; - } -+ #ifdef CONFIG_HIVE -+ err = save_aux_ptr_type(env, regs[insn->src_reg].type, true); -+ if (err) -+ return err; -+ #endif - } else { - verbose(env, "unknown insn class %d\n", class); - return -EINVAL; - } - - env->insn_idx++; + -+#ifdef CONFIG_HIVE -+next_insn: -+ if (check_list) { -+ struct bpf_sfi_check_unit *p; -+ list_for_each_entry(p, &check_list->list_head, list_head) { -+ int reg_no = p->reg_no; -+ regs[reg_no].sfi_check_state = p->proposed_state; -+ updated[reg_no] = true; -+ } ++ // Check if the pte table is legally allocated. ++ if ((pmd_val(pmd) & PMD_TABLE_BIT) && ++ !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) ++ panic("You can't use non-iee-pgtable\n"); + -+ if (!env->insn_check_lists[prev_insn_idx]) { -+ // fbpf_log("update into %d\n", prev_insn_idx); -+ env->insn_check_lists[prev_insn_idx] = check_list; -+ } -+ } -+ -+ for (i = 0; i < MAX_BPF_REG; i++) { -+ // if (base_type(regs_before[i].type) == PTR_TO_STACK) -+ // continue; -+ /* reg move from src to dst, use src_reg's check_state */ -+ if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X)) -+ continue; -+ /* use check_list proposed state if have */ -+ if (updated[i]) -+ continue; -+ /* if reg changed, reset sfi_check state */ -+ if (memcmp(regs_before + i, regs + i, sizeof(struct bpf_reg_state))) { -+ regs[i].sfi_check_state = BPF_SFI_UNSAFE_PTR; -+ } -+ } ++ // Avoid mapping a huge pmd to IEE physical page. ++ // if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) ++ // panic("Mapping IEE physical page to a huge pmd.\n"); + -+ // fbpf_log("debugging insn %d", prev_insn_idx); -+ // bpf_sfi_tmp_check(env); -+#endif - } - - return 0; -@@ -17866,6 +17967,168 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) - return 0; - } - ++ if (!check_pmd_dep(addr, pmd)) ++ return; + ++ WRITE_ONCE(*((pmd_t *)((unsigned long)pmdp | IEE_OFFSET)), pmd); ++} + -+#ifdef CONFIG_HIVE -+static int resolve_pseudo_ldimm64_again(struct bpf_verifier_env *env) ++void __iee_code _iee_set_pud(int flag, pud_t *pudp, pud_t pud) +{ -+ struct bpf_insn *insn = env->prog->insnsi; -+ int insn_cnt = env->prog->len; -+ int i, j, err; -+ -+ err = bpf_prog_calc_tag(env->prog); -+ if (err) -+ return err; -+ -+ for (i = 0; i < insn_cnt; i++, insn++) { -+ if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { -+ struct bpf_insn_aux_data *aux; -+ struct bpf_map *map; -+ struct fd f; -+ u64 addr; -+ u32 fd; -+ -+ if (insn[0].src_reg == 0) -+ /* valid generic load 64-bit imm */ -+ goto next_insn; ++ if (!(pud_val(pud) & PMD_SECT_VALID)) { ++ WRITE_ONCE(*((pud_t *)((unsigned long)pudp | IEE_OFFSET)), pud); ++ return; ++ } + -+ if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { -+ aux = &env->insn_aux_data[i]; -+ err = check_pseudo_btf_id(env, insn, aux); -+ if (err) -+ return err; -+ goto next_insn; -+ } ++ if ((pud_val(pud) & PMD_TABLE_BIT) && ++ !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) ++ panic("You can't use non-iee-pgtable\n"); + -+ if (insn[0].src_reg == BPF_PSEUDO_FUNC) { -+ aux = &env->insn_aux_data[i]; -+ aux->ptr_type = PTR_TO_FUNC; -+ goto next_insn; -+ } ++ WRITE_ONCE(*((pud_t *)((unsigned long)pudp | IEE_OFFSET)), pud); ++} + -+ /* In final convert_pseudo_ld_imm64() step, this is -+ * converted into regular 64-bit imm load insn. -+ */ -+ switch (insn[0].src_reg) { -+ case BPF_PSEUDO_MAP_VALUE: -+ break; -+ default: -+ goto next_insn; -+ } ++void __iee_code _iee_set_p4d(int flag, p4d_t *p4dp, p4d_t p4d) ++{ ++ if (!(p4d_val(p4d) & PMD_SECT_VALID)) { ++ WRITE_ONCE(*((p4d_t *)((unsigned long)p4dp | IEE_OFFSET)), p4d); ++ return; ++ } + -+ fd = insn[0].imm; ++ if ((p4d_val(p4d) & PMD_TABLE_BIT) && ++ !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) ++ panic("You can't use non-iee-pgtable\n"); + -+ f = fdget(fd); -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) { -+ verbose(env, "fd %d is not pointing to valid bpf_map\n", -+ insn[0].imm); -+ return PTR_ERR(map); -+ } ++ WRITE_ONCE(*((p4d_t *)((unsigned long)p4dp | IEE_OFFSET)), p4d); ++} + -+ err = check_map_prog_compatibility(env, map, env->prog); -+ if (err) { -+ fdput(f); -+ return err; -+ } ++void __iee_code _iee_set_swapper_pgd(int flag, pgd_t *pgdp, pgd_t pgd) ++{ ++ if (!(pgd_val(pgd) & PMD_SECT_VALID)) { ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); ++ return; ++ } + -+ aux = &env->insn_aux_data[i]; -+ if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || -+ insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { -+ addr = (unsigned long)map; -+ goto next_insn; -+ } else { -+ u32 off = insn[1].imm; -+ -+ /* do this after aggregation */ -+ if (!env->prog->shadow_region_addr) { -+ pr_err("bpf sfi aggregate memory failed\n"); -+ return -EINVAL; -+ } ++ if ((pgd_val(pgd) & PMD_TABLE_BIT) && ++ !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) ++ panic("You can't use non-iee-pgtable\n"); + -+ if (off >= BPF_MAX_VAR_OFF) { -+ verbose(env, "direct value offset of %u is not allowed\n", off); -+ fdput(f); -+ return -EINVAL; -+ } ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); ++} + -+ if (!map->ops->map_direct_value_addr) { -+ verbose(env, "no direct value access support for this map type\n"); -+ fdput(f); -+ return -EINVAL; -+ } ++pteval_t __iee_code _iee_set_xchg_relaxed(int flag, pte_t *ptep, pteval_t pteval) ++{ ++ pteval_t ret = xchg_relaxed((pteval_t *)((unsigned long)ptep | IEE_OFFSET), pteval); ++ return ret; ++} + -+ err = map->ops->map_direct_value_addr(map, &addr, off); -+ if (err) { -+ verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", -+ map->value_size, off); -+ fdput(f); -+ return err; -+ } ++pmdval_t __iee_code _iee_set_pmd_xchg_relaxed(int flag, pmd_t *pmdp, ++ pmdval_t pmdval) ++{ ++ pmdval_t ret = xchg_relaxed((pmdval_t *)((unsigned long)pmdp | IEE_OFFSET), ++ pmdval); ++ return ret; ++} + -+ aux->map_off = off; -+ addr += off; -+ aux->extra_map_ptr = (u64)map; -+ } ++pteval_t __iee_code _iee_set_cmpxchg_relaxed(int flag, pte_t *ptep, ++ pteval_t old_pteval, pteval_t new_pteval) ++{ ++ pteval_t pteval = cmpxchg_relaxed((pteval_t *)((unsigned long)ptep | IEE_OFFSET), ++ old_pteval, new_pteval); + -+ insn[0].imm = (u32)addr; -+ insn[1].imm = addr >> 32; ++ return pteval; ++} + -+ /* check whether we recorded this map already */ -+ for (j = 0; j < env->used_map_cnt; j++) { -+ if (env->used_maps[j] == map) { -+ aux->map_index = j; -+ fdput(f); -+ goto next_insn; -+ } -+ } ++pmdval_t __iee_code _iee_set_pmd_cmpxchg_relaxed(int flag, pmd_t *pmdp, ++ pmdval_t old_pmdval, pmdval_t new_pmdval) ++{ ++ pmdval_t pmdval = cmpxchg_relaxed((pmdval_t *)((unsigned long)pmdp | IEE_OFFSET), ++ old_pmdval, new_pmdval); + -+ if (env->used_map_cnt >= MAX_USED_MAPS) { -+ fdput(f); -+ return -E2BIG; -+ } ++ return pmdval; ++} + -+ if (env->prog->aux->sleepable) -+ atomic64_inc(&map->sleepable_refcnt); -+ /* hold the map. If the program is rejected by verifier, -+ * the map will be released by release_maps() or it -+ * will be used by the valid program until it's unloaded -+ * and all maps are released in bpf_free_used_maps() -+ */ -+ bpf_map_inc(map); ++void __iee_code _iee_set_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, ++ int order, int use_block_pmd) ++{ ++ int i; + -+ aux->map_index = env->used_map_cnt; -+ env->used_maps[env->used_map_cnt++] = map; ++ lm_ptep = (pte_t *)((unsigned long)lm_ptep | IEE_OFFSET); ++ iee_ptep = (pte_t *)((unsigned long)iee_ptep | IEE_OFFSET); ++ if (use_block_pmd) { ++ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + -+ if (bpf_map_is_cgroup_storage(map) && -+ bpf_cgroup_storage_assign(env->prog->aux, map)) { -+ verbose(env, "only one cgroup storage of each type is allowed\n"); -+ fdput(f); -+ return -EBUSY; -+ } ++ pmd = __pmd((pmd_val(pmd) | PMD_SECT_RDONLY) & ~PTE_DBM); ++ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*iee_ptep); + -+ fdput(f); -+next_insn: -+ insn++; -+ i++; -+ continue; ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ iee_ptep++; + } ++ } else { ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*lm_ptep); + -+ /* Basic sanity check before we invest more work here. */ -+ if (!bpf_opcode_in_insntable(insn->code)) { -+ verbose(env, "unknown opcode %02x\n", insn->code); -+ return -EINVAL; ++ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); ++ WRITE_ONCE(*lm_ptep, pte); ++ pte = READ_ONCE(*iee_ptep); ++ pte = __pte(pte_val(pte) | PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ lm_ptep++; ++ iee_ptep++; + } + } -+ -+ /* now all pseudo BPF_LD_IMM64 instructions load valid -+ * 'struct bpf_map *' into a register instead of user map_fd. -+ * These pointers will be used later by verifier to validate map access. -+ */ -+ return 0; +} -+#endif + - /* drop refcnt of maps used by the rejected program */ - static void release_maps(struct bpf_verifier_env *env) - { -@@ -17932,6 +18195,26 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - vfree(old_data); - } - -+#ifdef CONFIG_HIVE -+static void adjust_insn_check_lists(struct bpf_verifier_env *env, -+ struct bpf_sfi_check_unit **new_check_lists, -+ struct bpf_prog *new_prog, u32 off, u32 cnt) ++void __iee_code _iee_unset_sensitive_pte(int flag, pte_t *lm_ptep, pte_t *iee_ptep, ++ int order, int use_block_pmd) +{ -+ struct bpf_sfi_check_unit **old_check_lists = env->insn_check_lists; -+ u32 prog_len; ++ int i; + -+ if (cnt == 1) -+ return; -+ prog_len = new_prog->len; ++ lm_ptep = (pte_t *)((unsigned long)lm_ptep | IEE_OFFSET); ++ iee_ptep = (pte_t *)((unsigned long)iee_ptep | IEE_OFFSET); ++ if (use_block_pmd) { ++ pmd_t pmd = __pmd(pte_val(READ_ONCE(*lm_ptep))); + -+ memcpy(new_check_lists, old_check_lists, sizeof(struct bpf_insn_check_unit *) * off); -+ memcpy(new_check_lists + off + cnt - 1, old_check_lists + off, -+ sizeof(struct bpf_insn_check_unit *) * (prog_len - off - cnt + 1)); -+ env->insn_check_lists = new_check_lists; -+ vfree(old_check_lists); -+} -+#endif ++ pmd = __pmd(pmd_val(pmd) | PTE_DBM); ++ WRITE_ONCE(*lm_ptep, __pte(pmd_val(pmd))); ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*iee_ptep); + - static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) - { - int i; -@@ -17965,12 +18248,21 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of - { - struct bpf_prog *new_prog; - struct bpf_insn_aux_data *new_data = NULL; -+#ifdef CONFIG_HIVE -+ struct bpf_sfi_check_unit **new_check_lists = NULL; -+#endif - - if (len > 1) { - new_data = vzalloc(array_size(env->prog->len + len - 1, - sizeof(struct bpf_insn_aux_data))); - if (!new_data) - return NULL; -+ #ifdef CONFIG_HIVE -+ new_check_lists = vzalloc(array_size(env->prog->len + len - 1, -+ sizeof(struct bpf_sfi_check_unit *))); -+ if (!new_check_lists) -+ return NULL; -+ #endif - } - - new_prog = bpf_patch_insn_single(env->prog, off, patch, len); -@@ -17983,11 +18275,22 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of - return NULL; - } - adjust_insn_aux_data(env, new_data, new_prog, off, len); -+#ifdef CONFIG_HIVE -+ adjust_insn_check_lists(env, new_check_lists, new_prog, off, len); -+#endif - adjust_subprog_starts(env, off, len); - adjust_poke_descs(new_prog, off, len); - return new_prog; - } - -+#ifdef CONFIG_HIVE -+struct bpf_prog *_bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, -+ const struct bpf_insn *patch, u32 len) -+{ -+ return bpf_patch_insn_data(env, off, patch, len); -+} -+#endif ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ iee_ptep++; ++ } ++ } else { ++ for (i = 0; i < (1 << order); i++) { ++ pte_t pte = READ_ONCE(*lm_ptep); + - static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, - u32 off, u32 cnt) - { -@@ -18643,6 +18946,10 @@ static int jit_subprogs(struct bpf_verifier_env *env) - func[i]->aux->name[0] = 'F'; - func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; - func[i]->jit_requested = 1; -+ #ifdef CONFIG_HIVE -+ func[i]->shadow_stack_addr = env->prog->shadow_stack_addr; -+ func[i]->shadow_region_addr = env->prog->shadow_region_addr; -+ #endif - func[i]->blinding_requested = prog->blinding_requested; - func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; - func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; -@@ -20473,6 +20780,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 - len = (*prog)->len; - env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); -+#ifdef CONFIG_HIVE -+ env->insn_check_lists = vzalloc(array_size(sizeof(struct bpf_sfi_check_unit *), len)); -+ for (i = 0; i < len; i++) -+ env->insn_check_lists[i] = NULL; -+ mutex_init(&env->insn_check_list_mutex); -+#endif - ret = -ENOMEM; - if (!env->insn_aux_data) - goto err_free_env; -@@ -20564,6 +20877,48 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 - - if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) - ret = bpf_prog_offload_finalize(env); -+ -+#ifdef CONFIG_HIVE -+ /* bpf_sfi aggregate and check is after bpf_check, if skip full_check, -+ bpf_sfi checks are skipped either */ -+ fbpf_log("bpf_check finished\n"); -+ // dump_insns(env->prog, 0, env->prog->len, "after do_check"); -+ -+ // if (ret == 0) -+ // ret = bpf_sfi_tmp_check(env); -+ -+ /* if not support map or mode, just return to normal bpf */ -+ if (!bpf_sfi_can_instrument(env)) { -+ struct bpf_map *map; -+ for (i = 0; i < env->used_map_cnt; i++) { -+ map = env->used_maps[i]; -+ map->is_aggregated = false; ++ pte = __pte(pte_val(pte) | PTE_DBM); ++ WRITE_ONCE(*lm_ptep, pte); ++ pte = READ_ONCE(*iee_ptep); ++ pte = __pte(pte_val(pte) & ~PTE_VALID); ++ WRITE_ONCE(*iee_ptep, pte); ++ lm_ptep++; ++ iee_ptep++; + } -+ fbpf_log("skip sfi instrument\n"); -+ goto skip_sfi_instrument; + } ++} +diff --git a/arch/arm64/kernel/haoc/selinuxp/Makefile b/arch/arm64/kernel/haoc/selinuxp/Makefile +new file mode 100644 +index 000000000000..0a81a55dac9c +--- /dev/null ++++ b/arch/arm64/kernel/haoc/selinuxp/Makefile +@@ -0,0 +1,7 @@ ++ccflags-y += -I$(srctree)/security/selinux -I$(srctree)/security/selinux/include ++obj-y += selinuxp.o ++ ++ifdef CONFIG_IEE_SELINUX_P ++FLASK_HDR := $(srctree)/security/selinux/flask.h ++$(obj)/iee-selinuxp.o: $(FLASK_HDR) ++endif +\ No newline at end of file +diff --git a/arch/arm64/kernel/haoc/selinuxp/selinuxp.c b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c +new file mode 100644 +index 000000000000..af447a75f271 +--- /dev/null ++++ b/arch/arm64/kernel/haoc/selinuxp/selinuxp.c +@@ -0,0 +1,52 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include "security.h" ++#include "ss/services.h" ++#include + -+ // TODO unmap shadow data -+ /* fbpf aggregate values and caculate shadow stack addr */ -+ if (ret == 0) -+ ret = bpf_sfi_aggregate_memory(env); ++static inline void iee_verify_selinux_type(unsigned long va) ++{ ++ iee_verify_type(va, IEE_SELINUX, "selinux"); ++} + -+ if (ret == 0) -+ ret = resolve_pseudo_ldimm64_again(env); ++void __iee_code _iee_set_selinux_status_pg(unsigned long iee_offset, ++ struct page *new_page) ++{ ++ *__ptr_to_iee(&selinux_state.status_page) = new_page; ++} + -+ if (ret == 0) -+ ret = bpf_sfi_check_ldst(env); ++void __iee_code _iee_set_selinux_enforcing(unsigned long iee_offset, bool value) ++{ ++ *__ptr_to_iee(&selinux_state.enforcing) = value; ++} + -+ if (ret == 0) -+ ret = bpf_sfi_check_helper_args(env); ++void __iee_code _iee_mark_selinux_initialized(unsigned long iee_offset) ++{ ++ /* do a synchronized write to avoid race conditions */ ++ smp_store_release(__ptr_to_iee(&(selinux_state.initialized)), true); ++ pr_info("HAOC: Mark selinux initialized."); ++} + -+skip_sfi_instrument: -+ // dump_insns(env->prog, 0, env->prog->len, "after bpf_sfi_checks"); ++void __iee_code _iee_set_sel_policy_cap(unsigned long iee_offset, ++ unsigned int idx, int cap) ++{ ++ *__ptr_to_iee(&(selinux_state.policycap[idx])) = cap; ++} + -+ fbpf_log("bpf_sfi_check finished\n"); ++/* ++ * Please make sure param iee_new_policy is from policy_jar memcache. ++ * Need to free new_policy after calling this func as it's only used to ++ * trans data from kernel. ++ */ ++void __iee_code _iee_sel_rcu_assign_policy(unsigned long iee_offset, ++ struct selinux_policy *new_policy, ++ struct selinux_policy *iee_new_policy) ++{ ++ /* TODO: Verify information from incoming policy. */ ++ iee_verify_selinux_type((unsigned long)iee_new_policy); + -+ pr_info("[%s] bpf insn len = %d\n", env->prog->aux->name, env->prog->len); ++ /* Copy data from kernel to new allocated policy struct inside iee. */ ++ memcpy(__ptr_to_iee(iee_new_policy), new_policy, sizeof(struct selinux_policy)); ++ rcu_assign_pointer(*__ptr_to_iee(&selinux_state.policy), iee_new_policy); ++} +diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c +index c4999d2d7ce8..2df4bf89fed8 100644 +--- a/arch/arm64/kernel/mte.c ++++ b/arch/arm64/kernel/mte.c +@@ -23,6 +23,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include +#endif - skip_full_check: - kvfree(env->explored_states); -@@ -20685,5 +21040,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 - vfree(env->insn_aux_data); - err_free_env: - kvfree(env); -+#ifdef CONFIG_HIVE -+ pr_info("prog [%s] bpf_check's ret = %d\n", env->prog->aux->name, ret); + static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred); + +@@ -79,9 +82,14 @@ int memcmp_pages(struct page *page1, struct page *page2) + static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) + { + /* Enable MTE Sync Mode for EL1. */ ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_TCF_MASK, ++ SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, + SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); + isb(); +#endif - return ret; + + pr_info_once("MTE: enabled in %s mode at EL1\n", mode); } -diff --git a/kernel/cred.c b/kernel/cred.c -index c033a201c808..813db29f35df 100644 ---- a/kernel/cred.c -+++ b/kernel/cred.c -@@ -20,6 +20,11 @@ - #include - #include +diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c +index e9e5ce956f15..ef5af98ad1cc 100644 +--- a/arch/arm64/kernel/process.c ++++ b/arch/arm64/kernel/process.c +@@ -515,7 +515,11 @@ void update_sctlr_el1(u64 sctlr) + * EnIA must not be cleared while in the kernel as this is necessary for + * in-kernel PAC. It will be cleared on kernel exit if needed. + */ ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); ++#endif -+#ifdef CONFIG_CREDP -+#include -+#include + /* ISB required for the kernel uaccess routines when setting TCF0. */ + isb(); +diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c +index 22e5e23438af..6978ea28dea3 100644 +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -31,6 +31,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include +#endif -+ - #if 0 - #define kdebug(FMT, ...) \ - printk("[%-5.5s%5u] " FMT "\n", \ -@@ -33,7 +38,12 @@ do { \ - } while (0) - #endif -+#ifdef CONFIG_CREDP -+struct kmem_cache *cred_jar; -+static struct kmem_cache *rcu_jar; + /* + * We try to ensure that the mitigation state can never change as the result of +@@ -551,7 +554,11 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) + return state; + + if (spectre_v4_mitigations_off()) { ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, 0, SCTLR_ELx_DSSBS); +#else - static struct kmem_cache *cred_jar; + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); +#endif + set_pstate_ssbs(1); + return SPECTRE_VULNERABLE; + } +@@ -1018,8 +1025,12 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) + if (arm64_kernel_unmapped_at_el0()) + return; - /* init to 2 - one for init_task, one to ensure it is never freed */ - static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) }; -@@ -41,6 +51,32 @@ static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) }; - /* - * The initial credentials for the initial task - */ -+#ifdef CONFIG_CREDP -+struct cred init_cred __section(".iee.cred") = { -+ .usage = ATOMIC_INIT(4), -+#ifdef CONFIG_DEBUG_CREDENTIALS -+ .subscribers = ATOMIC_INIT(2), -+ .magic = CRED_MAGIC, -+#endif -+ .uid = GLOBAL_ROOT_UID, -+ .gid = GLOBAL_ROOT_GID, -+ .suid = GLOBAL_ROOT_UID, -+ .sgid = GLOBAL_ROOT_GID, -+ .euid = GLOBAL_ROOT_UID, -+ .egid = GLOBAL_ROOT_GID, -+ .fsuid = GLOBAL_ROOT_UID, -+ .fsgid = GLOBAL_ROOT_GID, -+ .securebits = SECUREBITS_DEFAULT, -+ .cap_inheritable = CAP_EMPTY_SET, -+ .cap_permitted = CAP_FULL_SET, -+ .cap_effective = CAP_FULL_SET, -+ .cap_bset = CAP_FULL_SET, -+ .user = INIT_USER, -+ .user_ns = &init_user_ns, -+ .group_info = &init_groups, -+ .ucounts = &init_ucounts, -+}; ++#ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_SET_VBAR, v); +#else - struct cred init_cred = { - .usage = ATOMIC_INIT(4), - .uid = GLOBAL_ROOT_UID, -@@ -61,13 +97,43 @@ struct cred init_cred = { - .group_info = &init_groups, - .ucounts = &init_ucounts, - }; + write_sysreg(v, vbar_el1); + isb(); +#endif -+ -+static inline void set_cred_subscribers(struct cred *cred, int n) -+{ -+#ifdef CONFIG_DEBUG_CREDENTIALS -+ atomic_set(&cred->subscribers, n); + } + + bool __read_mostly __nospectre_bhb; +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 09a15a9a8b2b..8cba0b63aa4e 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -55,6 +55,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include +#endif -+} -+ -+static inline int read_cred_subscribers(const struct cred *cred) -+{ -+#ifdef CONFIG_DEBUG_CREDENTIALS -+ return atomic_read(&cred->subscribers); -+#else -+ return 0; + + #include "../mm/internal.h" + +@@ -344,6 +347,9 @@ u64 cpu_logical_map(unsigned int cpu) + void __init __no_sanitize_address setup_arch(char **cmdline_p) + { + setup_initial_init_mm(_stext, _etext, _edata, _end); ++ #ifdef CONFIG_IEE_SIP ++ iee_si_setup_data(); ++ #endif + + *cmdline_p = boot_command_line; + +@@ -409,6 +415,10 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + unflatten_device_tree(); + + bootmem_init(); ++#ifdef CONFIG_PTP ++ /* Prepare page pool for page tables. */ ++ ptp_pg_cache_init(&pg_cache, 0, CONFIG_PGTABLE_LEVELS, "pg_cache"); ++#endif + + kasan_init(); + +diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S +index d4353741f331..eae2f1740936 100644 +--- a/arch/arm64/kernel/vmlinux.lds.S ++++ b/arch/arm64/kernel/vmlinux.lds.S +@@ -68,6 +68,9 @@ + #include + + #include "image.h" ++#ifdef CONFIG_IEE ++#include ++#endif + + OUTPUT_ARCH(aarch64) + ENTRY(_text) +@@ -181,6 +184,7 @@ SECTIONS + LOCK_TEXT + KPROBES_TEXT + HYPERVISOR_TEXT ++ IEE_TEXT + *(.gnu.warning) + } + +@@ -224,6 +228,10 @@ SECTIONS + swapper_pg_dir = .; + . += PAGE_SIZE; + ++#ifdef CONFIG_IEE ++ IEE_INIT_DATA +#endif -+} -+ -+static inline void alter_cred_subscribers(const struct cred *_cred, int n) -+{ -+#ifdef CONFIG_DEBUG_CREDENTIALS -+ struct cred *cred = (struct cred *) _cred; + -+ atomic_add(n, &cred->subscribers); + . = ALIGN(SEGMENT_ALIGN); + __init_begin = .; + __inittext_begin = .; +diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c +index b2ac06246327..240bb7b315b0 100644 +--- a/arch/arm64/mm/context.c ++++ b/arch/arm64/mm/context.c +@@ -16,6 +16,13 @@ + #include + #include + #include ++#ifdef CONFIG_IEE ++#include ++#include ++#endif ++#ifdef CONFIG_IEE_SIP ++#include +#endif -+} - /* - * The RCU callback to actually dispose of a set of credentials - */ - static void put_cred_rcu(struct rcu_head *rcu) + static u32 asid_bits; + static DEFINE_RAW_SPINLOCK(cpu_asid_lock); +@@ -348,7 +355,9 @@ asmlinkage void post_ttbr_update_workaround(void) + + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) { -+ #ifdef CONFIG_CREDP -+ struct cred *cred = *(struct cred **)(rcu + 1); ++#ifndef CONFIG_IEE_SIP + unsigned long ttbr1 = read_sysreg(ttbr1_el1); ++#endif + unsigned long asid = ASID(mm); + unsigned long ttbr0 = phys_to_ttbr(pgd_phys); + +@@ -360,6 +369,22 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) + if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN)) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); + ++#ifdef CONFIG_IEE ++ /* ++ * IEE requires the reserved ASID stored in TTBR1 and User ASID stored ++ * in TTBR0 to support ASID switch by changing TCR.A1. ++ */ ++ ttbr0 &= ~TTBR_ASID_MASK; ++ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); ++ ++ cpu_set_reserved_ttbr0_nosync(); ++ #ifdef CONFIG_IEE_SIP ++ iee_rwx_gate(IEE_SI_CONTEXT_SWITCH, ttbr0); + #else - struct cred *cred = container_of(rcu, struct cred, rcu); ++ write_sysreg(ttbr0, ttbr0_el1); ++ isb(); + #endif ++#else + /* Set ASID in TTBR1 since TCR.A1 is set */ + ttbr1 &= ~TTBR_ASID_MASK; + ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); +@@ -368,6 +393,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) + write_sysreg(ttbr1, ttbr1_el1); + write_sysreg(ttbr0, ttbr0_el1); + isb(); ++#endif + post_ttbr_update_workaround(); + } - kdebug("put_cred_rcu(%p)", cred); - -@@ -86,6 +152,9 @@ static void put_cred_rcu(struct rcu_head *rcu) - if (cred->ucounts) - put_ucounts(cred->ucounts); - put_user_ns(cred->user_ns); -+ #ifdef CONFIG_CREDP -+ kmem_cache_free(rcu_jar, (struct rcu_head *)(cred->rcu.func)); +@@ -379,6 +405,12 @@ static int asids_update_limit(void) + num_available_asids /= 2; + if (pinned_asid_map) + set_kpti_asid_bits(pinned_asid_map); ++#ifdef CONFIG_IEE ++ if (pinned_asid_map) { ++ __set_bit(ctxid2asid(IEE_ASID), pinned_asid_map); ++ __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), pinned_asid_map); ++ } ++#endif + } + /* + * Expect allocation after rollover to fail if we don't have at least +@@ -417,6 +449,14 @@ static int asids_init(void) + */ + if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) + set_kpti_asid_bits(asid_map); ++#ifdef CONFIG_IEE ++ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++ __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), asid_map); ++ __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), pinned_asid_map); + #endif - kmem_cache_free(cred_jar, cred); ++ __set_bit(ctxid2asid(IEE_ASID), asid_map); ++ __set_bit(ctxid2asid(IEE_ASID), pinned_asid_map); ++#endif + return 0; } + early_initcall(asids_init); +diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c +index 647bbd9bad5b..3e87fded4b4e 100644 +--- a/arch/arm64/mm/fault.c ++++ b/arch/arm64/mm/fault.c +@@ -43,6 +43,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif -@@ -104,10 +173,22 @@ void __put_cred(struct cred *cred) - BUG_ON(cred == current->cred); - BUG_ON(cred == current->real_cred); + static int sysctl_machine_check_safe = IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC); -+ #ifdef CONFIG_CREDP -+ if (*(int *)(&(((struct rcu_head *)(cred->rcu.func))->next))) -+ #else - if (cred->non_rcu) -+ #endif -+ #ifdef CONFIG_CREDP -+ put_cred_rcu((struct rcu_head *)(cred->rcu.func)); -+ #else - put_cred_rcu(&cred->rcu); -+ #endif - else -+ #ifdef CONFIG_CREDP -+ call_rcu((struct rcu_head *)(cred->rcu.func), put_cred_rcu); +@@ -261,7 +264,11 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, + pteval ^= PTE_RDONLY; + pteval |= pte_val(entry); + pteval ^= PTE_RDONLY; ++ #ifdef CONFIG_PTP ++ pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval); + #else - call_rcu(&cred->rcu, put_cred_rcu); + pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); + #endif - } - EXPORT_SYMBOL(__put_cred); - -@@ -178,7 +259,18 @@ struct cred *cred_alloc_blank(void) - if (!new) - return NULL; - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_rcu(new,kmem_cache_zalloc(rcu_jar, GFP_KERNEL)); -+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; -+ iee_set_cred_atomic_set_usage(new,1); -+ #else - atomic_long_set(&new->usage, 1); -+ #endif -+ -+ #ifdef CONFIG_DEBUG_CREDENTIALS -+ new->magic = CRED_MAGIC; -+ #endif -+ - if (security_cred_alloc_blank(new, GFP_KERNEL_ACCOUNT) < 0) - goto error; + } while (pteval != old_pteval); -@@ -213,13 +305,25 @@ struct cred *prepare_creds(void) - if (!new) - return NULL; + /* Invalidate a stale read-only entry */ +@@ -376,9 +383,14 @@ static void do_tag_recovery(unsigned long addr, unsigned long esr, + * It will be done lazily on the other CPUs when they will hit a + * tag fault. + */ ++#ifdef CONFIG_IEE_SIP ++ iee_si_sysreg_clear_set(SCTLR_EL1, SCTLR_EL1_TCF_MASK, ++ SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, + SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); + isb(); ++#endif + } -+ #ifdef CONFIG_CREDP -+ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL)); -+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; -+ #endif -+ - kdebug("prepare_creds() alloc %p", new); + static bool is_el1_mte_sync_tag_check_fault(unsigned long esr) +diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c +index a55b36d04590..4574b04ed23b 100644 +--- a/arch/arm64/mm/fixmap.c ++++ b/arch/arm64/mm/fixmap.c +@@ -15,6 +15,9 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#endif - old = task->cred; -+ #ifdef CONFIG_CREDP -+ iee_copy_cred(old,new); -+ -+ iee_set_cred_non_rcu(new,0); -+ iee_set_cred_atomic_set_usage(new,1); -+ #else - memcpy(new, old, sizeof(struct cred)); + #define NR_BM_PTE_TABLES \ + SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) +@@ -28,9 +31,15 @@ static_assert(NR_BM_PMD_TABLES == 1); - new->non_rcu = 0; - atomic_long_set(&new->usage, 1); -+ #endif - get_group_info(new->group_info); - get_uid(new->user); - get_user_ns(new->user_ns); -@@ -232,10 +336,18 @@ struct cred *prepare_creds(void) - #endif + #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) - #ifdef CONFIG_SECURITY -+#ifdef CONFIG_CREDP -+ iee_set_cred_security(new,NULL); ++#ifdef CONFIG_PTP ++pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __section(".iee.ptp") __aligned(PAGE_SIZE); ++pmd_t bm_pmd[PTRS_PER_PMD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; ++pud_t bm_pud[PTRS_PER_PUD] __section(".iee.ptp") __aligned(PAGE_SIZE) __maybe_unused; +#else - new->security = NULL; + static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; + static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; + static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +#endif - #endif - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); -+ #else - new->ucounts = get_ucounts(new->ucounts); -+ #endif - if (!new->ucounts) - goto error; - -@@ -265,15 +377,30 @@ struct cred *prepare_exec_creds(void) - #ifdef CONFIG_KEYS - /* newly exec'd tasks don't get a thread keyring */ - key_put(new->thread_keyring); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(new,NULL); -+ #else - new->thread_keyring = NULL; -+ #endif - - /* inherit the session keyring; new process keyring */ - key_put(new->process_keyring); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_process_keyring(new,NULL); -+ #else - new->process_keyring = NULL; -+ #endif - #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,new->euid); -+ iee_set_cred_suid(new,new->euid); -+ iee_set_cred_fsgid(new,new->egid); -+ iee_set_cred_sgid(new,new->egid); -+ #else - new->suid = new->fsuid = new->euid; - new->sgid = new->fsgid = new->egid; -+ #endif + static inline pte_t *fixmap_pte(unsigned long addr) + { +@@ -44,8 +53,12 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) - return new; - } -@@ -327,7 +454,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) - * had one */ - if (new->thread_keyring) { - key_put(new->thread_keyring); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(new,NULL); + if (pmd_none(pmd)) { + ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; ++ #ifdef CONFIG_PTP ++ __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); + #else - new->thread_keyring = NULL; + __pmd_populate(pmdp, __pa_symbol(ptep), + PMD_TYPE_TABLE | PMD_TABLE_AF); + #endif - if (clone_flags & CLONE_THREAD) - install_thread_keyring_to_cred(new); } -@@ -337,7 +468,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) - */ - if (!(clone_flags & CLONE_THREAD)) { - key_put(new->process_keyring); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_process_keyring(new,NULL); + } + +@@ -57,8 +70,12 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, + pmd_t *pmdp; + + if (pud_none(pud)) ++ #ifdef CONFIG_PTP ++ __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); + #else - new->process_keyring = NULL; + __pud_populate(pudp, __pa_symbol(bm_pmd), + PUD_TYPE_TABLE | PUD_TABLE_AF); + #endif + + pmdp = pmd_offset_kimg(pudp, addr); + do { +@@ -85,8 +102,12 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, } - #endif -@@ -594,7 +729,11 @@ int set_cred_ucounts(struct cred *new) - if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid))) - return -EAGAIN; + if (p4d_none(p4d)) ++ #ifdef CONFIG_PTP ++ __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); ++ #else + __p4d_populate(p4dp, __pa_symbol(bm_pud), + P4D_TYPE_TABLE | P4D_TABLE_AF); ++ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_ucounts(new, new_ucounts); -+ #else - new->ucounts = new_ucounts; -+ #endif - put_ucounts(old_ucounts); + pudp = pud_offset_kimg(p4dp, addr); + early_fixmap_init_pmd(pudp, addr, end); +@@ -124,9 +145,17 @@ void __set_fixmap(enum fixed_addresses idx, + ptep = fixmap_pte(addr); - return 0; -@@ -606,8 +745,21 @@ int set_cred_ucounts(struct cred *new) - void __init cred_init(void) + if (pgprot_val(flags)) { ++ #ifdef CONFIG_PTP ++ iee_set_bm_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ #else + __set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ #endif + } else { ++ #ifdef CONFIG_PTP ++ iee_set_bm_pte(ptep, __pte(0)); ++ #else + __pte_clear(&init_mm, addr, ptep); ++ #endif + flush_tlb_kernel_range(addr, addr+PAGE_SIZE); + } + } +@@ -174,6 +203,7 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) + /* + * Copy the fixmap region into a new pgdir. + */ ++#ifndef CONFIG_PTP + void __init fixmap_copy(pgd_t *pgdir) { - /* allocate a slab in which we can store credentials */ -+ #ifdef CONFIG_CREDP - cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, -+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE, NULL); -+ rcu_jar = kmem_cache_create("rcu_jar", sizeof(struct rcu_head) + sizeof(struct cred *), 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -+ // Map init_cred -+ *((struct rcu_head **)(&(init_cred.rcu.func))) = (struct rcu_head *)kmem_cache_zalloc(rcu_jar, GFP_KERNEL); -+ *(struct cred **)(((struct rcu_head *)(init_cred.rcu.func)) + 1) = &init_cred; -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa_symbol(&init_cred))); -+ iee_set_logical_mem_ro((unsigned long)&init_cred); -+ iee_set_logical_mem_ro((unsigned long)__va(__pa_symbol(&init_cred))); -+ #else -+ cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, -+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -+ #endif + if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdir, FIXADDR_TOT_START)))) { +@@ -204,3 +234,4 @@ void __init fixmap_copy(pgd_t *pgdir) + BUG(); + } } - - /** -@@ -638,29 +790,56 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon) - if (!new) - return NULL; - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL)); -+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; -+ #endif -+ - kdebug("prepare_kernel_cred() alloc %p", new); - - old = get_task_cred(daemon); - -+ #ifdef CONFIG_CREDP -+ iee_copy_cred(old,new); -+ iee_set_cred_non_rcu(new,0); -+ iee_set_cred_atomic_set_usage(new,1); -+ #else - *new = *old; - new->non_rcu = 0; - atomic_long_set(&new->usage, 1); -+ #endif - get_uid(new->user); - get_user_ns(new->user_ns); - get_group_info(new->group_info); - - #ifdef CONFIG_KEYS -+#ifdef CONFIG_CREDP -+ iee_set_cred_session_keyring(new,NULL); -+ iee_set_cred_process_keyring(new,NULL); -+ iee_set_cred_thread_keyring(new,NULL); -+ iee_set_cred_request_key_auth(new,NULL); -+ iee_set_cred_jit_keyring(new,KEY_REQKEY_DEFL_THREAD_KEYRING); -+#else - new->session_keyring = NULL; - new->process_keyring = NULL; - new->thread_keyring = NULL; - new->request_key_auth = NULL; - new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; - #endif +#endif - - #ifdef CONFIG_SECURITY -+#ifdef CONFIG_CREDP -+ iee_set_cred_security(new,NULL); -+#else - new->security = NULL; - #endif +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index da75dd9d964b..7a4b0e29ddba 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -663,6 +663,10 @@ void __init mem_init(void) + */ + sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; + } ++#ifdef CONFIG_IEE ++ extern void iee_stack_init(void); ++ iee_stack_init(); +#endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); -+ #else - new->ucounts = get_ucounts(new->ucounts); -+ #endif - if (!new->ucounts) - goto error; - -@@ -727,8 +906,13 @@ int set_create_files_as(struct cred *new, struct inode *inode) - { - if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) - return -EINVAL; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,inode->i_uid); -+ iee_set_cred_fsgid(new,inode->i_gid); -+ #else - new->fsuid = inode->i_uid; - new->fsgid = inode->i_gid; -+ #endif - return security_kernel_create_files_as(new, inode); } - EXPORT_SYMBOL(set_create_files_as); -diff --git a/kernel/exit.c b/kernel/exit.c -index 3540b2c9b1b6..3bb305d7edf2 100644 ---- a/kernel/exit.c -+++ b/kernel/exit.c -@@ -74,6 +74,11 @@ - #include - #include + void free_initmem(void) +diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c +index b2ef6c56db74..f0a77c282d08 100644 +--- a/arch/arm64/mm/mmu.c ++++ b/arch/arm64/mm/mmu.c +@@ -42,6 +42,17 @@ + #include + #include + #include +#ifdef CONFIG_IEE -+#include -+#include ++#include ++#include ++#include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif ++#ifdef CONFIG_PTP ++#include +#endif -+ - /* - * The default value should be high enough to not crash a system that randomly - * crashes its kernel from time to time, but low enough to at least not permit -@@ -560,6 +565,9 @@ static void exit_mm(void) - smp_mb__after_spinlock(); - local_irq_disable(); - current->mm = NULL; -+ #ifdef CONFIG_IEE -+ iee_set_token_pgd(current, NULL); -+ #endif - membarrier_update_current_mm(NULL); - enter_lazy_tlb(mm, current); - local_irq_enable(); -diff --git a/kernel/fork.c b/kernel/fork.c -index f30b24c68442..617cb03938ff 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -115,6 +115,13 @@ - #define CREATE_TRACE_POINTS - #include - -+#ifdef CONFIG_IEE -+#include -+#include -+#include -+#include +#endif -+ - /* - * Minimum number of threads to boot the kernel - */ -@@ -128,14 +135,14 @@ - /* - * Protected counters by write_lock_irq(&tasklist_lock) - */ --unsigned long total_forks; /* Handle normal Linux uptimes. */ --int nr_threads; /* The idle threads do not count.. */ -+unsigned long total_forks; /* Handle normal Linux uptimes. */ -+int nr_threads; /* The idle threads do not count.. */ - --static int max_threads; /* tunable limit on nr_threads */ -+static int max_threads; /* tunable limit on nr_threads */ - --#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) -+#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) - --static const char * const resident_page_types[] = { -+static const char *const resident_page_types[] = { - NAMED_ARRAY_INDEX(MM_FILEPAGES), - NAMED_ARRAY_INDEX(MM_ANONPAGES), - NAMED_ARRAY_INDEX(MM_SWAPENTS), -@@ -144,7 +151,7 @@ static const char * const resident_page_types[] = { - - DEFINE_PER_CPU(unsigned long, process_counts) = 0; --__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ -+__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ + #define NO_BLOCK_MAPPINGS BIT(0) + #define NO_CONT_MAPPINGS BIT(1) +@@ -82,6 +93,7 @@ long __section(".mmuoff.data.write") __early_cpu_boot_status; + unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; + EXPORT_SYMBOL(empty_zero_page); - #ifdef CONFIG_PROVE_RCU - int lockdep_tasklist_lock_is_held(void) -@@ -159,7 +166,7 @@ int nr_processes(void) - int cpu; - int total = 0; ++#ifndef CONFIG_IEE + static DEFINE_SPINLOCK(swapper_pgdir_lock); + static DEFINE_MUTEX(fixmap_lock); -- for_each_possible_cpu(cpu) -+ for_each_possible_cpu (cpu) - total += per_cpu(process_counts, cpu); +@@ -452,6 +464,7 @@ static phys_addr_t pgd_pgtable_alloc(int shift) - return total; -@@ -170,7 +177,11 @@ void __weak arch_release_task_struct(struct task_struct *tsk) + return pa; } ++#endif - #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR -+#if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+struct kmem_cache *task_struct_cachep; -+#else - static struct kmem_cache *task_struct_cachep; -+#endif - - static inline struct task_struct *alloc_task_struct_node(int node) - { -@@ -190,7 +201,7 @@ static inline void free_task_struct(struct task_struct *tsk) - * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a - * kmemcache based allocator. - */ --# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) -+#if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) - - # ifdef CONFIG_VMAP_STACK /* -@@ -311,8 +322,8 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) - * so memcg accounting is performed manually on assigning/releasing - * stacks to tasks. Drop __GFP_ACCOUNT. - */ -- stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, -- VMALLOC_START, VMALLOC_END, -+ stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START, -+ VMALLOC_END, - THREADINFO_GFP & ~__GFP_ACCOUNT, - PAGE_KERNEL, - 0, node, __builtin_return_address(0)); -@@ -410,9 +421,10 @@ static void free_thread_stack(struct task_struct *tsk) - - void thread_stack_cache_init(void) + * This function can only be used to modify existing table entries, +@@ -466,8 +479,13 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, + &phys, virt); + return; + } ++ #ifdef CONFIG_PTP ++ __create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL, ++ NO_CONT_MAPPINGS); ++ #else + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, + NO_CONT_MAPPINGS); ++ #endif + } + + void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, +@@ -504,8 +522,13 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, + static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, + phys_addr_t end, pgprot_t prot, int flags) { -- thread_stack_cache = kmem_cache_create_usercopy("thread_stack", -- THREAD_SIZE, THREAD_SIZE, 0, 0, -- THREAD_SIZE, NULL); -+ thread_stack_cache = -+ kmem_cache_create_usercopy("thread_stack", THREAD_SIZE, -+ THREAD_SIZE, 0, 0, THREAD_SIZE, -+ NULL); - BUG_ON(thread_stack_cache == NULL); ++ #ifdef CONFIG_PTP ++ __create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), end - start, ++ prot, early_pgtable_alloc, flags); ++ #else + __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, + prot, early_pgtable_alloc, flags); ++ #endif } -@@ -502,7 +514,8 @@ struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) + void __init mark_linear_text_alias_ro(void) +@@ -665,8 +688,13 @@ static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, + BUG_ON(!PAGE_ALIGNED(pa_start)); + BUG_ON(!PAGE_ALIGNED(size)); - struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) - { -- struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); -+ struct vm_area_struct *new = -+ kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); ++ #ifdef CONFIG_PTP ++ __create_pgd_mapping_pre_init(pgdp, pa_start, (unsigned long)va_start, size, prot, ++ early_pgtable_alloc, flags); ++ #else + __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, + early_pgtable_alloc, flags); ++ #endif - if (!new) - return NULL; -@@ -602,6 +615,10 @@ void put_task_stack(struct task_struct *tsk) - } - #endif + if (!(vm_flags & VM_NO_GUARD)) + size += PAGE_SIZE; +@@ -697,7 +725,11 @@ static int __init map_entry_trampoline(void) + pgprot_val(prot) &= ~PTE_NG; -+#ifdef CONFIG_KOI -+extern s64 koi_offset; -+#endif -+ - void free_task(struct task_struct *tsk) + /* Map only the text into the trampoline page table */ ++ #ifdef CONFIG_PTP ++ iee_memset(__va(__pa_symbol(tramp_pg_dir)), 0, PGD_SIZE); ++ #else + memset(tramp_pg_dir, 0, PGD_SIZE); ++ #endif + __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, + entry_tramp_text_size(), prot, + __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); +@@ -739,6 +771,9 @@ static void __init map_kernel(pgd_t *pgdp) { - #ifdef CONFIG_SECCOMP -@@ -633,6 +650,21 @@ void free_task(struct task_struct *tsk) - if (dynamic_affinity_enabled()) - sched_prefer_cpus_free(tsk); - #endif -+#ifdef CONFIG_IEE -+ iee_invalidate_token(tsk); -+#ifdef CONFIG_KOI -+ // Free koi stack. -+ unsigned long koi_stack = iee_rw_gate(IEE_READ_KOI_STACK_BASE, tsk); -+ if (koi_stack != 0) -+ free_pages(koi_stack, 2); + static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, + vmlinux_initdata, vmlinux_data; ++#ifdef CONFIG_IEE_SIP ++ static struct vm_struct vmlinux_iee_text, vmlinux_text_end; +#endif + + /* + * External debuggers may need to write directly to the text +@@ -755,12 +790,23 @@ static void __init map_kernel(pgd_t *pgdp) + if (arm64_early_this_cpu_has_bti()) + text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); + ++#ifdef CONFIG_IEE_SIP ++ /* Ensure iee si code are not mapped with block descriptor. */ ++ map_kernel_segment(pgdp, _stext, __iee_si_text_start, text_prot, ++ &vmlinux_text, 0, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __iee_si_text_start, __iee_si_text_end, ++ SET_NG(text_prot), &vmlinux_iee_text, NO_CONT_MAPPINGS | ++ NO_BLOCK_MAPPINGS, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __iee_si_text_end, _etext, text_prot, ++ &vmlinux_text_end, 0, VM_NO_GUARD); +#else -+#ifdef CONFIG_KOI -+ struct task_token *token = (struct task_token *)((unsigned long)tsk + KOI_OFFSET); -+ memset(token, 0, sizeof(struct task_token)); -+#endif + /* + * Only rodata will be remapped with different permissions later on, + * all other segments are allowed to use contiguous mappings. + */ + map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, + VM_NO_GUARD); +#endif -+ - #ifdef CONFIG_QOS_SCHED_SMART_GRID - if (smart_grid_enabled()) - sched_grid_qos_free(tsk); -@@ -661,7 +693,7 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) - - #ifdef CONFIG_MMU - static __latent_entropy int dup_mmap(struct mm_struct *mm, -- struct mm_struct *oldmm) -+ struct mm_struct *oldmm) - { - struct vm_area_struct *mpnt, *tmp; - int retval; -@@ -772,7 +804,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - flush_dcache_mmap_lock(mapping); - /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, -- &mapping->i_mmap); -+ &mapping->i_mmap); - flush_dcache_mmap_unlock(mapping); - i_mmap_unlock_write(mapping); - } -@@ -846,7 +878,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) - mmap_write_unlock(oldmm); - return 0; - } --#define mm_alloc_pgd(mm) (0) -+#define mm_alloc_pgd(mm) (0) - #define mm_free_pgd(mm) - #endif /* CONFIG_MMU */ + map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, + &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); + map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, +@@ -805,25 +851,49 @@ static void __init create_idmap(void) -@@ -854,20 +886,22 @@ static void check_mm(struct mm_struct *mm) + void __init paging_init(void) { - int i; - -- BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, -- "Please make sure 'struct resident_page_types[]' is updated as well"); -+ BUILD_BUG_ON_MSG( -+ ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, -+ "Please make sure 'struct resident_page_types[]' is updated as well"); - - for (i = 0; i < NR_MM_COUNTERS; i++) { - long x = mm_counter_sum(mm, i); - - if (unlikely(x)) -- pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", -- mm, resident_page_types[i], x); -+ pr_alert( -+ "BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", -+ mm, resident_page_types[i], x); - } - - if (mm_pgtables_bytes(mm)) - pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", -- mm_pgtables_bytes(mm)); -+ mm_pgtables_bytes(mm)); - - #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS - VM_BUG_ON_MM(mm->pmd_huge_pte, mm); -@@ -1018,14 +1052,6 @@ void __put_task_struct(struct task_struct *tsk) - } - EXPORT_SYMBOL_GPL(__put_task_struct); ++ #ifdef CONFIG_PTP ++ pgd_t *pgdp = pgd_set_fixmap_pre_init(__pa_symbol(swapper_pg_dir)); ++ #else + pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); ++ #endif + extern pgd_t init_idmap_pg_dir[]; --void __put_task_struct_rcu_cb(struct rcu_head *rhp) --{ -- struct task_struct *task = container_of(rhp, struct task_struct, rcu); -- -- __put_task_struct(task); --} --EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb); -- - void __init __weak arch_task_cache_init(void) { } + idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0)); - /* -@@ -1043,8 +1069,8 @@ static void set_max_threads(unsigned int max_threads_suggested) - if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64) - threads = MAX_THREADS; - else -- threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE, -- (u64) THREAD_SIZE * 8UL); -+ threads = div64_u64((u64)nr_pages * (u64)PAGE_SIZE, -+ (u64)THREAD_SIZE * 8UL); - - if (threads > max_threads_suggested) - threads = max_threads_suggested; -@@ -1079,17 +1105,24 @@ void __init fork_init(void) - int i; - #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR - #ifndef ARCH_MIN_TASKALIGN --#define ARCH_MIN_TASKALIGN 0 -+#define ARCH_MIN_TASKALIGN 0 - #endif - int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); - unsigned long useroffset, usersize; ++ #ifdef CONFIG_IEE ++ early_iee_data_cache_init(); ++ #endif ++ + map_kernel(pgdp); + map_mem(pgdp); - /* create a slab on which task_structs can be allocated */ - task_struct_whitelist(&useroffset, &usersize); + #ifdef CONFIG_IEE - task_struct_cachep = kmem_cache_create_usercopy("task_struct", - arch_task_struct_size, align, -- SLAB_PANIC|SLAB_ACCOUNT, -+ SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE, - useroffset, usersize, NULL); ++ iee_init_mappings(pgdp); ++ #endif ++ ++ #ifdef CONFIG_PTP ++ pgd_clear_fixmap_pre_init(); + #else -+ task_struct_cachep = -+ kmem_cache_create_usercopy("task_struct", arch_task_struct_size, -+ align, SLAB_PANIC | SLAB_ACCOUNT, -+ useroffset, usersize, NULL); + pgd_clear_fixmap(); + #endif - #endif - /* do the arch specific task caches init */ -@@ -1097,8 +1130,8 @@ void __init fork_init(void) + cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir); + init_mm.pgd = swapper_pg_dir; - set_max_threads(MAX_THREADS); ++ #ifdef CONFIG_IEE ++ init_early_iee_data(); ++ #endif ++ + memblock_phys_free(__pa_symbol(init_pg_dir), + __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); -- init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; -- init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; -+ init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads / 2; -+ init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads / 2; - init_task.signal->rlim[RLIMIT_SIGPENDING] = - init_task.signal->rlim[RLIMIT_NPROC]; + memblock_allow_resize(); -@@ -1111,8 +1144,8 @@ void __init fork_init(void) - set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); + create_idmap(); ++ ++ #ifdef CONFIG_IEE ++ iee_init_post(); ++ #endif + } - #ifdef CONFIG_VMAP_STACK -- cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", -- NULL, free_vm_stack_cache); -+ cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", NULL, -+ free_vm_stack_cache); - #endif + #ifdef CONFIG_MEMORY_HOTPLUG +diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c +index 4a64089e5771..7e96cf3065fb 100644 +--- a/arch/arm64/mm/pgd.c ++++ b/arch/arm64/mm/pgd.c +@@ -14,6 +14,9 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#endif - scs_init(); -@@ -1122,7 +1155,7 @@ void __init fork_init(void) - } + static struct kmem_cache *pgd_cache __ro_after_init; - int __weak arch_dup_task_struct(struct task_struct *dst, -- struct task_struct *src) -+ struct task_struct *src) - { - *dst = *src; - return 0; -@@ -1133,14 +1166,14 @@ void set_task_stack_end_magic(struct task_struct *tsk) - unsigned long *stackend; +@@ -22,7 +25,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm) + gfp_t gfp = GFP_PGTABLE_USER; - stackend = end_of_stack(tsk); -- *stackend = STACK_END_MAGIC; /* for overflow detection */ -+ *stackend = STACK_END_MAGIC; /* for overflow detection */ + if (PGD_SIZE == PAGE_SIZE) ++ #ifdef CONFIG_PTP ++ return ptp_pg_alloc(&pg_cache, gfp); ++ #else + return (pgd_t *)__get_free_page(gfp); ++ #endif + else + return kmem_cache_alloc(pgd_cache, gfp); } - - static bool dup_resvd_task_struct(struct task_struct *dst, - struct task_struct *orig, int node) +@@ -30,7 +37,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm) + void pgd_free(struct mm_struct *mm, pgd_t *pgd) { -- dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd), -- GFP_KERNEL, node); -+ dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd), GFP_KERNEL, -+ node); - if (!dst->_resvd) - return false; - -@@ -1321,7 +1354,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm) + if (PGD_SIZE == PAGE_SIZE) ++ #ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, pgd); ++ #else + free_page((unsigned long)pgd); ++ #endif + else + kmem_cache_free(pgd_cache, pgd); } +diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S +index 9ef7b07349d7..dafdc63c6aed 100644 +--- a/arch/arm64/mm/proc.S ++++ b/arch/arm64/mm/proc.S +@@ -21,6 +21,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE ++#include ++#endif - static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, -- struct user_namespace *user_ns) -+ struct user_namespace *user_ns) - { - mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); - mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); -@@ -1438,8 +1471,8 @@ EXPORT_SYMBOL_GPL(mmput); - #ifdef CONFIG_MMU - static void mmput_async_fn(struct work_struct *work) - { -- struct mm_struct *mm = container_of(work, struct mm_struct, -- async_put_work); -+ struct mm_struct *mm = -+ container_of(work, struct mm_struct, async_put_work); - - __mmput(mm); - } -@@ -1615,13 +1648,12 @@ struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) - struct mm_struct *mm; - int err; - -- err = down_read_killable(&task->signal->exec_update_lock); -+ err = down_read_killable(&task->signal->exec_update_lock); - if (err) - return ERR_PTR(err); - - mm = get_task_mm(task); -- if (mm && mm != current->mm && -- !ptrace_may_access(task, mode)) { -+ if (mm && mm != current->mm && !ptrace_may_access(task, mode)) { - mmput(mm); - mm = ERR_PTR(-EACCES); - } -@@ -1644,7 +1676,7 @@ static void complete_vfork_done(struct task_struct *tsk) - } + #ifdef CONFIG_ARM64_64K_PAGES + #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K +@@ -173,6 +176,9 @@ SYM_FUNC_END(cpu_do_resume) + adrp \tmp1, reserved_pg_dir + phys_to_ttbr \tmp2, \tmp1 + offset_ttbr1 \tmp2, \tmp1 ++ #ifdef CONFIG_IEE ++ orr \tmp2, \tmp2, #IEE_ASM_ASID ++ #endif + msr ttbr1_el1, \tmp2 + isb + tlbi vmalle1 +@@ -190,6 +196,9 @@ SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) + __idmap_cpu_set_reserved_ttbr1 x1, x3 - static int wait_for_vfork_done(struct task_struct *child, -- struct completion *vfork) -+ struct completion *vfork) - { - unsigned int state = TASK_UNINTERRUPTIBLE|TASK_KILLABLE|TASK_FREEZABLE; - int killed; -@@ -1695,8 +1727,8 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm) - * not set up a proper pointer then tough luck. - */ - put_user(0, tsk->clear_child_tid); -- do_futex(tsk->clear_child_tid, FUTEX_WAKE, -- 1, NULL, NULL, 0, 0); -+ do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1, NULL, -+ NULL, 0, 0); - } - tsk->clear_child_tid = NULL; - } -@@ -1780,6 +1812,9 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) - #endif + offset_ttbr1 x0, x3 ++ #ifdef CONFIG_IEE ++ orr x0, x0, #IEE_ASM_ASID ++ #endif + msr ttbr1_el1, x0 + isb - tsk->mm = NULL; +@@ -422,9 +431,15 @@ SYM_FUNC_START(__cpu_setup) + tcr .req x16 + tcr2 .req x15 + mov_q mair, MAIR_EL1_SET +#ifdef CONFIG_IEE -+ iee_set_token_pgd(tsk, NULL); ++ mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ ++ TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ ++ TCR_TBI0 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS ++#else + mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ + TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ + TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS +#endif - tsk->active_mm = NULL; + mov tcr2, xzr - /* -@@ -1811,6 +1846,9 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) - } + tcr_clear_errata_bits tcr, x9, x5 +diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c +index 5139a28130c0..c7a356adb51b 100644 +--- a/arch/arm64/mm/trans_pgd.c ++++ b/arch/arm64/mm/trans_pgd.c +@@ -252,7 +252,11 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, + level_mask = GENMASK_ULL(level_msb, level_lsb); - tsk->mm = mm; -+#ifdef CONFIG_IEE -+ iee_set_token_pgd(tsk, mm->pgd); -+#endif - tsk->active_mm = mm; - sched_mm_cid_fork(tsk); - return 0; -@@ -2025,8 +2063,8 @@ static inline void init_task_pid_links(struct task_struct *task) - INIT_HLIST_NODE(&task->pid_links[type]); - } + index = (dst_addr & level_mask) >> level_lsb; ++ #ifdef CONFIG_PTP ++ set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry)); ++ #else + *(levels[this_level] + index) = prev_level_entry; ++ #endif --static inline void --init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) -+static inline void init_task_pid(struct task_struct *task, enum pid_type type, -+ struct pid *pid) - { - if (type == PIDTYPE_PID) - task->thread_pid = pid; -@@ -2288,6 +2326,11 @@ static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk) - mutex_unlock(&oom_adj_mutex); - } + pfn = virt_to_pfn(levels[this_level]); + prev_level_entry = pte_val(pfn_pte(pfn, +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index eb3551d427a1..c562a22cc35c 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -1552,6 +1552,27 @@ config AMD_MEM_ENCRYPT + This requires an AMD processor that supports Secure Memory + Encryption (SME). -+#if defined(CONFIG_KOI) && !defined(CONFIG_IEE) -+extern s64 koi_offset; -+#endif ++config IEE ++ depends on X86_64 ++ def_bool y + ++config IEE_SIP ++ depends on IEE ++ def_bool y + - #ifdef CONFIG_RV - static void rv_task_fork(struct task_struct *p) - { -@@ -2325,10 +2368,12 @@ __latent_entropy struct task_struct *copy_process( - * Don't allow sharing the root directory with processes in a different - * namespace - */ -- if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) -+ if ((clone_flags & (CLONE_NEWNS | CLONE_FS)) == -+ (CLONE_NEWNS | CLONE_FS)) - return ERR_PTR(-EINVAL); ++config IEE_PTRP ++ depends on IEE ++ def_bool y ++ ++config PTP ++ depends on IEE ++ def_bool y ++ ++config PTP_RESERVE_ORDER ++ depends on PTP ++ int "maximum allowable 2^PTP_RESERVE_ORDER pages for one level page table" ++ range 9 15 ++ default 12 + # Common NUMA Features + config NUMA + bool "NUMA Memory Allocation and Scheduler Support" +diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c +index 8ecb4d40e20d..05b444e67dc2 100644 +--- a/arch/x86/boot/compressed/ident_map_64.c ++++ b/arch/x86/boot/compressed/ident_map_64.c +@@ -23,7 +23,11 @@ + /* Use the static base for this part of the boot process */ + #undef __PAGE_OFFSET + #define __PAGE_OFFSET __PAGE_OFFSET_BASE ++#ifdef CONFIG_PTP ++#include "../../kernel/haoc/ptp/ident_map.c" ++#else + #include "../../mm/ident_map.c" ++#endif -- if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) -+ if ((clone_flags & (CLONE_NEWUSER | CLONE_FS)) == -+ (CLONE_NEWUSER | CLONE_FS)) - return ERR_PTR(-EINVAL); + #define _SETUP + #include /* For COMMAND_LINE_SIZE */ +@@ -101,9 +105,15 @@ void kernel_add_identity_map(unsigned long start, unsigned long end) + return; - /* -@@ -2353,7 +2398,7 @@ __latent_entropy struct task_struct *copy_process( - * from creating siblings. - */ - if ((clone_flags & CLONE_PARENT) && -- current->signal->flags & SIGNAL_UNKILLABLE) -+ current->signal->flags & SIGNAL_UNKILLABLE) - return ERR_PTR(-EINVAL); + /* Build the mapping. */ ++ #ifdef CONFIG_PTP ++ ret = ptp_kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); ++ if (ret) ++ error("Error: kernel_ident_mapping_init_for_iee() failed\n"); ++#else + ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); + if (ret) + error("Error: kernel_ident_mapping_init() failed\n"); ++#endif + } + + /* Locates and clears a region for a new top level page table. */ +@@ -180,7 +190,11 @@ void initialize_identity_maps(void *rmode) + sev_prep_identity_maps(top_level_pgt); + + /* Load the new page-table. */ ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr3_early(top_level_pgt); ++ #else + write_cr3(top_level_pgt); ++ #endif /* -@@ -2398,6 +2443,10 @@ __latent_entropy struct task_struct *copy_process( - p = dup_task_struct(current, node); - if (!p) - goto fork_out; -+#ifdef CONFIG_IEE -+ iee_validate_token(p); -+#else + * Now that the required page table mappings are established and a +@@ -208,7 +222,11 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, + + /* Populate the PTEs */ + for (i = 0; i < PTRS_PER_PMD; i++) { ++#ifdef CONFIG_PTP ++ ptp_set_pte_pre_init(&pte[i], __pte(address | page_flags)); ++#else + set_pte(&pte[i], __pte(address | page_flags)); ++#endif + address += PAGE_SIZE; + } + +@@ -222,9 +240,17 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, + * of a TLB multihit. + */ + pmd = __pmd((unsigned long)pte | info->kernpg_flag); ++#ifdef CONFIG_PTP ++ ptp_set_pmd_pre_init(pmdp, pmd); ++#else + set_pmd(pmdp, pmd); ++#endif + /* Flush TLB to establish the new PMD */ ++#ifdef CONFIG_IEE_SIP ++ iee_write_cr3_early(top_level_pgt); ++#else + write_cr3(top_level_pgt); +#endif - p->flags &= ~PF_KTHREAD; - if (args->kthread) - p->flags |= PF_KTHREAD; -@@ -2419,7 +2468,8 @@ __latent_entropy struct task_struct *copy_process( - /* - * Clear TID on mm_release()? - */ -- p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; -+ p->clear_child_tid = -+ (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; - - ftrace_graph_init_task(p); -@@ -2539,10 +2589,10 @@ __latent_entropy struct task_struct *copy_process( - #endif - #ifdef CONFIG_TRACE_IRQFLAGS - memset(&p->irqtrace, 0, sizeof(p->irqtrace)); -- p->irqtrace.hardirq_disable_ip = _THIS_IP_; -- p->irqtrace.softirq_enable_ip = _THIS_IP_; -- p->softirqs_enabled = 1; -- p->softirq_context = 0; -+ p->irqtrace.hardirq_disable_ip = _THIS_IP_; -+ p->irqtrace.softirq_enable_ip = _THIS_IP_; -+ p->softirqs_enabled = 1; -+ p->softirq_context = 0; - #endif + return pte + pte_index(__address); + } +@@ -314,7 +340,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, + pte = *ptep; + pte = pte_set_flags(pte, set); + pte = pte_clear_flags(pte, clr); ++#ifdef CONFIG_PTP ++ ptp_set_pte_pre_init(ptep, pte); ++#else + set_pte(ptep, pte); ++#endif - p->pagefault_disabled = 0; -@@ -2555,8 +2605,8 @@ __latent_entropy struct task_struct *copy_process( - p->blocked_on = NULL; /* not blocked yet */ - #endif - #ifdef CONFIG_BCACHE -- p->sequential_io = 0; -- p->sequential_io_avg = 0; -+ p->sequential_io = 0; -+ p->sequential_io_avg = 0; - #endif - #ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); -@@ -2643,7 +2693,7 @@ __latent_entropy struct task_struct *copy_process( /* - * sigaltstack should be cleared when sharing the same VM - */ -- if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) -+ if ((clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) - sas_ss_reset(p); + * If the encryption attribute is being set, then change the page state to +@@ -325,7 +355,11 @@ static int set_clr_page_flags(struct x86_mapping_info *info, + snp_set_page_private(__pa(address & PAGE_MASK)); - /* -@@ -2722,7 +2772,7 @@ __latent_entropy struct task_struct *copy_process( - write_lock_irq(&tasklist_lock); - - /* CLONE_PARENT re-uses the old parent */ -- if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { -+ if (clone_flags & (CLONE_PARENT | CLONE_THREAD)) { - p->real_parent = current->real_parent; - p->parent_exec_id = current->parent_exec_id; - if (clone_flags & CLONE_THREAD) -@@ -2786,8 +2836,9 @@ __latent_entropy struct task_struct *copy_process( - * tasklist_lock with adding child to the process tree - * for propagate_has_child_subreaper optimization. - */ -- p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || -- p->real_parent->signal->is_child_subreaper; -+ p->signal->has_child_subreaper = -+ p->real_parent->signal->has_child_subreaper || -+ p->real_parent->signal->is_child_subreaper; - list_add_tail(&p->sibling, &p->real_parent->children); - list_add_tail_rcu(&p->tasks, &init_task.tasks); - attach_pid(p, PIDTYPE_TGID); -@@ -2938,8 +2989,8 @@ struct task_struct * __init fork_idle(int cpu) - */ - struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) - { -- unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| -- CLONE_IO; -+ unsigned long flags = CLONE_FS | CLONE_FILES | CLONE_SIGHAND | -+ CLONE_THREAD | CLONE_IO; - struct kernel_clone_args args = { - .flags = ((lower_32_bits(flags) | CLONE_VM | - CLONE_UNTRACED) & ~CSIGNAL), -@@ -3103,8 +3154,8 @@ SYSCALL_DEFINE0(fork) - SYSCALL_DEFINE0(vfork) - { - struct kernel_clone_args args = { -- .flags = CLONE_VFORK | CLONE_VM, -- .exit_signal = SIGCHLD, -+ .flags = CLONE_VFORK | CLONE_VM, -+ .exit_signal = SIGCHLD, - }; - - return kernel_clone(&args); -@@ -3114,35 +3165,30 @@ SYSCALL_DEFINE0(vfork) - #ifdef __ARCH_WANT_SYS_CLONE - #ifdef CONFIG_CLONE_BACKWARDS - SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, -- int __user *, parent_tidptr, -- unsigned long, tls, -- int __user *, child_tidptr) -+ int __user *, parent_tidptr, unsigned long, tls, int __user *, -+ child_tidptr) - #elif defined(CONFIG_CLONE_BACKWARDS2) - SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, -- int __user *, parent_tidptr, -- int __user *, child_tidptr, -- unsigned long, tls) --#elif defined(CONFIG_CLONE_BACKWARDS3) --SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, -- int, stack_size, -- int __user *, parent_tidptr, -- int __user *, child_tidptr, -+ int __user *, parent_tidptr, int __user *, child_tidptr, - unsigned long, tls) -+#elif defined(CONFIG_CLONE_BACKWARDS3) -+SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, int, -+ stack_size, int __user *, parent_tidptr, int __user *, -+ child_tidptr, unsigned long, tls) - #else - SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, -- int __user *, parent_tidptr, -- int __user *, child_tidptr, -- unsigned long, tls) -+ int __user *, parent_tidptr, int __user *, child_tidptr, -+ unsigned long, tls) - #endif - { - struct kernel_clone_args args = { -- .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), -- .pidfd = parent_tidptr, -- .child_tid = child_tidptr, -- .parent_tid = parent_tidptr, -- .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), -- .stack = newsp, -- .tls = tls, -+ .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), -+ .pidfd = parent_tidptr, -+ .child_tid = child_tidptr, -+ .parent_tid = parent_tidptr, -+ .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), -+ .stack = newsp, -+ .tls = tls, - }; - - return kernel_clone(&args); -@@ -3198,21 +3244,21 @@ noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs, - return -EINVAL; + /* Flush TLB after changing encryption attribute */ ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr3_early(top_level_pgt); ++ #else + write_cr3(top_level_pgt); ++ #endif - *kargs = (struct kernel_clone_args){ -- .flags = args.flags, -- .pidfd = u64_to_user_ptr(args.pidfd), -- .child_tid = u64_to_user_ptr(args.child_tid), -- .parent_tid = u64_to_user_ptr(args.parent_tid), -- .exit_signal = args.exit_signal, -- .stack = args.stack, -- .stack_size = args.stack_size, -- .tls = args.tls, -- .set_tid_size = args.set_tid_size, -- .cgroup = args.cgroup, -+ .flags = args.flags, -+ .pidfd = u64_to_user_ptr(args.pidfd), -+ .child_tid = u64_to_user_ptr(args.child_tid), -+ .parent_tid = u64_to_user_ptr(args.parent_tid), -+ .exit_signal = args.exit_signal, -+ .stack = args.stack, -+ .stack_size = args.stack_size, -+ .tls = args.tls, -+ .set_tid_size = args.set_tid_size, -+ .cgroup = args.cgroup, - }; - - if (args.set_tid && -- copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), -- (kargs->set_tid_size * sizeof(pid_t)))) -+ copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), -+ (kargs->set_tid_size * sizeof(pid_t)))) - return -EFAULT; - - kargs->set_tid = kset_tid; -@@ -3307,7 +3353,8 @@ SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size) + return 0; } - #endif +diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c +index 15354673d3aa..c48af717b52e 100644 +--- a/arch/x86/boot/compressed/pgtable_64.c ++++ b/arch/x86/boot/compressed/pgtable_64.c +@@ -197,7 +197,11 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) + * Move the top level page table out of trampoline memory. + */ + memcpy(pgtable, trampoline_32bit, PAGE_SIZE); ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr3_early((unsigned long)pgtable); ++ #else + native_write_cr3((unsigned long)pgtable); ++ #endif --void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data) -+void walk_process_tree(struct task_struct *top, proc_visitor visitor, -+ void *data) - { - struct task_struct *leader, *parent, *child; - int res; -@@ -3315,8 +3362,8 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data - read_lock(&tasklist_lock); - leader = top = top->group_leader; - down: -- for_each_thread(leader, parent) { -- list_for_each_entry(child, &parent->children, sibling) { -+ for_each_thread (leader, parent) { -+ list_for_each_entry (child, &parent->children, sibling) { - res = visitor(child, data); - if (res) { - if (res < 0) -@@ -3324,8 +3371,7 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data - leader = child; - goto down; - } --up: -- ; -+ up:; - } - } + /* Restore trampoline memory */ + memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); +diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h +index ab97b22ac04a..405689ba6e30 100644 +--- a/arch/x86/include/asm/desc.h ++++ b/arch/x86/include/asm/desc.h +@@ -13,6 +13,10 @@ + #include + #include -@@ -3402,11 +3448,11 @@ void __init proc_caches_init(void) - */ - static int check_unshare_flags(unsigned long unshare_flags) ++#ifdef CONFIG_IEE_SIP ++#include ++#endif ++ + static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info) { -- if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| -- CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| -- CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| -- CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP| -- CLONE_NEWTIME)) -+ if (unshare_flags & -+ ~(CLONE_THREAD | CLONE_FS | CLONE_NEWNS | CLONE_SIGHAND | CLONE_VM | -+ CLONE_FILES | CLONE_SYSVSEM | CLONE_NEWUTS | CLONE_NEWIPC | -+ CLONE_NEWNET | CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWCGROUP | -+ CLONE_NEWTIME)) - return -EINVAL; - /* - * Not implemented, but pretend it works if there is nothing -@@ -3516,7 +3562,7 @@ int ksys_unshare(unsigned long unshare_flags) - * to a new ipc namespace, the semaphore arrays from the old - * namespace are unreachable. - */ -- if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) -+ if (unshare_flags & (CLONE_NEWIPC | CLONE_SYSVSEM)) - do_sysvsem = 1; - err = unshare_fs(unshare_flags, &new_fs); - if (err) -@@ -3527,8 +3573,8 @@ int ksys_unshare(unsigned long unshare_flags) - err = unshare_userns(unshare_flags, &new_cred); - if (err) - goto bad_unshare_cleanup_fd; -- err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, -- new_cred, new_fs); -+ err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_cred, -+ new_fs); - if (err) - goto bad_unshare_cleanup_cred; - -@@ -3625,8 +3671,8 @@ int unshare_files(void) - return 0; + desc->limit0 = info->limit & 0x0ffff; +@@ -210,9 +214,20 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) + asm volatile("lgdt %0"::"m" (*dtr)); } --int sysctl_max_threads(struct ctl_table *table, int write, -- void *buffer, size_t *lenp, loff_t *ppos) -+int sysctl_max_threads(struct ctl_table *table, int write, void *buffer, -+ size_t *lenp, loff_t *ppos) - { - struct ctl_table t; - int ret; -diff --git a/kernel/groups.c b/kernel/groups.c -index 9b43da22647d..8045812e8a3c 100644 ---- a/kernel/groups.c -+++ b/kernel/groups.c -@@ -11,6 +11,9 @@ - #include - #include - #include -+#ifdef CONFIG_CREDP -+#include ++#ifdef CONFIG_IEE_SIP ++static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) ++{ ++ asm volatile("lidt %0"::"m" (*dtr)); ++} +#endif - - struct group_info *groups_alloc(int gidsetsize) - { -@@ -119,7 +122,11 @@ void set_groups(struct cred *new, struct group_info *group_info) ++ + static __always_inline void native_load_idt(const struct desc_ptr *dtr) { - put_group_info(new->group_info); - get_group_info(group_info); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_group_info(new,group_info); ++ #ifdef CONFIG_IEE_SIP ++ iee_load_idt((void *)dtr); + #else - new->group_info = group_info; + asm volatile("lidt %0"::"m" (*dtr)); + #endif } - EXPORT_SYMBOL(set_groups); -diff --git a/kernel/kthread.c b/kernel/kthread.c -index 980e6b325b7d..d83342b67608 100644 ---- a/kernel/kthread.c -+++ b/kernel/kthread.c -@@ -30,6 +30,11 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#include + static inline void native_store_gdt(struct desc_ptr *dtr) +diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h +new file mode 100644 +index 000000000000..170ecc7b7ac3 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/haoc-def.h +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_HAOC_DEF_H ++#define _LINUX_HAOC_DEF_H ++ ++enum { ++ IEE_OP_MEMCPY, ++ IEE_OP_MEMSET, ++ IEE_OP_SET_FREEPTR, ++ IEE_OP_TEST_CLEAR_BIT, ++#ifdef CONFIG_IEE_PTRP ++ IEE_OP_SET_TOKEN_PGD, ++ IEE_OP_INVALIDATE_TOKEN, ++ IEE_OP_VALIDATE_TOKEN, ++#endif ++#ifdef CONFIG_PTP ++#ifdef CONFIG_IEE_PTRP ++ IEE_OP_UNSET_TOKEN, ++ IEE_OP_SET_TOKEN, ++#endif ++ IEE_OP_SET_PTE, ++ IEE_OP_SET_PMD, ++ IEE_OP_SET_PUD, ++ IEE_OP_SET_P4D, ++ IEE_OP_SET_PGD, ++ IEE_OP_SET_PTE_TEXT_POKE, +#endif ++ IEE_FLAG_END ++}; + - - static DEFINE_SPINLOCK(kthread_create_lock); - static LIST_HEAD(kthread_create_list); -@@ -1457,6 +1462,9 @@ void kthread_use_mm(struct mm_struct *mm) - tsk->active_mm = mm; - tsk->mm = mm; - membarrier_update_current_mm(mm); -+ #ifdef CONFIG_IEE -+ iee_set_token_pgd(tsk, mm->pgd); -+ #endif - switch_mm_irqs_off(active_mm, mm, tsk); - local_irq_enable(); - task_unlock(tsk); -@@ -1501,7 +1509,11 @@ void kthread_unuse_mm(struct mm_struct *mm) - local_irq_disable(); - tsk->mm = NULL; - membarrier_update_current_mm(NULL); -+ #ifdef CONFIG_IEE -+ iee_set_token_pgd(tsk, NULL); -+ #endif - mmgrab_lazy_tlb(mm); -+ - /* active_mm is still 'mm' */ - enter_lazy_tlb(mm, tsk); - local_irq_enable(); -diff --git a/kernel/module/main.c b/kernel/module/main.c -index 14a51af2fbea..f810f005d90a 100644 ---- a/kernel/module/main.c -+++ b/kernel/module/main.c -@@ -63,6 +63,10 @@ - #define CREATE_TRACE_POINTS - #include - -+#ifdef CONFIG_KOI -+#include "asm/koi.h" ++#ifdef CONFIG_IEE_SIP ++#define IEE_SIP_TEST 0 ++#define IEE_WRITE_CR0 1 ++#define IEE_WRITE_CR3 2 ++#define IEE_WRITE_CR4 3 ++#define IEE_LOAD_IDT 4 ++#endif ++ ++#endif +diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h +new file mode 100644 +index 000000000000..3e187069a5f2 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/haoc.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_HAOC_H ++#define _LINUX_HAOC_H ++ ++#include ++#include ++ ++void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); ++void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); ++void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); ++unsigned long _iee_test_and_clear_bit(unsigned long __unused, ++ long nr, volatile unsigned long *addr); ++#ifdef CONFIG_IEE_PTRP ++void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, ++ pgd_t *pgd); ++void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); ++void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); ++ ++#ifdef CONFIG_PTP ++#ifdef CONFIG_IEE_PTRP ++void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, ++ pte_t *token_page_ptep, unsigned long token, unsigned int order); ++void _iee_set_token(unsigned long __unused, pte_t *token_ptep, ++ pte_t *token_page_ptep, unsigned long token_page, unsigned int order); ++#endif ++void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte); ++void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd); ++void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud); ++void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d); ++void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd); ++void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte); ++#endif ++#endif ++#endif +diff --git a/arch/x86/include/asm/haoc/iee-access.h b/arch/x86/include/asm/haoc/iee-access.h +new file mode 100644 +index 000000000000..8a01feadabbd +--- /dev/null ++++ b/arch/x86/include/asm/haoc/iee-access.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_ACCESS_H ++#define _LINUX_IEE_ACCESS_H ++ ++#include ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++static inline void iee_memcpy(void *dst, const void *src, size_t n) ++{ ++ iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); ++} ++ ++static inline void iee_memset(void *ptr, int data, size_t n) ++{ ++ iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); ++} ++ ++static inline void iee_set_freeptr(void **pptr, void *ptr) ++{ ++ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); ++} ++ ++static inline unsigned long iee_test_and_clear_bit(long nr, volatile unsigned long *addr) ++{ ++ return iee_rw_gate(IEE_OP_TEST_CLEAR_BIT, nr, addr); ++} ++ ++#endif +diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h +new file mode 100644 +index 000000000000..a25cd4d2fc90 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/iee-func.h +@@ -0,0 +1,11 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_FUNC_H ++#define _LINUX_IEE_FUNC_H ++ ++#include ++extern void set_iee_page(unsigned long addr, unsigned int order); ++extern void unset_iee_page(unsigned long addr, unsigned int order); ++extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); ++extern unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order); ++ ++#endif /* _LINUX_IEE_FUNC_H */ +diff --git a/arch/x86/include/asm/haoc/iee-si.h b/arch/x86/include/asm/haoc/iee-si.h +new file mode 100644 +index 000000000000..98251319acc6 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/iee-si.h +@@ -0,0 +1,43 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_SI_H ++#define _LINUX_IEE_SI_H ++#include ++#include ++ ++#define __iee_si_code __section(".iee.si_text") ++#define __iee_si_data __section(".iee.si_data") ++ ++extern const unsigned long cr4_pinned_mask; ++extern struct static_key_false cr_pinning; ++extern unsigned long cr4_pinned_bits; ++ ++extern unsigned long __iee_si_text_start[]; ++extern unsigned long __iee_si_text_end[]; ++extern unsigned long __iee_si_data_start[]; ++extern unsigned long __iee_si_data_end[]; ++extern void iee_sip_init(void); ++ ++extern void iee_rwx_gate(int flag, ...); ++ ++static inline void iee_sip_test(void) ++{ ++ iee_rwx_gate(IEE_SIP_TEST); ++} ++static inline void iee_write_cr0(unsigned long val) ++{ ++ iee_rwx_gate(IEE_WRITE_CR0, val); ++} ++static inline void iee_write_cr3(unsigned long val) ++{ ++ iee_rwx_gate(IEE_WRITE_CR3, val); ++} ++static inline void iee_write_cr4(unsigned long val) ++{ ++ iee_rwx_gate(IEE_WRITE_CR4, val); ++} ++static inline void iee_load_idt(void *ptr) ++{ ++ iee_rwx_gate(IEE_LOAD_IDT, ptr); ++} ++ ++#endif +diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h +new file mode 100644 +index 000000000000..d9b521b26a08 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/iee-token.h +@@ -0,0 +1,53 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_TOKEN_H ++#define _LINUX_IEE_TOKEN_H ++#include ++#include ++#include ++#include ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++extern struct kmem_cache *task_struct_cachep; ++ ++extern void iee_set_token_page_valid(unsigned long token, unsigned long new, ++ unsigned int order); ++extern void iee_set_token_page_invalid(unsigned long token_addr, ++ unsigned long __unused, unsigned int order); ++extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, ++ struct slab *slab, unsigned int order); ++ ++struct task_token { ++ pgd_t *pgd; ++ bool valid; ++}; ++ ++static inline void iee_verify_token_pgd(struct task_struct *tsk) ++{ ++ struct task_token *token; ++ ++ if (tsk == &init_task) ++ return; ++ ++ token = (struct task_token *)__addr_to_iee(tsk); ++ if (token->pgd != tsk->mm->pgd) ++ panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", ++ (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); ++} ++ ++static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) ++{ ++ iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); ++} ++ ++static inline void iee_invalidate_token(struct task_struct *tsk) ++{ ++ iee_rw_gate(IEE_OP_INVALIDATE_TOKEN, tsk); ++} ++ ++static inline void iee_validate_token(struct task_struct *tsk) ++{ ++ iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); ++} ++ ++#endif +diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h +new file mode 100644 +index 000000000000..2cc319929415 +--- /dev/null ++++ b/arch/x86/include/asm/haoc/iee.h +@@ -0,0 +1,26 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_IEE_H ++#define _LINUX_IEE_H ++ ++#include ++ ++extern unsigned long IEE_OFFSET; ++#define __iee_pa(x) (__pa(x - IEE_OFFSET)) ++#define __phys_to_iee(x) ((void *)(__va(x) + IEE_OFFSET)) ++#define __page_to_phys(x) (page_to_pfn(x) << PAGE_SHIFT) ++#define __page_to_iee(x) ((unsigned long)(__phys_to_iee(__page_to_phys(x)))) ++#define __slab_to_iee(x) (__page_to_iee(folio_page(slab_folio(x), 0))) ++#define __addr_to_iee(x) (__phys_to_iee(__pa(x))) ++ ++#define IEE_DATA_ORDER (PMD_SHIFT - PAGE_SHIFT) ++#define IEE_STACK_ORDER 0 ++struct iee_stack { ++ void *stack; ++}; ++ ++DECLARE_PER_CPU(struct iee_stack, iee_stacks); ++ ++extern void *alloc_low_pages(unsigned int num); ++extern void iee_init(void); ++extern bool iee_init_done; +#endif +diff --git a/arch/x86/include/asm/haoc/ptp.h b/arch/x86/include/asm/haoc/ptp.h +new file mode 100644 +index 000000000000..e435d7fd608c +--- /dev/null ++++ b/arch/x86/include/asm/haoc/ptp.h +@@ -0,0 +1,64 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_PTP_H ++#define _LINUX_PTP_H + - /* - * Mutex protects: - * 1) List of modules (also safely readable with preempt_disable), -@@ -769,7 +773,11 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user, - /* Store the name and taints of the last unloaded module for diagnostic purposes */ - strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); - strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints)); -- -+#ifdef CONFIG_KOI -+ koi_destroy_pagetable(mod); -+ -+ printk(KERN_ERR "pagetable destroyed\n"); -+#endif - free_module(mod); - /* someone could wait for the module in add_unformed_module() */ - wake_up_all(&module_wq); -@@ -2590,6 +2598,11 @@ static noinline int do_init_module(struct module *mod) - module_enable_ro(mod, true); - mod_tree_remove_init(mod); - module_arch_freeing_init(mod); -+#ifdef CONFIG_KOI -+ for_class_mod_mem_type(type, init) { -+ koi_unmap_mem(mod, (unsigned long)mod->mem[type].base, mod->mem[type].size); -+ } -+#endif - for_class_mod_mem_type(type, init) { - mod->mem[type].base = NULL; - mod->mem[type].size = 0; -@@ -2636,6 +2649,11 @@ static noinline int do_init_module(struct module *mod) - MODULE_STATE_GOING, mod); - klp_module_going(mod); - ftrace_release_mod(mod); -+#ifdef CONFIG_KOI -+ koi_destroy_pagetable(mod); -+ -+ printk(KERN_ERR "[KOI] pagetable for module: %s destroyed\n", mod->name); -+#endif - free_module(mod); - wake_up_all(&module_wq); - -@@ -2840,6 +2858,32 @@ static int early_mod_check(struct load_info *info, int flags) - return err; - } - -+#ifdef CONFIG_KOI -+void shared_variable_dfs(struct mm_struct *ko_mm, pgd_t *ko_pg_dir, struct shared_variable_descriptor *desc, unsigned int *get_val_id, unsigned int *from_id_get_ad, unsigned int i, unsigned long addr) ++#include ++#include ++ ++extern unsigned long long ptp_rw_gate(int flag, ...); ++ ++static inline void ptp_set_pte(pte_t *ptep, pte_t pte) +{ -+ unsigned int j; -+ unsigned int new_desc_num; -+ unsigned long *ptr; -+ for(j=0; iasync_probe_requested = async_probe; -+#ifdef CONFIG_KOI -+ /* -+ * find the shared_vars_section and get the shared var list -+ */ -+ ndx = find_sec(info, ".shared_vars_section"); -+ if (ndx){ -+ koi_create_pagetable(mod); -+ target = NULL; -+ rcu_read_lock(); -+ hash_for_each_possible_rcu (koi_mem_htbl, target, node, -+ (unsigned long)mod) { -+ if (target->mod == mod) { -+ break; -+ } -+ } -+ rcu_read_unlock(); - -+ if (target == NULL) { -+ printk("[Error] shared_vars_section detected, But mem node for module: %s not found\n", mod->name); -+ koi_destroy_pagetable(mod); -+ goto bug_cleanup; -+ } ++static inline void ptp_set_pmd(pmd_t *pmdp, pmd_t pmd) ++{ ++ compiletime_assert_rwonce_type(*pmdp); ++ ptp_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); ++} + -+ ko_mm = target->ko_mm; -+ ko_pg_dir = target->pgdp; -+ -+ desc = (struct shared_variable_descriptor *)info->sechdrs[ndx].sh_addr; -+ -+ num_desc = info->sechdrs[ndx].sh_size / sizeof(struct shared_variable_descriptor); -+ get_val_id = kmalloc(DRIVER_ISOLATION_MAX_VAL, GFP_KERNEL); -+ from_id_get_ad = kmalloc(DRIVER_ISOLATION_MAX_VAL, GFP_KERNEL); -+ -+ for (j = 0; j < num_desc; j++) { -+ unsigned int desc_id = desc[j].id; -+ get_val_id[j] = desc_id; -+ from_id_get_ad[desc_id] = j; -+ } -+ -+ for (i = 0; i < num_desc; i++) { -+ if (desc[i].type == 0) { -+ addr = kallsyms_lookup_name(desc[i].name); -+ koi_copy_pagetable(ko_mm, ko_pg_dir, addr & PAGE_MASK, (addr + desc[i].size + PAGE_SIZE) & PAGE_MASK, 0); -+ shared_variable_dfs(ko_mm, ko_pg_dir, desc, get_val_id, from_id_get_ad, i, addr); -+ } -+ } -+ kfree(get_val_id); -+ kfree(from_id_get_ad); -+ } -+#endif - /* Module is ready to execute: parsing args may do that. */ - after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, - -32768, 32767, mod, -diff --git a/kernel/sys.c b/kernel/sys.c -index 44b575990333..fbc47f83af50 100644 ---- a/kernel/sys.c -+++ b/kernel/sys.c -@@ -75,6 +75,10 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif ++static inline void ptp_set_pud(pud_t *pudp, pud_t pud) ++{ ++ compiletime_assert_rwonce_type(*pudp); ++ ptp_rw_gate(IEE_OP_SET_PUD, pudp, pud); ++} + - #include "uid16.h" - - #ifndef SET_UNALIGN_CTL -@@ -395,7 +399,11 @@ long __sys_setregid(gid_t rgid, gid_t egid) - if (gid_eq(old->gid, krgid) || - gid_eq(old->egid, krgid) || - ns_capable_setid(old->user_ns, CAP_SETGID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_gid(new,krgid); -+ #else - new->gid = krgid; -+ #endif - else - goto error; - } -@@ -404,15 +412,27 @@ long __sys_setregid(gid_t rgid, gid_t egid) - gid_eq(old->egid, kegid) || - gid_eq(old->sgid, kegid) || - ns_capable_setid(old->user_ns, CAP_SETGID)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_egid(new,kegid); -+ #else - new->egid = kegid; -+ #endif - else - goto error; - } - - if (rgid != (gid_t) -1 || - (egid != (gid_t) -1 && !gid_eq(kegid, old->gid))) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_sgid(new,new->egid); -+ #else - new->sgid = new->egid; -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsgid(new,new->egid); -+ #else - new->fsgid = new->egid; -+ #endif - - retval = security_task_fix_setgid(new, old, LSM_SETID_RE); - if (retval < 0) -@@ -454,9 +474,25 @@ long __sys_setgid(gid_t gid) - - retval = -EPERM; - if (ns_capable_setid(old->user_ns, CAP_SETGID)) -+ #ifdef CONFIG_CREDP -+ { -+ iee_set_cred_fsgid(new,kgid); -+ iee_set_cred_sgid(new,kgid); -+ iee_set_cred_egid(new,kgid); -+ iee_set_cred_gid(new,kgid); -+ } -+ #else - new->gid = new->egid = new->sgid = new->fsgid = kgid; -+ #endif - else if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->sgid)) -+ #ifdef CONFIG_CREDP -+ { -+ iee_set_cred_fsgid(new,kgid); -+ iee_set_cred_egid(new,kgid); -+ } -+ #else - new->egid = new->fsgid = kgid; -+ #endif - else - goto error; - -@@ -488,7 +524,11 @@ static int set_user(struct cred *new) - return -EAGAIN; - - free_uid(new->user); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_user(new,new_user); -+ #else - new->user = new_user; -+ #endif - return 0; - } - -@@ -549,7 +589,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) - - retval = -EPERM; - if (ruid != (uid_t) -1) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_uid(new,kruid); -+ #else - new->uid = kruid; -+ #endif - if (!uid_eq(old->uid, kruid) && - !uid_eq(old->euid, kruid) && - !ns_capable_setid(old->user_ns, CAP_SETUID)) -@@ -557,7 +601,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) - } - - if (euid != (uid_t) -1) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_euid(new,keuid); -+ #else - new->euid = keuid; -+ #endif - if (!uid_eq(old->uid, keuid) && - !uid_eq(old->euid, keuid) && - !uid_eq(old->suid, keuid) && -@@ -572,8 +620,16 @@ long __sys_setreuid(uid_t ruid, uid_t euid) - } - if (ruid != (uid_t) -1 || - (euid != (uid_t) -1 && !uid_eq(keuid, old->uid))) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_suid(new,new->euid); -+ #else - new->suid = new->euid; -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,new->euid); -+ #else - new->fsuid = new->euid; -+ #endif - - retval = security_task_fix_setuid(new, old, LSM_SETID_RE); - if (retval < 0) -@@ -626,7 +682,12 @@ long __sys_setuid(uid_t uid) - - retval = -EPERM; - if (ns_capable_setid(old->user_ns, CAP_SETUID)) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_uid(new,kuid); -+ iee_set_cred_suid(new,kuid); -+ #else - new->suid = new->uid = kuid; -+ #endif - if (!uid_eq(kuid, old->uid)) { - retval = set_user(new); - if (retval < 0) -@@ -636,7 +697,12 @@ long __sys_setuid(uid_t uid) - goto error; - } - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_euid(new,kuid); -+ iee_set_cred_fsuid(new,kuid); -+ #else - new->fsuid = new->euid = kuid; -+ #endif - - retval = security_task_fix_setuid(new, old, LSM_SETID_ID); - if (retval < 0) -@@ -710,7 +776,11 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) - return -ENOMEM; - - if (ruid != (uid_t) -1) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_uid(new,kruid); -+ #else - new->uid = kruid; -+ #endif - if (!uid_eq(kruid, old->uid)) { - retval = set_user(new); - if (retval < 0) -@@ -718,10 +788,22 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) - } - } - if (euid != (uid_t) -1) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_euid(new,keuid); -+ #else - new->euid = keuid; -+ #endif - if (suid != (uid_t) -1) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_suid(new,ksuid); -+ #else - new->suid = ksuid; -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,new->euid); -+ #else - new->fsuid = new->euid; -+ #endif - - retval = security_task_fix_setuid(new, old, LSM_SETID_RES); - if (retval < 0) -@@ -810,12 +892,29 @@ long __sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid) - return -ENOMEM; - - if (rgid != (gid_t) -1) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_gid(new,krgid); -+ #else - new->gid = krgid; -+ #endif - if (egid != (gid_t) -1) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_egid(new,kegid); -+ #else - new->egid = kegid; -+ #endif - if (sgid != (gid_t) -1) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_sgid(new,ksgid); -+ #else - new->sgid = ksgid; -+ #endif ++static inline void ptp_set_p4d(p4d_t *p4dp, p4d_t p4d) ++{ ++ compiletime_assert_rwonce_type(*p4dp); ++ ptp_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); ++} + -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsgid(new,new->egid); -+ #else - new->fsgid = new->egid; -+ #endif - - retval = security_task_fix_setgid(new, old, LSM_SETID_RES); - if (retval < 0) -@@ -882,7 +981,11 @@ long __sys_setfsuid(uid_t uid) - uid_eq(kuid, old->suid) || uid_eq(kuid, old->fsuid) || - ns_capable_setid(old->user_ns, CAP_SETUID)) { - if (!uid_eq(kuid, old->fsuid)) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,kuid); -+ #else - new->fsuid = kuid; -+ #endif - if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0) - goto change_okay; - } -@@ -926,7 +1029,11 @@ long __sys_setfsgid(gid_t gid) - gid_eq(kgid, old->sgid) || gid_eq(kgid, old->fsgid) || - ns_capable_setid(old->user_ns, CAP_SETGID)) { - if (!gid_eq(kgid, old->fsgid)) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsgid(new,kgid); -+ #else - new->fsgid = kgid; -+ #endif - if (security_task_fix_setgid(new,old,LSM_SETID_FS) == 0) - goto change_okay; - } -diff --git a/kernel/umh.c b/kernel/umh.c -index 1b13c5d34624..32f5c88e10bf 100644 ---- a/kernel/umh.c -+++ b/kernel/umh.c -@@ -32,6 +32,10 @@ - - #include - -+#ifdef CONFIG_CREDP -+#include -+#endif ++static inline void ptp_set_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ compiletime_assert_rwonce_type(*pgdp); ++ ptp_rw_gate(IEE_OP_SET_PGD, pgdp, pgd); ++} + - static kernel_cap_t usermodehelper_bset = CAP_FULL_SET; - static kernel_cap_t usermodehelper_inheritable = CAP_FULL_SET; - static DEFINE_SPINLOCK(umh_sysctl_lock); -@@ -91,9 +95,15 @@ static int call_usermodehelper_exec_async(void *data) - goto out; - - spin_lock(&umh_sysctl_lock); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_bset(new,cap_intersect(usermodehelper_bset, new->cap_bset)); -+ iee_set_cred_cap_inheritable(new,cap_intersect(usermodehelper_inheritable, -+ new->cap_inheritable)); -+ #else - new->cap_bset = cap_intersect(usermodehelper_bset, new->cap_bset); - new->cap_inheritable = cap_intersect(usermodehelper_inheritable, - new->cap_inheritable); -+ #endif - spin_unlock(&umh_sysctl_lock); ++static inline void ptp_set_pte_text_poke(pte_t *ptep, pte_t pte) ++{ ++ compiletime_assert_rwonce_type(*ptep); ++ ptp_rw_gate(IEE_OP_SET_PTE_TEXT_POKE, ptep, pte); ++} ++ ++extern pgprotval_t ptp_xchg(pgprotval_t *pgprotp, pgprotval_t pgprotval); ++extern pgprotval_t ptp_try_cmpxchg(pgprotval_t *pgprotp, ++ pgprotval_t old_pgprot, pgprotval_t new_pgprotval); ++extern void ptp_mark_all_pgtable_ro(void); ++extern struct pg_cache pgd_cache; ++ ++#include ++struct iee_disable_t { ++ /* Writable but considered safe to expose */ ++ unsigned long disabled_cnt; ++}; ++ ++DECLARE_PER_CPU(struct iee_disable_t, iee_disables); ++extern void ptp_iee_disable_init(void); ++extern void ptp_disable_iee(unsigned long *reg); ++extern void ptp_enable_iee(unsigned long reg); ++extern void ptp_context_enable_iee(int *disabled_cnt, unsigned long *reg); ++extern void ptp_context_restore_iee(int disabled_cnt, unsigned long reg); ++#endif +diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h +index c7ec5bb88334..0c8e93652ce3 100644 +--- a/arch/x86/include/asm/pgalloc.h ++++ b/arch/x86/include/asm/pgalloc.h +@@ -9,6 +9,9 @@ + #define __HAVE_ARCH_PTE_ALLOC_ONE + #define __HAVE_ARCH_PGD_FREE + #include ++#ifdef CONFIG_PTP ++#include ++#endif - if (sub_info->init) { -diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c -index 1d8e47bed3f1..9f1921025539 100644 ---- a/kernel/user_namespace.c -+++ b/kernel/user_namespace.c -@@ -22,6 +22,10 @@ - #include - #include + static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } -+#ifdef CONFIG_CREDP -+#include -+#endif -+ - static struct kmem_cache *user_ns_cachep __read_mostly; - static DEFINE_MUTEX(userns_state_mutex); +@@ -153,7 +156,11 @@ static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) -@@ -45,6 +49,19 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) - /* Start with the same capabilities as init but useless for doing - * anything as the capabilities are bound to the new user namespace. - */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_securebits(cred,SECUREBITS_DEFAULT); -+ iee_set_cred_cap_inheritable(cred,CAP_EMPTY_SET); -+ iee_set_cred_cap_permitted(cred,CAP_FULL_SET); -+ iee_set_cred_cap_effective(cred,CAP_FULL_SET); -+ iee_set_cred_cap_ambient(cred,CAP_EMPTY_SET); -+ iee_set_cred_cap_bset(cred,CAP_FULL_SET); -+#ifdef CONFIG_KEYS -+ key_put(cred->request_key_auth); -+ iee_set_cred_request_key_auth(cred,NULL); + if (mm == &init_mm) + gfp &= ~__GFP_ACCOUNT; ++#ifdef CONFIG_PTP ++ return (p4d_t *)ptp_pg_alloc(&pg_cache, GFP_KERNEL | __GFP_ZERO); ++#else + return (p4d_t *)get_zeroed_page(gfp); +#endif -+ iee_set_cred_user_ns(cred,user_ns); -+ #else - cred->securebits = SECUREBITS_DEFAULT; - cred->cap_inheritable = CAP_EMPTY_SET; - cred->cap_permitted = CAP_FULL_SET; -@@ -57,6 +74,7 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) - #endif - /* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */ - cred->user_ns = user_ns; -+ #endif } - static unsigned long enforced_nproc_rlimit(void) -diff --git a/lib/digsig.c b/lib/digsig.c -index 04b5e55ed95f..0a10a459bdaa 100644 ---- a/lib/digsig.c -+++ b/lib/digsig.c -@@ -81,7 +81,11 @@ static int digsig_verify_rsa(struct key *key, - const struct user_key_payload *ukp; - struct pubkey_hdr *pkh; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - ukp = user_key_payload_locked(key); - - if (!ukp) { -@@ -176,7 +180,11 @@ static int digsig_verify_rsa(struct key *key, - while (--i >= 0) - mpi_free(pkey[i]); - err1: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - - return err; - } -diff --git a/mm/Kconfig b/mm/Kconfig -index 782c43f08e8f..7706340f189c 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -530,6 +530,11 @@ config NUMA_KEEP_MEMINFO - config MEMORY_ISOLATION - bool - -+# Configs for pgtable isolation -+config PTP -+ depends on IEE -+ def_bool y -+ - # IORESOURCE_SYSTEM_RAM regions in the kernel resource tree that are marked - # IORESOURCE_EXCLUSIVE cannot be mapped to user space, for example, via - # /dev/mem. -diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c -index d25d99cb5f2b..2ea51f559d4e 100644 ---- a/mm/damon/ops-common.c -+++ b/mm/damon/ops-common.c -@@ -44,6 +44,7 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr - if (!folio) + static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) +@@ -162,7 +169,11 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) return; -+ - if (ptep_clear_young_notify(vma, addr, pte)) - folio_set_young(folio); - -diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c -index 68af76ca8bc9..2789784777dd 100644 ---- a/mm/debug_vm_pgtable.c -+++ b/mm/debug_vm_pgtable.c -@@ -437,7 +437,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args) - * X86 defined pmd_set_huge() verifies that the given - * PMD is not a populated non-leaf entry. - */ -+ #ifdef CONFIG_PTP -+ set_pmd(args->pmdp, __pmd(0)); -+ #else - WRITE_ONCE(*args->pmdp, __pmd(0)); -+ #endif - WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); - WARN_ON(!pmd_clear_huge(args->pmdp)); - pmd = READ_ONCE(*args->pmdp); -@@ -457,7 +461,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args) - * X86 defined pud_set_huge() verifies that the given - * PUD is not a populated non-leaf entry. - */ -+ #ifdef CONFIG_PTP -+ set_pud(args->pudp, __pud(0)); -+ #else - WRITE_ONCE(*args->pudp, __pud(0)); -+ #endif - WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); - WARN_ON(!pud_clear_huge(args->pudp)); - pud = READ_ONCE(*args->pudp); -@@ -496,6 +504,12 @@ static void __init pud_clear_tests(struct pgtable_debug_args *args) - - pr_debug("Validating PUD clear\n"); - WARN_ON(pud_none(pud)); -+ pud = __pud(pud_val(pud) | RANDOM_ORVALUE); -+ #ifdef CONFIG_PTP -+ set_pud(args->pudp, pud); -+ #else -+ WRITE_ONCE(*args->pudp, pud); -+ #endif - pud_clear(args->pudp); - pud = READ_ONCE(*args->pudp); - WARN_ON(!pud_none(pud)); -@@ -532,6 +546,12 @@ static void __init p4d_clear_tests(struct pgtable_debug_args *args) + BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); ++#ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, p4d); ++#else + free_page((unsigned long)p4d); ++#endif + } - pr_debug("Validating P4D clear\n"); - WARN_ON(p4d_none(p4d)); -+ p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE); -+ #ifdef CONFIG_PTP -+ set_p4d(args->p4dp, p4d); -+ #else -+ WRITE_ONCE(*args->p4dp, p4d); -+ #endif - p4d_clear(args->p4dp); - p4d = READ_ONCE(*args->p4dp); - WARN_ON(!p4d_none(p4d)); -@@ -565,6 +585,12 @@ static void __init pgd_clear_tests(struct pgtable_debug_args *args) - - pr_debug("Validating PGD clear\n"); - WARN_ON(pgd_none(pgd)); -+ pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE); -+ #ifdef CONFIG_PTP -+ set_pgd(args->pgdp, pgd); -+ #else -+ WRITE_ONCE(*args->pgdp, pgd); -+ #endif - pgd_clear(args->pgdp); - pgd = READ_ONCE(*args->pgdp); - WARN_ON(!pgd_none(pgd)); -@@ -630,6 +656,12 @@ static void __init pmd_clear_tests(struct pgtable_debug_args *args) - - pr_debug("Validating PMD clear\n"); - WARN_ON(pmd_none(pmd)); -+ pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE); -+ #ifdef CONFIG_PTP -+ set_pmd(args->pmdp, pmd); -+ #else -+ WRITE_ONCE(*args->pmdp, pmd); -+ #endif - pmd_clear(args->pmdp); - pmd = READ_ONCE(*args->pmdp); - WARN_ON(!pmd_none(pmd)); -diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c -index ce06b2884789..a039c7a50ec5 100644 ---- a/mm/early_ioremap.c -+++ b/mm/early_ioremap.c -@@ -147,7 +147,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) - if (after_paging_init) - __late_set_fixmap(idx, phys_addr, prot); - else -+ #ifdef CONFIG_PTP -+ __iee_set_fixmap_pre_init(idx, phys_addr, prot); -+ #else - __early_set_fixmap(idx, phys_addr, prot); -+ #endif - phys_addr += PAGE_SIZE; - --idx; - --nrpages; -@@ -199,13 +203,66 @@ void __init early_iounmap(void __iomem *addr, unsigned long size) - if (after_paging_init) - __late_clear_fixmap(idx); - else -+ #ifdef CONFIG_PTP -+ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR); -+ #else - __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); -+ #endif - --idx; - --nrpages; - } - prev_map[slot] = NULL; + extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); +diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h +index c170ca166e78..89cddc2a2996 100644 +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -23,6 +23,10 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#include ++#endif + + extern pgd_t early_top_pgt[PTRS_PER_PGD]; + bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); +@@ -1326,9 +1330,15 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, + pte_t old_pte, new_pte; + + old_pte = READ_ONCE(*ptep); ++#ifdef CONFIG_PTP ++ do { ++ new_pte = pte_wrprotect(old_pte); ++ } while (!ptp_try_cmpxchg((long *)ptep, pte_val(old_pte), pte_val(new_pte))); ++#else + do { + new_pte = pte_wrprotect(old_pte); + } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte)); ++#endif } + #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) +@@ -1388,9 +1398,15 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, + pmd_t old_pmd, new_pmd; + + old_pmd = READ_ONCE(*pmdp); +#ifdef CONFIG_PTP -+void __init early_iounmap_after_init(void __iomem *addr, unsigned long size) -+{ -+ unsigned long virt_addr; -+ unsigned long offset; -+ unsigned int nrpages; -+ enum fixed_addresses idx; -+ int i, slot; -+ -+ slot = -1; -+ for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { -+ if (prev_map[i] == addr) { -+ slot = i; -+ break; -+ } -+ } -+ -+ if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n", -+ addr, size)) -+ return; -+ -+ if (WARN(prev_size[slot] != size, -+ "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", -+ addr, size, slot, prev_size[slot])) -+ return; -+ -+ WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n", -+ addr, size, slot); -+ -+ virt_addr = (unsigned long)addr; -+ if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN))) -+ return; -+ -+ offset = offset_in_page(virt_addr); -+ nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; -+ -+ idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; -+ while (nrpages > 0) { -+ if (after_paging_init) -+ __late_clear_fixmap(idx); -+ else -+ __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); -+ --idx; -+ --nrpages; -+ } -+ prev_map[slot] = NULL; -+} ++ do { ++ new_pmd = pmd_wrprotect(old_pmd); ++ } while (!ptp_try_cmpxchg((long *)pmdp, pmd_val(old_pmd), pmd_val(new_pmd))); ++#else + do { + new_pmd = pmd_wrprotect(old_pmd); + } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd)); +#endif -+ - /* Remap an IO device */ - void __init __iomem * - early_ioremap(resource_size_t phys_addr, unsigned long size) -diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index 3c58e95a33eb..404e5ee540d3 100644 ---- a/mm/huge_memory.c -+++ b/mm/huge_memory.c -@@ -40,6 +40,10 @@ - #include - #include + } + #ifndef pmdp_establish +@@ -1400,10 +1416,18 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + { + page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + if (IS_ENABLED(CONFIG_SMP)) { +#ifdef CONFIG_PTP -+#include ++ return native_make_pmd(ptp_xchg((pgprotval_t *)pmdp, pmd_val(pmd))); ++#else + return xchg(pmdp, pmd); +#endif -+ - #include - #include - #include "internal.h" -@@ -2730,8 +2734,17 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, - old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); + } else { + pmd_t old = *pmdp; ++#ifdef CONFIG_PTP ++ set_pmd(pmdp, pmd); ++#else + WRITE_ONCE(*pmdp, pmd); ++#endif + return old; + } + } +@@ -1491,14 +1515,23 @@ static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) + */ + static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) + { ++#ifdef CONFIG_PTP ++ iee_memcpy(dst, src, count * sizeof(pgd_t)); ++#else + memcpy(dst, src, count * sizeof(pgd_t)); ++#endif + #ifdef CONFIG_PAGE_TABLE_ISOLATION + if (!static_cpu_has(X86_FEATURE_PTI)) + return; + /* Clone the user space pgd as well */ ++#ifdef CONFIG_PTP ++ iee_memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), ++ count * sizeof(pgd_t)); ++#else + memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src), + count * sizeof(pgd_t)); + #endif ++#endif + } - pgtable = pgtable_trans_huge_withdraw(mm, pmd); -+ #ifdef CONFIG_PTP -+ #ifdef CONFIG_X86_64 -+ unsigned long pfn = page_to_pfn(pgtable); -+ paravirt_alloc_pte(mm, pfn); -+ _pmd = __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE); -+ #else -+ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE); -+ #endif -+ #else - pmd_populate(mm, &_pmd, pgtable); -- -+ #endif - pte = pte_offset_map(&_pmd, haddr); - VM_BUG_ON(!pte); - for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { -@@ -2900,8 +2913,17 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, - * This's critical for some architectures (Power). - */ - pgtable = pgtable_trans_huge_withdraw(mm, pmd); -+ #ifdef CONFIG_PTP -+ #ifdef CONFIG_X86_64 -+ unsigned long pfn = page_to_pfn(pgtable); -+ paravirt_alloc_pte(mm, pfn); -+ _pmd = __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE); -+ #else -+ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE); -+ #endif -+ #else - pmd_populate(mm, &_pmd, pgtable); -- -+ #endif - pte = pte_offset_map(&_pmd, haddr); - VM_BUG_ON(!pte); + #define PTE_SHIFT ilog2(PTRS_PER_PTE) +diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h +index a629b1b9f65a..f2c86607a6db 100644 +--- a/arch/x86/include/asm/pgtable_64.h ++++ b/arch/x86/include/asm/pgtable_64.h +@@ -15,6 +15,9 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#endif -diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c -index 149ab629855c..93b6a7ebde82 100644 ---- a/mm/hugetlb_vmemmap.c -+++ b/mm/hugetlb_vmemmap.c -@@ -83,7 +83,7 @@ static int split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start) + extern p4d_t level4_kernel_pgt[512]; + extern p4d_t level4_ident_pgt[512]; +@@ -64,7 +67,11 @@ void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); - if (!head) - return 0; -- -+ - pgtable = pte_alloc_one_kernel(&init_mm); - if (!pgtable) - return -ENOMEM; -diff --git a/mm/init-mm.c b/mm/init-mm.c -index 24c809379274..4539eb8e7cce 100644 ---- a/mm/init-mm.c -+++ b/mm/init-mm.c -@@ -55,3 +55,19 @@ void setup_initial_init_mm(void *start_code, void *end_code, - init_mm.end_data = (unsigned long)end_data; - init_mm.brk = (unsigned long)brk; + static inline void native_set_pte(pte_t *ptep, pte_t pte) + { ++#ifdef CONFIG_PTP ++ ptp_set_pte(ptep, pte); ++#else + WRITE_ONCE(*ptep, pte); ++#endif } -+ -+#ifdef CONFIG_KOI -+/* -+ * This is used to init ko_mm when creating pgtable for a ko to be isolated -+ * the ko_mm belongs to a specific ko, pgdp is allocated by koi_pgd_alloc -+ */ -+void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp) { -+ ko_mm->pgd = pgdp; -+ ko_mm->mm_users = (atomic_t)ATOMIC_INIT(2); -+ ko_mm->mm_count = (atomic_t)ATOMIC_INIT(1); -+ ko_mm->mmap_lock = (struct rw_semaphore)__RWSEM_INITIALIZER(ko_mm->mmap_lock); -+ ko_mm->page_table_lock = __SPIN_LOCK_UNLOCKED(ko_mm.page_table_lock); -+ ko_mm->arg_lock = __SPIN_LOCK_UNLOCKED(ko_mm->arg_lock); -+ ko_mm->mmlist = (struct list_head)LIST_HEAD_INIT(ko_mm->mmlist); -+} + + static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, +@@ -80,7 +87,11 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) + + static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) + { ++#ifdef CONFIG_PTP ++ ptp_set_pmd(pmdp, pmd); ++#else + WRITE_ONCE(*pmdp, pmd); +#endif -diff --git a/mm/memory.c b/mm/memory.c -index badaf096a344..a7553a457754 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -80,6 +80,11 @@ - #include - #include + } + static inline void native_pmd_clear(pmd_t *pmd) +@@ -91,7 +102,11 @@ static inline void native_pmd_clear(pmd_t *pmd) + static inline pte_t native_ptep_get_and_clear(pte_t *xp) + { + #ifdef CONFIG_SMP +#ifdef CONFIG_PTP -+#include -+#include ++ return native_make_pte(ptp_xchg((pgprotval_t *)xp, 0)); ++#else + return native_make_pte(xchg(&xp->pte, 0)); +#endif -+ - #include + #else + /* native_local_ptep_get_and_clear, + but duplicated because of cyclic dependency */ +@@ -104,7 +119,11 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) + static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) + { + #ifdef CONFIG_SMP ++#ifdef CONFIG_PTP ++ return native_make_pmd(ptp_xchg((pgprotval_t *)xp, 0)); ++#else + return native_make_pmd(xchg(&xp->pmd, 0)); ++#endif + #else + /* native_local_pmdp_get_and_clear, + but duplicated because of cyclic dependency */ +@@ -116,7 +135,11 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) - #include -@@ -1111,6 +1116,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, - ret = -ENOMEM; - goto out; - } -+ - src_pte = pte_offset_map_nolock(src_mm, src_pmd, addr, &src_ptl); - if (!src_pte) { - pte_unmap_unlock(dst_pte, dst_ptl); -@@ -6626,12 +6632,20 @@ bool ptlock_alloc(struct ptdesc *ptdesc) - ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL); - if (!ptl) - return false; -+ #ifdef CONFIG_PTP -+ (iee_ptdesc(ptdesc))->ptl = ptl; -+ #else - ptdesc->ptl = ptl; -+ #endif - return true; + static inline void native_set_pud(pud_t *pudp, pud_t pud) + { ++#ifdef CONFIG_PTP ++ ptp_set_pud(pudp, pud); ++#else + WRITE_ONCE(*pudp, pud); ++#endif } - void ptlock_free(struct ptdesc *ptdesc) + static inline void native_pud_clear(pud_t *pud) +@@ -127,7 +150,11 @@ static inline void native_pud_clear(pud_t *pud) + static inline pud_t native_pudp_get_and_clear(pud_t *xp) { -+ #ifdef CONFIG_PTP -+ kmem_cache_free(page_ptl_cachep, (iee_ptdesc(ptdesc))->ptl); -+ #else - kmem_cache_free(page_ptl_cachep, ptdesc->ptl); -+ #endif + #ifdef CONFIG_SMP ++#ifdef CONFIG_PTP ++ return native_make_pud(ptp_xchg((pgprotval_t *)xp, 0)); ++#else + return native_make_pud(xchg(&xp->pud, 0)); ++#endif + #else + /* native_local_pudp_get_and_clear, + * but duplicated because of cyclic dependency +@@ -144,13 +171,21 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) + pgd_t pgd; + + if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { ++#ifdef CONFIG_PTP ++ ptp_set_p4d(p4dp, p4d); ++#else + WRITE_ONCE(*p4dp, p4d); ++#endif + return; + } + + pgd = native_make_pgd(native_p4d_val(p4d)); + pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); ++#ifdef CONFIG_PTP ++ ptp_set_p4d(p4dp, native_make_p4d(native_pgd_val(pgd))); ++#else + WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); ++#endif } - #endif -diff --git a/mm/mmap.c b/mm/mmap.c -index dfa3d2bfe289..9139c6838279 100644 ---- a/mm/mmap.c -+++ b/mm/mmap.c -@@ -2413,10 +2413,17 @@ static void unmap_region(struct mm_struct *mm, struct ma_state *mas, - update_hiwater_rss(mm); - unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); - mas_set(mas, mt_start); -+ #ifdef CONFIG_PTP -+ tlb_flush_mmu(&tlb); -+ #endif - free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, - next ? next->vm_start : USER_PGTABLES_CEILING, - mm_wr_locked); -+ #ifdef CONFIG_PTP -+ iee_tlb_finish_mmu(&tlb); -+ #else - tlb_finish_mmu(&tlb); -+ #endif + + static inline void native_p4d_clear(p4d_t *p4d) +@@ -160,7 +195,11 @@ static inline void native_p4d_clear(p4d_t *p4d) + + static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) + { ++#ifdef CONFIG_PTP ++ ptp_set_pgd(pgdp, pti_set_user_pgtbl(pgdp, pgd)); ++#else + WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); ++#endif } + static inline void native_pgd_clear(pgd_t *pgd) +diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h +index 1c5513b04f03..266e45a73001 100644 +--- a/arch/x86/include/asm/special_insns.h ++++ b/arch/x86/include/asm/special_insns.h +@@ -9,6 +9,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif + /* -@@ -3360,9 +3367,16 @@ void exit_mmap(struct mm_struct *mm) - mmap_write_lock(mm); - mt_clear_in_rcu(&mm->mm_mt); - mas_set(&mas, vma->vm_end); -+ #ifdef CONFIG_PTP -+ tlb_flush_mmu(&tlb); -+ #endif - free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, - USER_PGTABLES_CEILING, true); -+ #ifdef CONFIG_PTP -+ iee_tlb_finish_mmu(&tlb); -+ #else - tlb_finish_mmu(&tlb); -+ #endif - - /* - * Walk the list again, actually closing and freeing it, with preemption -diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c -index 99b3e9408aa0..ff58ad5365ce 100644 ---- a/mm/mmu_gather.c -+++ b/mm/mmu_gather.c -@@ -141,6 +141,54 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) - } + * The compiler should not reorder volatile asm statements with respect to each +@@ -49,9 +52,20 @@ static inline unsigned long __native_read_cr3(void) + return val; } -+#ifdef CONFIG_PTP -+static void __iee_tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) ++#ifdef CONFIG_IEE_SIP ++static inline void iee_write_cr3_early(unsigned long val) +{ -+ struct encoded_page **pages = batch->encoded_pages; -+ unsigned int nr, nr_pages; -+ -+ while (batch->nr) { -+ if (!page_poisoning_enabled_static() && !want_init_on_free()) { -+ nr = min(MAX_NR_FOLIOS_PER_FREE, batch->nr); -+ -+ /* -+ * Make sure we cover page + nr_pages, and don't leave -+ * nr_pages behind when capping the number of entries. -+ */ -+ if (unlikely(encoded_page_flags(pages[nr - 1]) & -+ ENCODED_PAGE_BIT_NR_PAGES_NEXT)) -+ nr++; -+ } else { -+ /* -+ * With page poisoning and init_on_free, the time it -+ * takes to free memory grows proportionally with the -+ * actual memory size. Therefore, limit based on the -+ * actual memory size and not the number of involved -+ * folios. -+ */ -+ for (nr = 0, nr_pages = 0; -+ nr < batch->nr && nr_pages < MAX_NR_FOLIOS_PER_FREE; -+ nr++) { -+ if (unlikely(encoded_page_flags(pages[nr]) & -+ ENCODED_PAGE_BIT_NR_PAGES_NEXT)) -+ nr_pages += encoded_nr_pages(pages[++nr]); -+ else -+ nr_pages++; -+ } -+ } -+ -+ for (int i = 0; i < nr; i++) { -+ struct page *page = encoded_page_ptr(pages[i]); -+ __iee_tlb_remove_table((void *)page); -+ } -+ pages += nr; -+ batch->nr -= nr; -+ -+ cond_resched(); -+ } ++ asm volatile("mov %0,%%cr3" : : "r" (val) : "memory"); +} +#endif + - static void tlb_batch_pages_flush(struct mmu_gather *tlb) + static inline void native_write_cr3(unsigned long val) { - struct mmu_gather_batch *batch; -@@ -150,6 +198,17 @@ static void tlb_batch_pages_flush(struct mmu_gather *tlb) - tlb->active = &tlb->local; ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr3(val); ++ #else + asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); ++ #endif + } + + static inline unsigned long native_read_cr4(void) +diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h +index 4d3c9d00d6b6..a9db313c344d 100644 +--- a/arch/x86/include/asm/tlb.h ++++ b/arch/x86/include/asm/tlb.h +@@ -34,6 +34,14 @@ static inline void __tlb_remove_table(void *table) + free_page_and_swap_cache(table); } +#ifdef CONFIG_PTP -+static void iee_tlb_batch_pages_flush(struct mmu_gather *tlb) ++#include ++static inline void __ptp_tlb_remove_table(void *table) +{ -+ struct mmu_gather_batch *batch; -+ -+ for (batch = &tlb->local; batch && batch->nr; batch = batch->next) -+ __iee_tlb_batch_free_encoded_pages(batch); -+ tlb->active = &tlb->local; ++ ptp_pg_free(&pg_cache, page_to_virt((struct page *)table)); +} +#endif + - static void tlb_batch_list_free(struct mmu_gather *tlb) + static inline void invlpg(unsigned long addr) { - struct mmu_gather_batch *batch, *next; -@@ -347,6 +406,71 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) - tlb_table_flush(tlb); - } + asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); +diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile +index 2b86fa2d8c64..9e3688b9d2a2 100644 +--- a/arch/x86/kernel/Makefile ++++ b/arch/x86/kernel/Makefile +@@ -164,3 +164,4 @@ ifeq ($(CONFIG_X86_64),y) + endif + + obj-$(CONFIG_HYGON_CSV) += csv.o ++obj-$(CONFIG_IEE) += haoc/ +diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c +index aae7456ece07..7d4eaa731075 100644 +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -1786,6 +1786,25 @@ static void text_poke_memset(void *dst, const void *src, size_t len) + + typedef void text_poke_f(void *dst, const void *src, size_t len); +#ifdef CONFIG_PTP -+static void __iee_tlb_remove_table_free(struct mmu_table_batch *batch) -+{ -+ int i; -+ -+ for (i = 0; i < batch->nr; i++) -+ __iee_tlb_remove_table(batch->tables[i]); -+ -+ free_page((unsigned long)batch); -+} -+ -+#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE -+static void iee_tlb_remove_table_rcu(struct rcu_head *head) -+{ -+ __iee_tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu)); -+} -+ -+static void iee_tlb_remove_table_free(struct mmu_table_batch *batch) -+{ -+ call_rcu(&batch->rcu, iee_tlb_remove_table_rcu); -+} -+#else -+static void iee_tlb_remove_table_free(struct mmu_table_batch *batch) -+{ -+ __iee_tlb_remove_table_free(batch); -+} -+#endif -+ -+static void iee_tlb_remove_table_one(void *table) -+{ -+ tlb_remove_table_sync_one(); -+ __iee_tlb_remove_table(table); -+} -+ -+static void iee_tlb_table_flush(struct mmu_gather *tlb) -+{ -+ struct mmu_table_batch **batch = &tlb->batch; -+ -+ if (*batch) { -+ tlb_table_invalidate(tlb); -+ iee_tlb_remove_table_free(*batch); -+ *batch = NULL; -+ } -+} -+ -+void iee_tlb_remove_table(struct mmu_gather *tlb, void *table) ++static inline void set_ptes_text_poke(struct mm_struct *mm, unsigned long addr, ++ pte_t *ptep, pte_t pte, unsigned int nr) +{ -+ struct mmu_table_batch **batch = &tlb->batch; ++ page_table_check_ptes_set(mm, ptep, pte, nr); + -+ if (*batch == NULL) { -+ *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); -+ if (*batch == NULL) { -+ tlb_table_invalidate(tlb); -+ iee_tlb_remove_table_one(table); -+ return; -+ } -+ (*batch)->nr = 0; ++ arch_enter_lazy_mmu_mode(); ++ for (;;) { ++ compiletime_assert_rwonce_type(*ptep); ++ ptp_set_pte_text_poke(ptep, pte); ++ if (--nr == 0) ++ break; ++ ptep++; ++ pte = pte_next_pfn(pte); + } -+ -+ (*batch)->tables[(*batch)->nr++] = table; -+ if ((*batch)->nr == MAX_TABLE_BATCH) -+ iee_tlb_table_flush(tlb); ++ arch_leave_lazy_mmu_mode(); +} +#endif + - static inline void tlb_table_init(struct mmu_gather *tlb) + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { - tlb->batch = NULL; -@@ -469,3 +593,55 @@ void tlb_finish_mmu(struct mmu_gather *tlb) - #endif - dec_tlb_flush_pending(tlb->mm); - } -+ + bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; +@@ -1837,11 +1856,19 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l + local_irq_save(flags); + + pte = mk_pte(pages[0], pgprot); +#ifdef CONFIG_PTP -+static void iee_tlb_flush_mmu_free(struct mmu_gather *tlb) -+{ -+ iee_tlb_table_flush(tlb); -+#ifndef CONFIG_MMU_GATHER_NO_GATHER -+ iee_tlb_batch_pages_flush(tlb); ++ set_ptes_text_poke(poking_mm, poking_addr, ptep, pte, 1); ++#else + set_pte_at(poking_mm, poking_addr, ptep, pte); +#endif -+} -+ -+void iee_tlb_flush_mmu(struct mmu_gather *tlb) -+{ -+ tlb_flush_mmu_tlbonly(tlb); -+ iee_tlb_flush_mmu_free(tlb); -+} -+ -+void iee_tlb_finish_mmu(struct mmu_gather *tlb) -+{ -+ /* -+ * If there are parallel threads are doing PTE changes on same range -+ * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB -+ * flush by batching, one thread may end up seeing inconsistent PTEs -+ * and result in having stale TLB entries. So flush TLB forcefully -+ * if we detect parallel PTE batching threads. -+ * -+ * However, some syscalls, e.g. munmap(), may free page tables, this -+ * needs force flush everything in the given range. Otherwise this -+ * may result in having stale TLB entries for some architectures, -+ * e.g. aarch64, that could specify flush what level TLB. -+ */ -+ if (mm_tlb_flush_nested(tlb->mm)) { -+ /* -+ * The aarch64 yields better performance with fullmm by -+ * avoiding multiple CPUs spamming TLBI messages at the -+ * same time. -+ * -+ * On x86 non-fullmm doesn't yield significant difference -+ * against fullmm. -+ */ -+ tlb->fullmm = 1; -+ __tlb_reset_range(tlb); -+ tlb->freed_tables = 1; -+ } -+ -+ iee_tlb_flush_mmu(tlb); -+ -+#ifndef CONFIG_MMU_GATHER_NO_GATHER -+ tlb_batch_list_free(tlb); + + if (cross_page_boundary) { + pte = mk_pte(pages[1], pgprot); ++#ifdef CONFIG_PTP ++ set_ptes_text_poke(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte, 1); ++#else + set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); +#endif -+ dec_tlb_flush_pending(tlb->mm); -+} + } + + /* +diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c +index 50383bc46dd7..93c4d78b2835 100644 +--- a/arch/x86/kernel/asm-offsets.c ++++ b/arch/x86/kernel/asm-offsets.c +@@ -25,6 +25,9 @@ + #include + #endif + ++#ifdef CONFIG_IEE ++#include +#endif -\ No newline at end of file -diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c -index 4fcd959dcc4d..17f2d8b81285 100644 ---- a/mm/pgtable-generic.c -+++ b/mm/pgtable-generic.c -@@ -169,9 +169,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + #ifdef CONFIG_X86_32 + # include "asm-offsets_32.c" + #else +@@ -127,4 +130,10 @@ static void __used common(void) + OFFSET(ARIA_CTX_rounds, aria_ctx, rounds); + #endif - /* FIFO */ - if (!pmd_huge_pte(mm, pmdp)) -+ #ifdef CONFIG_PTP -+ INIT_LIST_HEAD(&((page_to_iee_ptdesc(pgtable))->pt_list)); -+ #else - INIT_LIST_HEAD(&pgtable->lru); -+ #endif - else -+ #ifdef CONFIG_PTP -+ list_add(&((page_to_iee_ptdesc(pgtable))->pt_list), &((page_to_iee_ptdesc(pmd_huge_pte(mm, pmdp)))->pt_list)); -+ #else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); -+ #endif - pmd_huge_pte(mm, pmdp) = pgtable; ++#ifdef CONFIG_IEE ++ OFFSET(IEE_STACK, iee_stack, stack); ++#endif ++#ifdef CONFIG_PTP ++ OFFSET(IEE_DISABLE, iee_disable_t, disabled_cnt); ++#endif } - #endif -@@ -186,10 +194,23 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) +diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c +index b5fdc06661b3..f91ff13c9f5b 100644 +--- a/arch/x86/kernel/cpu/common.c ++++ b/arch/x86/kernel/cpu/common.c +@@ -65,6 +65,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_SIP ++#include ++#endif - /* FIFO */ - pgtable = pmd_huge_pte(mm, pmdp); -+ #ifdef CONFIG_PTP -+ struct ptdesc_t *ptdesc = list_first_entry_or_null(&((page_to_iee_ptdesc(pgtable))->pt_list), -+ struct ptdesc_t, pt_list); -+ if(ptdesc) -+ pmd_huge_pte(mm, pmdp) = ptdesc->page; -+ else -+ pmd_huge_pte(mm, pmdp) = NULL; + #include "cpu.h" + +@@ -403,14 +406,25 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) + } + + /* These bits should not change their value after CPU init is finished. */ ++#ifdef CONFIG_IEE_SIP ++const unsigned long cr4_pinned_mask = ++ X86_CR4_UMIP | ++ X86_CR4_FSGSBASE | X86_CR4_CET; ++DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); ++unsigned long cr4_pinned_bits __ro_after_init; ++#else + static const unsigned long cr4_pinned_mask = + X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | + X86_CR4_FSGSBASE | X86_CR4_CET; + static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); + static unsigned long cr4_pinned_bits __ro_after_init; ++#endif + + void native_write_cr0(unsigned long val) + { ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr0(val); + #else - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + unsigned long bits_missing = 0; + + set_register: +@@ -425,11 +439,15 @@ void native_write_cr0(unsigned long val) + /* Warn after we've set the missing bits. */ + WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); + } + #endif - if (pmd_huge_pte(mm, pmdp)) -+ #ifdef CONFIG_PTP -+ list_del(&((iee_ptdesc(pgtable))->pt_list)); -+ #else - list_del(&pgtable->lru); -+ #endif - return pgtable; } - #endif -@@ -238,7 +259,11 @@ static void pte_free_now(struct rcu_head *head) - { - struct page *page; + EXPORT_SYMBOL(native_write_cr0); -+ #ifdef CONFIG_PTP -+ page = container_of(head, struct ptdesc_t, pt_rcu_head)->page; + void __no_profile native_write_cr4(unsigned long val) + { ++ #ifdef CONFIG_IEE_SIP ++ iee_write_cr4(val); + #else - page = container_of(head, struct page, rcu_head); + unsigned long bits_changed = 0; + + set_register: +@@ -445,6 +463,7 @@ void __no_profile native_write_cr4(unsigned long val) + WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", + bits_changed); + } + #endif - pte_free(NULL /* mm not passed and not used */, (pgtable_t)page); } + #if IS_MODULE(CONFIG_LKDTM) + EXPORT_SYMBOL_GPL(native_write_cr4); +@@ -610,7 +629,9 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) + else + wrmsrl(MSR_IA32_S_CET, 0); -@@ -247,7 +272,11 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) - struct page *page; ++ #ifndef CONFIG_IEE + cr4_set_bits(X86_CR4_CET); ++ #endif - page = pgtable; -+ #ifdef CONFIG_PTP -+ call_rcu(&((page_to_iee_ptdesc(page))->pt_rcu_head), pte_free_now); -+ #else - call_rcu(&page->rcu_head, pte_free_now); + if (kernel_ibt && ibt_selftest()) { + pr_err("IBT selftest: Failed!\n"); +@@ -2503,4 +2524,7 @@ void __init arch_cpu_finalize_init(void) + * hypercalls work when the SWIOTLB bounce buffers are decrypted. + */ + mem_encrypt_init(); ++ #ifdef CONFIG_IEE_SIP ++ iee_sip_init(); + #endif } - #endif /* pte_free_defer */ - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -diff --git a/mm/slab.h b/mm/slab.h -index 3d07fb428393..67ff0beb4ec3 100644 ---- a/mm/slab.h -+++ b/mm/slab.h -@@ -26,6 +26,35 @@ typedef u64 freelist_full_t; - #undef system_has_freelist_aba - #endif - -+#ifdef CONFIG_IEE -+extern struct kmem_cache *iee_stack_jar; -+#endif +diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile +new file mode 100644 +index 000000000000..9f0ce58ca1fe +--- /dev/null ++++ b/arch/x86/kernel/haoc/Makefile +@@ -0,0 +1,3 @@ ++obj-y += haoc.o ++obj-y += iee/ ++obj-$(CONFIG_PTP) += ptp/ +diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c +new file mode 100644 +index 000000000000..b62c02914469 +--- /dev/null ++++ b/arch/x86/kernel/haoc/haoc.c +@@ -0,0 +1,28 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include + ++typedef void (*iee_func)(void); ++iee_func iee_funcs[] = { ++ (iee_func)_iee_memcpy, ++ (iee_func)_iee_memset, ++ (iee_func)_iee_set_freeptr, ++ (iee_func)_iee_test_and_clear_bit, ++#ifdef CONFIG_IEE_PTRP ++ (iee_func)_iee_set_token_pgd, ++ (iee_func)_iee_invalidate_token, ++ (iee_func)_iee_validate_token, ++#endif +#ifdef CONFIG_PTP -+extern struct kmem_cache *pgtable_jar; -+extern struct kmem_cache *ptdesc_jar; -+extern struct kmem_cache *pgd_jar; ++#ifdef CONFIG_IEE_PTRP ++ (iee_func)_iee_unset_token, ++ (iee_func)_iee_set_token, +#endif -+ -+#ifdef CONFIG_CREDP -+extern struct kmem_cache *cred_jar; ++ (iee_func)_iee_set_pte, ++ (iee_func)_iee_set_pmd, ++ (iee_func)_iee_set_pud, ++ (iee_func)_iee_set_p4d, ++ (iee_func)_iee_set_pgd, ++ (iee_func)_iee_set_pte_text_poke, ++#endif ++ NULL ++}; +diff --git a/arch/x86/kernel/haoc/iee/Makefile b/arch/x86/kernel/haoc/iee/Makefile +new file mode 100644 +index 000000000000..73a7f1470c5c +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/Makefile +@@ -0,0 +1,4 @@ ++obj-y += iee-gate.o iee-init.o iee.o iee-func.o ++obj-$(CONFIG_IEE_SIP) += iee-si.o ++obj-$(CONFIG_IEE_PTRP) += iee-token.o ++ccflags-y += -I$(srctree)/mm +diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c +new file mode 100644 +index 000000000000..8bb80cd9731b +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee-func.c +@@ -0,0 +1,71 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include "slab.h" ++#ifdef CONFIG_IEE_PTRP ++#include +#endif + -+#ifdef CONFIG_KEYP -+extern struct kmem_cache *key_jar; -+extern struct kmem_cache *key_union_jar; -+extern struct kmem_cache *key_struct_jar; -+extern struct kmem_cache *key_payload_jar; ++void set_iee_page(unsigned long addr, unsigned int order) ++{ ++ set_memory_ro(addr, 1 << order); ++} ++ ++void unset_iee_page(unsigned long addr, unsigned int order) ++{ ++ set_memory_rw(addr, 1 << order); ++} ++ ++struct iee_free_slab_work { ++ struct work_struct work; ++ struct kmem_cache *s; ++ struct slab *slab; ++}; ++ ++void iee_free_slab(struct kmem_cache *s, struct slab *slab, ++ void (*do_free_slab)(struct work_struct *work)) ++{ ++ struct iee_free_slab_work *iee_free_slab_work = ++ kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); ++ ++ iee_free_slab_work->s = s; ++ iee_free_slab_work->slab = slab; ++ INIT_WORK(&iee_free_slab_work->work, do_free_slab); ++ schedule_work(&iee_free_slab_work->work); ++} ++ ++#ifdef CONFIG_IEE_PTRP ++static void iee_free_task_struct_slab(struct work_struct *work) ++{ ++ struct iee_free_slab_work *iee_free_slab_work = ++ container_of(work, struct iee_free_slab_work, work); ++ struct slab *slab = iee_free_slab_work->slab; ++ struct folio *folio = slab_folio(slab); ++ unsigned int order = folio_order(folio); ++ unsigned long token = __slab_to_iee(slab); ++ // Free token. ++ iee_set_token_page_invalid(token, 0, order); ++ __free_pages(&folio->page, order); ++ kfree(iee_free_slab_work); ++} +#endif + -+#ifdef CONFIG_IEE_SELINUX_P -+extern struct kmem_cache *policy_jar; ++bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order) ++{ ++#ifdef CONFIG_IEE_PTRP ++ if (s == task_struct_cachep) { ++ iee_free_slab(s, slab, iee_free_task_struct_slab); ++ return true; ++ } +#endif ++ return false; ++} + -+#if defined(CONFIG_KOI) || defined(CONFIG_IEE) -+extern struct kmem_cache *task_struct_cachep; ++unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) ++{ ++#ifdef CONFIG_IEE_PTRP ++ if (strcmp(s->name, "task_struct") == 0) ++ return IEE_DATA_ORDER; +#endif ++ return order; ++} +diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S +new file mode 100644 +index 000000000000..cdec5a31d248 +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee-gate.S +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#include ++#include ++#include ++#include ++#include + - /* - * Freelist pointer and counter to cmpxchg together, avoids the typical ABA - * problems with cmpxchg of just a pointer. -@@ -767,7 +796,6 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, - s->flags, flags); - kmsan_slab_alloc(s, p[i], flags); - } -- - memcg_slab_post_alloc_hook(s, objcg, flags, size, p); - } - -@@ -843,6 +871,15 @@ static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } - - static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) - { -+ #ifdef CONFIG_IEE -+ if(c == iee_stack_jar) -+ return false; -+ #endif -+ #ifdef CONFIG_PTP -+ if(c == pgtable_jar || c == pgd_jar) -+ return false; -+ #endif ++#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) + - if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, - &init_on_alloc)) { - if (c->ctor) -diff --git a/mm/slab_common.c b/mm/slab_common.c -index 735f59618556..a6252bc223be 100644 ---- a/mm/slab_common.c -+++ b/mm/slab_common.c -@@ -145,6 +145,26 @@ int slab_unmergeable(struct kmem_cache *s) - if (s->ctor) - return 1; - -+ #ifdef CONFIG_IEE -+ if(strcmp(s->name, "iee_stack_jar") == 0) -+ return 1; -+ #endif -+ #ifdef CONFIG_PTP -+ if(strcmp(s->name, "pgtable_jar") == 0) -+ return 1; -+ if(strcmp(s->name, "pgd_jar") == 0) -+ return 1; -+ #endif -+ #ifdef CONFIG_KEYP -+ if(strcmp(s->name, "key_jar") == 0) -+ return 1; -+ #endif -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(strcmp(s->name, "policy_jar") == 0) -+ return 1; -+ #endif ++/* scratch_reg would be changed, ++caller should dertimine if scratch_reg should be saved and restored */ ++.macro DISABLE_WP scratch_reg:req ++ /* Disable write protection*/ ++ movq %cr0, %\scratch_reg ++ andq $(~X86_CR0_WP), %\scratch_reg ++ movq %\scratch_reg, %cr0 ++.endm ++ ++.macro ENABLE_WP scratch_reg:req ++ /* Enable write protection */ ++ movq %cr0, %\scratch_reg ++1: ++ orq $X86_CR0_WP, %\scratch_reg ++ movq %\scratch_reg, %cr0 ++ testq $X86_CR0_WP, %\scratch_reg ++ je 1b ++.endm + ++SYM_FUNC_START(iee_rw_gate) ++ /* save Interrupt flag */ ++ pushfq ++ /* close irq*/ ++ cli + - #ifdef CONFIG_HARDENED_USERCOPY - if (s->usersize) - return 1; -@@ -170,6 +190,25 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, - if (ctor) - return NULL; - -+ #ifdef CONFIG_IEE -+ if(strcmp(name, "iee_stack_jar") == 0) -+ return NULL; -+ #endif -+ #ifdef CONFIG_PTP -+ if(strcmp(name, "pgtable_jar") == 0) -+ return NULL; -+ if(strcmp(name, "pgd_jar") == 0) -+ return NULL; -+ #endif -+ #ifdef CONFIG_KEYP -+ if(strcmp(name, "key_jar") == 0) -+ return NULL; -+ #endif -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(strcmp(name, "policy_jar") == 0) -+ return NULL; -+ #endif ++ pushq %r12 + - size = ALIGN(size, sizeof(void *)); - align = calculate_alignment(flags, align, size); - size = ALIGN(size, align); -diff --git a/mm/slub.c b/mm/slub.c -index a5ff41567a8f..49fee6d57f8f 100644 ---- a/mm/slub.c -+++ b/mm/slub.c -@@ -42,11 +42,30 @@ - #include - #include - -+#ifdef CONFIG_IEE -+#include -+#include -+#include -+#include -+#else -+#ifdef CONFIG_KOI -+#include -+extern void koi_add_page_mapping(void *token, void *token_page, unsigned int order); -+extern void koi_init_token(struct task_struct *tsk); ++#ifdef CONFIG_PTP ++ movq PER_CPU_VAR(iee_disables) + IEE_DISABLE, %r12 ++ testq %r12, %r12 ++ jz rw_wp_enabled ++ ++ leaq iee_funcs(%rip), %rax ++ call *(%rax, %rdi, 8) ++ ++ popq %r12 ++ popfq ++ jmp __x86_return_thunk ++ ++rw_wp_enabled: +#endif ++ DISABLE_WP r12 ++ ++ /* switch to iee stack */ ++ movq %rsp, %r12 ++ movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp ++ ++ /* call iee func */ ++ leaq iee_funcs(%rip), %rax ++ call *(%rax, %rdi, 8) ++ ++ /* switch to kernel stack */ ++ movq %r12, %rsp ++ ++ ENABLE_WP r12 ++ ++ popq %r12 ++ ++ /* restore irq*/ ++ popfq ++ ++ jmp __x86_return_thunk /* ret */ ++SYM_FUNC_END(iee_rw_gate) ++#ifdef CONFIG_PTP ++EXPORT_SYMBOL(iee_rw_gate) +#endif ++ ++#ifdef CONFIG_IEE_SIP ++SYM_FUNC_START(iee_rwx_gate) ++ pushq %r12 ++ ++ /* save Interrupt flag*/ ++ pushfq ++ /* close irq */ ++ cli ++ +#ifdef CONFIG_PTP -+#include -+extern unsigned long pgtable_jar_offset; -+extern unsigned long pgd_jar_offset; ++ movq PER_CPU_VAR(iee_disables) + IEE_DISABLE, %r12 ++ testq %r12, %r12 ++ jz rwx_wp_enabled ++ ++ movq %cr4, %rax /* rax -> cr4 */ ++ andq $(~X86_CR4_SMEP), %rax ++ movq %rax, %cr4 ++ ++ call _iee_si_handler ++ ++ movq %cr4, %rax /* rax -> cr4 */ ++3: orq $X86_CR4_SMEP_SMAP, %rax ++ movq %rax, %cr4 ++ andq $X86_CR4_SMEP_SMAP, %rax ++ cmpq $X86_CR4_SMEP_SMAP, %rax ++ jnz 3 ++ ++ popfq ++ popq %r12 ++ jmp __x86_return_thunk /* ret */ ++ ++rwx_wp_enabled: +#endif ++ /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ ++ movq %cr4, %rax /* rax -> cr4 */ ++ andq $(~X86_CR4_SMEP), %rax ++ movq %rax, %cr4 + - #include - #include - - #include "internal.h" - ++ DISABLE_WP r12 + - /* - * Lock order: - * 1. slab_mutex (Global Mutex) -@@ -317,6 +336,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) - /* - * Tracking user of a slab. - */ -+#ifndef CONFIG_IEE - #define TRACK_ADDRS_COUNT 16 - struct track { - unsigned long addr; /* Called from address */ -@@ -329,6 +349,7 @@ struct track { - }; - - enum track_item { TRACK_ALLOC, TRACK_FREE }; ++ movq %rsp, %r12 ++ /* If iee hasn't been initialized, skip stack switch. */ ++ cmpb $0, iee_init_done(%rip) ++ jz 2f ++ ++ /* switch to iee stack */ ++ movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp ++ ++2: call _iee_si_handler ++ /* switch to kernel stack. If iee hasn't been initialized, skip switch*/ ++ movq %r12, %rsp ++ ++ ENABLE_WP r12 ++ ++ /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ ++ movq %cr4, %rax /* rax -> cr4 */ ++1: orq $X86_CR4_SMEP_SMAP, %rax ++ movq %rax, %cr4 ++ andq $X86_CR4_SMEP_SMAP, %rax ++ cmpq $X86_CR4_SMEP_SMAP, %rax ++ jnz 1 ++ ++ /* restore irq*/ ++ popfq ++ ++ popq %r12 ++ jmp __x86_return_thunk /* ret */ ++SYM_FUNC_END(iee_rwx_gate) +#endif - - #ifdef SLAB_SUPPORTS_SYSFS - static int sysfs_slab_add(struct kmem_cache *); -@@ -379,7 +400,9 @@ static struct workqueue_struct *flushwq; - * freeptr_t represents a SLUB freelist pointer, which might be encoded - * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled. - */ -+#ifndef CONFIG_IEE - typedef struct { unsigned long v; } freeptr_t; +diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c +new file mode 100644 +index 000000000000..ef7909714858 +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee-init.c +@@ -0,0 +1,157 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_IEE_SIP ++#include +#endif - - /* - * Returns freelist pointer (ptr). With hardening, this is obfuscated -@@ -419,7 +442,14 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) - - object = kasan_reset_tag(object); - ptr_addr = (unsigned long)object + s->offset; -+ #ifdef CONFIG_IEE -+ if(s == iee_stack_jar) -+ p.v = (unsigned long)iee_read_freeptr(ptr_addr); -+ else -+ p = *(freeptr_t *)(ptr_addr); -+ #else - p = *(freeptr_t *)(ptr_addr); -+ #endif - return freelist_ptr_decode(s, p, ptr_addr); - } - -@@ -464,6 +494,36 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) - #endif - - freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); -+ #ifdef CONFIG_IEE -+ if (s == iee_stack_jar) { -+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); -+ return; -+ } -+ #endif -+ #ifdef CONFIG_PTP -+ if (s == pgtable_jar || s == pgd_jar) { -+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); -+ return; -+ } -+ #endif -+ #ifdef CONFIG_CREDP -+ if (s == cred_jar) { -+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); -+ return; -+ } -+ #endif -+ #ifdef CONFIG_KEYP -+ if(s == key_jar) { -+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); -+ return; ++ ++/* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ ++unsigned long IEE_OFFSET = 0x200000000000; ++#ifdef CONFIG_IEE_SIP ++bool iee_init_done __iee_si_data; ++#else ++bool iee_init_done; ++#endif ++DEFINE_PER_CPU(struct iee_stack, iee_stacks); ++ ++static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, unsigned long end) ++{ ++ void *p; ++ pmd_t *pmd; ++ unsigned long pmd_next; ++ phys_addr_t phys; ++ pgprot_t pgprot_shadow_pmd; ++ ++ addr = ALIGN_DOWN(addr, PMD_SIZE); ++ phys = __iee_pa(addr); ++ pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & (~__RW) & (~___D)); ++ ++ if (pud_none(*pud)) { ++ p = alloc_low_pages(1); ++ pud_populate(&init_mm, pud, p); + } -+ #endif -+ #ifdef CONFIG_IEE_SELINUX_P -+ if (s == policy_jar) { -+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); -+ return; ++ ++ pmd = pmd_offset(pud, addr); ++ do { ++ pmd_next = pmd_addr_end(addr, end); ++ set_pmd(pmd, __pmd(phys | pgprot_val(pgprot_shadow_pmd))); ++ phys += pmd_next - addr; ++ } while (pmd++, addr = pmd_next, addr != end); ++} ++ ++static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, unsigned long end) ++{ ++ void *p; ++ pud_t *pud; ++ unsigned long pud_next; ++ ++ if (p4d_none(*p4d)) { ++ p = alloc_low_pages(1); ++ p4d_populate(&init_mm, p4d, p); + } -+ #endif - *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); - } - -@@ -853,7 +913,31 @@ static void set_track_update(struct kmem_cache *s, void *object, - depot_stack_handle_t handle) - { - struct track *p = get_track(s, object, alloc); -- -+#ifdef CONFIG_CREDP -+ struct track tmp; -+ if(s == cred_jar) -+ { -+ tmp = *p; -+ #ifdef CONFIG_STACKDEPOT -+ tmp.handle = handle; -+ #endif -+ tmp.addr = addr; -+ tmp.cpu = smp_processor_id(); -+ tmp.pid = current->pid; -+ tmp.when = jiffies; -+ iee_set_track(p,&tmp); ++ ++ pud = pud_offset(p4d, addr); ++ do { ++ pud_next = pud_addr_end(addr, end); ++ pr_info("IEE: iee_populate_pud(%#010lx, %#010lx)\n", ++ addr, pud_next); ++ _iee_mapping_populate_pud(pud, addr, pud_next); ++ } while (pud++, addr = pud_next, addr != end); ++} ++ ++static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, unsigned long end) ++{ ++ void *p; ++ p4d_t *p4d; ++ unsigned long p4d_next; ++ ++ if (pgd_none(*pgd)) { ++ p = alloc_low_pages(1); ++ pgd_populate(&init_mm, pgd, p); + } -+ else -+ { -+ #ifdef CONFIG_STACKDEPOT -+ p->handle = handle; -+ #endif -+ p->addr = addr; -+ p->cpu = smp_processor_id(); -+ p->pid = current->pid; -+ p->when = jiffies; ++ ++ p4d = p4d_offset(pgd, addr); ++ do { ++ p4d_next = p4d_addr_end(addr, end); ++ pr_info("IEE: iee_populate_p4d(%#010lx, %#010lx)\n", ++ addr, p4d_next); ++ _iee_mapping_populate_p4d(p4d, addr, p4d_next); ++ } while (p4d++, addr = p4d_next, addr != end); ++} ++ ++static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_paddr) ++{ ++ unsigned long addr = (unsigned long)__phys_to_iee(start_paddr); ++ unsigned long end = (unsigned long)__phys_to_iee(end_paddr); ++ unsigned long pgd_next; ++ ++ pgd_t *pgd = pgd_offset_k(addr); ++ ++ spin_lock(&pgd_lock); ++ do { ++ pgd_next = pgd_addr_end(addr, end); ++ pr_info("IEE: iee_populate_pgd(%#010lx, %#010lx)\n", ++ addr, pgd_next); ++ _iee_mapping_populate_pgd(pgd, addr, pgd_next); ++ } while (pgd++, addr = pgd_next, addr != end); ++ spin_unlock(&pgd_lock); ++} ++ ++static void __init _iee_mapping_init(void) ++{ ++ struct memblock_region *r; ++ unsigned long start_pfn, end_pfn; ++ phys_addr_t start_paddr, end_paddr; ++ unsigned long nr_pages = 0; ++ ++ for_each_mem_region(r) { ++ start_pfn = memblock_region_memory_base_pfn(r); ++ end_pfn = memblock_region_memory_end_pfn(r); ++ ++ start_paddr = PFN_PHYS(start_pfn); ++ end_paddr = PFN_PHYS(end_pfn); ++ ++ nr_pages += end_pfn - start_pfn; ++ ++ pr_info("IEE: mapping iee mapping [mem %#010lx-%#010lx]\n", ++ (unsigned long)start_paddr, (unsigned long)end_paddr); ++ ++ _iee_init_mapping(start_paddr, end_paddr); + } -+#else - #ifdef CONFIG_STACKDEPOT - p->handle = handle; - #endif -@@ -861,6 +945,7 @@ static void set_track_update(struct kmem_cache *s, void *object, - p->cpu = smp_processor_id(); - p->pid = current->pid; - p->when = jiffies; -+#endif - } - - static __always_inline void set_track(struct kmem_cache *s, void *object, -@@ -879,7 +964,14 @@ static void init_tracking(struct kmem_cache *s, void *object) - return; - - p = get_track(s, object, TRACK_ALLOC); -+ #ifdef CONFIG_CREDP -+ if(s == cred_jar) -+ iee_memset(p, 0, 2*sizeof(struct track)); -+ else -+ memset(p, 0, 2*sizeof(struct track)); -+ #else - memset(p, 0, 2*sizeof(struct track)); -+ #endif - } - - static void print_track(const char *s, struct track *t, unsigned long pr_time) -@@ -1045,7 +1137,14 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) - unsigned int poison_size = s->object_size; - - if (s->flags & SLAB_RED_ZONE) { -+ #ifdef CONFIG_CREDP -+ if(s == cred_jar) -+ iee_memset(p - s->red_left_pad, val, s->red_left_pad); -+ else -+ memset(p - s->red_left_pad, val, s->red_left_pad); -+ #else - memset(p - s->red_left_pad, val, s->red_left_pad); -+ #endif - - if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { - /* -@@ -1058,12 +1157,33 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) - } - - if (s->flags & __OBJECT_POISON) { -+ #ifdef CONFIG_CREDP -+ if(s == cred_jar) -+ { -+ iee_memset(p, POISON_FREE, poison_size - 1); -+ iee_memset(&p[poison_size - 1], POISON_END, 1); -+ } -+ else -+ { -+ memset(p, POISON_FREE, poison_size - 1); -+ p[poison_size - 1] = POISON_END; -+ } -+ #else - memset(p, POISON_FREE, poison_size - 1); - p[poison_size - 1] = POISON_END; -+ #endif - } - -- if (s->flags & SLAB_RED_ZONE) -+ if (s->flags & SLAB_RED_ZONE) { -+ #ifdef CONFIG_CREDP -+ if(s == cred_jar) -+ iee_memset(p + poison_size, val, s->inuse - poison_size); -+ else -+ memset(p + poison_size, val, s->inuse - poison_size); -+ #else - memset(p + poison_size, val, s->inuse - poison_size); -+ #endif ++ pr_info("IEE: IEE shadow mapping init done"); ++} ++ ++static void __init _iee_stack_init(void) ++{ ++ int cpu; ++ struct iee_stack *iee_stack; ++ void *stack_base; ++ struct page *page; ++ ++ for_each_possible_cpu(cpu) { ++ stack_base = (void *)page_address(alloc_pages(GFP_KERNEL, IEE_STACK_ORDER)); ++ iee_stack = per_cpu_ptr(&iee_stacks, cpu); ++ page = alloc_pages(GFP_KERNEL, IEE_STACK_ORDER); ++ iee_stack->stack = (void *)page_address(page) + PAGE_SIZE * (1 << IEE_STACK_ORDER); ++ pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, (unsigned long)iee_stack->stack); ++ set_memory_ro((unsigned long)stack_base, (1 << IEE_STACK_ORDER)); + } - } - - static void restore_bytes(struct kmem_cache *s, char *message, u8 data, -@@ -1433,7 +1553,14 @@ void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) - return; - - metadata_access_enable(); -+ #ifdef CONFIG_CREDP -+ if(s == cred_jar) -+ iee_memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); -+ else -+ memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); -+ #else - memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); -+ #endif - metadata_access_disable(); - } - -@@ -1990,6 +2117,36 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) - cur = setup_object(s, cur); - slab->freelist = cur; - -+ #ifdef CONFIG_IEE -+ if(s == task_struct_cachep) -+ { -+ int i; -+ void *pstack; -+ void *obj; -+ for(i = 0; i < freelist_count; i++) -+ { -+ pstack = get_iee_stack(); -+ if (!pstack) { -+ pr_err("SLUB: Unable to get IEE stack for %s\n", s->name); -+ return false; ++} ++ ++static void __init _iee_offset_init(void) ++{ ++ if (pgtable_l5_enabled()) ++ IEE_OFFSET = 0x40000000000000; ++} ++ ++void __init iee_init(void) ++{ ++ _iee_offset_init(); ++ _iee_mapping_init(); ++ _iee_stack_init(); ++} +diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c +new file mode 100644 +index 000000000000..60a2694d3c09 +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee-si.c +@@ -0,0 +1,127 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++ ++unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) ++{ ++ va_list pArgs; ++ unsigned long val; ++ ++ va_start(pArgs, flag); ++ switch (flag) { ++ case IEE_SIP_TEST: ++ break; ++ case IEE_WRITE_CR0: { ++ val = va_arg(pArgs, u64); ++ unsigned long bits_missing = 0; ++set_register_cr0: ++ asm volatile("mov %0,%%cr0" : "+r"(val) : : "memory"); ++ if (static_branch_likely(&cr_pinning)) { ++ if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { ++ bits_missing = X86_CR0_WP; ++ val |= bits_missing; ++ goto set_register_cr0; + } -+ obj = start + s->random_seq[i]; -+ iee_init_token((struct task_struct *)obj, pstack + PAGE_SIZE * 4, (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 0)); ++ /* Warn after we've set the missing bits. */ ++ WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); + } ++ break; + } -+ #endif -+ #if defined(CONFIG_KOI) && !defined(CONFIG_IEE) -+ if(s == task_struct_cachep) -+ { -+ int i; -+ void *obj; -+ for(i = 0; i < freelist_count; i++) -+ { -+ obj = start + s->random_seq[i]; -+ koi_init_token((struct task_struct *)obj); ++ case IEE_WRITE_CR3: { ++ val = va_arg(pArgs, u64); ++ asm volatile("mov %0,%%cr3" : : "r"(val) : "memory"); ++ break; ++ } ++ case IEE_WRITE_CR4: { ++ val = va_arg(pArgs, u64); ++ val &= ~(X86_CR4_SMEP); ++ unsigned long bits_changed = 0; ++set_register_cr4: ++ asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); ++ if (static_branch_likely(&cr_pinning)) { ++ if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) { ++ bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits; ++ val = (val & ~cr4_pinned_mask) | cr4_pinned_bits; ++ goto set_register_cr4; ++ } ++ /* Warn after we've corrected the changed bits. */ ++ WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", ++ bits_changed); + } ++ break; + } -+ #endif - for (idx = 1; idx < slab->objects; idx++) { - next = next_freelist_entry(s, &pos, start, page_limit, - freelist_count); -@@ -2021,6 +2178,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) - void *start, *p, *next; - int idx; - bool shuffle; -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ unsigned int order; -+ #endif - - flags &= gfp_allowed_mask; - -@@ -2034,15 +2194,46 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) - if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) - alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; - -+ #ifdef CONFIG_PTP -+ if(s == pgtable_jar || s == pgd_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif -+ #ifdef CONFIG_KEYP -+ if(s == key_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif ++ case IEE_LOAD_IDT: { ++ const struct desc_ptr *new_val; + -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(s == policy_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif - slab = alloc_slab_page(alloc_gfp, node, oo); -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ order = oo_order(oo); -+ #endif - if (unlikely(!slab)) { - oo = s->min; - alloc_gfp = flags; -+ #ifdef CONFIG_PTP -+ if(s == pgtable_jar || s == pgd_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif -+ #ifdef CONFIG_KEYP -+ if(s == key_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(s == policy_jar) -+ alloc_gfp |= __GFP_ZERO; -+ #endif - /* - * Allocation may have failed due to fragmentation. - * Try a lower order alloc if possible - */ - slab = alloc_slab_page(alloc_gfp, node, oo); -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ order = oo_order(oo); -+ #endif - if (unlikely(!slab)) - return NULL; - stat(s, ORDER_FALLBACK); -@@ -2052,6 +2243,63 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) - slab->inuse = 0; - slab->frozen = 0; - -+ #ifdef CONFIG_IEE -+ if (s == task_struct_cachep) -+ { -+ void *token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(slab_folio(slab), 0))); -+ void *alloc_token = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); -+ iee_set_token_page_valid(token_addr, alloc_token, order); ++ new_val = va_arg(pArgs, const struct desc_ptr*); ++ asm volatile("lidt %0"::"m" (*new_val)); ++ break; + } -+ if (s == iee_stack_jar) { -+ set_iee_stack_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); + } ++ va_end(pArgs); ++ return 0; ++} + -+ #ifdef CONFIG_PTP -+ if (s == pgtable_jar || s == pgd_jar) -+ { -+ set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); ++static void __init _iee_set_kernel_upage(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ pgd_t pgd = READ_ONCE(*pgdp); ++ ++ pgd = __pgd((pgd_val(pgd) | _USR) & ~___G); ++ set_pgd(pgdp, pgd); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ ++ p4d = __p4d((p4d_val(p4d) | _USR) & ~___G); ++ set_p4d(p4dp, p4d); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ pud_t pud = READ_ONCE(*pudp); ++ ++ pud = __pud((pud_val(pud) | _USR) & ~___G); ++ set_pud(pudp, pud); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ pmd = __pmd((pmd_val(pmd) | _USR) & ~___G); ++ set_pmd(pmdp, pmd); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ pte = __pte((pte_val(pte) | _USR) & ~___G); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range(addr, addr + PAGE_SIZE); ++} ++ ++void __init iee_sip_init(void) ++{ ++ unsigned long addr, start, end; ++ int num_pages; ++ /* Map .iee.text as U RWX pages */ ++ start = (unsigned long)__iee_si_text_start; ++ end = (unsigned long)__iee_si_text_end; ++ pr_info("IEE: mapping .iee.text:[0x%lx, 0x%lx] as U pages...", start, end); ++ addr = start; ++ for ( ; addr < end; addr += PAGE_SIZE) { ++ set_memory_4k(addr, 1); ++ _iee_set_kernel_upage(addr); ++ set_memory_4k((unsigned long)__va(__pa(addr)), 1); ++ _iee_set_kernel_upage((unsigned long)__va(__pa(addr))); + } -+ #endif ++ iee_init_done = true; ++ /* Map .iee.data as RO pages */ ++ start = (unsigned long)__iee_si_data_start; ++ end = (unsigned long)__iee_si_data_end; ++ num_pages = (end - start) / PAGE_SIZE; ++ set_memory_ro(start, num_pages); ++ /* All initialization is done. Do some simple tests. */ ++ pr_info("IEE: testing iee_exec_entry si_test..."); ++ iee_sip_test(); ++} +diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c +new file mode 100644 +index 000000000000..293ff6a7c0db +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee-token.c +@@ -0,0 +1,181 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include "slab.h" + -+ #ifdef CONFIG_CREDP -+ if (s == cred_jar) -+ { -+ set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); ++void iee_set_token_page_valid(unsigned long token, unsigned long token_page, unsigned int order) ++{ ++ set_memory_4k(token, 1 << order); ++ set_memory_4k(token_page, 1 << order); ++ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, token); ++ p4d_t *p4dp = p4d_offset(pgdp, token); ++ pud_t *pudp = pud_offset(p4dp, token); ++ pmd_t *token_pmdp = pmd_offset(pudp, token); ++ pte_t *token_ptep = pte_offset_kernel(token_pmdp, token); ++ ++ if (!token_page) ++ panic("Token of task_struct was unset.\n"); ++ ++ pgdp = pgd_offset_pgd(pgdir, token_page); ++ p4dp = p4d_offset(pgdp, token_page); ++ pudp = pud_offset(p4dp, token_page); ++ pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); ++ pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); ++ ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_TOKEN, token_ptep, token_page_ptep, token_page, order); ++#else ++ for (int i = 0; i < (0x1 << order); i++) { ++ pte_t pte = READ_ONCE(*token_ptep); ++ ++ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | ++ (__phys_to_pfn(__pa(token_page + i * PAGE_SIZE)) << PAGE_SHIFT)); ++ WRITE_ONCE(*token_ptep, pte); ++ pte = READ_ONCE(*token_page_ptep); ++ pte = __pte((pte_val(pte) & ~__RW) & ~___D); ++ WRITE_ONCE(*token_page_ptep, pte); ++ token_ptep++; ++ token_page_ptep++; + } -+ #endif ++#endif + -+ #ifdef CONFIG_KEYP -+ if(s == key_jar) -+ { -+ set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); ++ flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); ++ flush_tlb_kernel_range(token_page, token_page + (PAGE_SIZE * (1 << order))); ++} ++ ++void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, unsigned int order) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, token); ++ p4d_t *p4dp = p4d_offset(pgdp, token); ++ pud_t *pudp = pud_offset(p4dp, token); ++ pmd_t *token_pmdp = pmd_offset(pudp, token); ++ pte_t *token_ptep = pte_offset_kernel(token_pmdp, token); ++ unsigned long token_page = (unsigned long)page_address(pte_page(*token_ptep)); ++ ++ if (!token_page) ++ panic("Token of task_struct was unset.\n"); ++ ++ pgdp = pgd_offset_pgd(pgdir, token_page); ++ p4dp = p4d_offset(pgdp, token_page); ++ pudp = pud_offset(p4dp, token_page); ++ pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); ++ pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); ++ ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_UNSET_TOKEN, token_ptep, token_page_ptep, token, order); ++#else ++ for (int i = 0; i < (0x1 << order); i++) { ++ pte_t pte = READ_ONCE(*token_ptep); ++ ++ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | ++ (__phys_to_pfn(__iee_pa(token + i * PAGE_SIZE)) << PAGE_SHIFT)); ++ WRITE_ONCE(*token_ptep, pte); ++ pte = READ_ONCE(*token_page_ptep); ++ pte = __pte(pte_val(pte) | ___D | __RW); ++ WRITE_ONCE(*token_page_ptep, pte); ++ token_ptep++; ++ token_page_ptep++; + } -+ #endif ++#endif ++ free_pages(token_page, order); ++ flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); ++ flush_tlb_kernel_range(token_page, token_page + (PAGE_SIZE * (1 << order))); ++} + -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(s == policy_jar) -+ { -+ set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); ++struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, ++ struct slab *slab, unsigned int order) ++{ ++ if (!slab || s != task_struct_cachep) ++ return slab; ++ ++ struct folio *folio = slab_folio(slab); ++ unsigned long token_addr = __slab_to_iee(slab); ++ unsigned long alloc_token = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); ++ ++ /* Allocation of task_struct and token pages must be done at the same time. */ ++ if (!alloc_token) { ++ /* Failed on allocation of token page. Free the allocated ones, ++ * return and try smaller order. ++ */ ++ __slab_clear_pfmemalloc(slab); ++ folio->mapping = NULL; ++ /* Make the mapping reset visible before clearing the flag */ ++ smp_wmb(); ++ __folio_clear_slab(folio); ++ __free_pages((struct page *)folio, order); ++ return NULL; + } -+ #endif -+ #else -+ #ifdef CONFIG_KOI -+ if (s == task_struct_cachep) { -+#ifdef CONFIG_X86_64 -+ void *token_addr = (void *)__phys_to_koi(page_to_phys(folio_page(slab_folio(slab), 0))); -+ void *alloc_token = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); -+ koi_add_page_mapping(token_addr, alloc_token, order); -+#else -+ int i; -+ for (i = 0; i < (0x1 << order); i++) { -+ unsigned long token_addr = __phys_to_virt(page_to_phys(folio_page(slab_folio(slab), i))) + (unsigned long)KOI_OFFSET; -+ unsigned long alloc_token = __get_free_page(GFP_KERNEL | __GFP_ZERO); -+ koi_add_page_mapping(token_addr, alloc_token); -+ } -+#endif -+ } -+ #endif -+ #endif + - account_slab(slab, oo_order(oo), s, flags); - - slab->slab_cache = s; -@@ -2091,6 +2339,114 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) - flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); - } - -+#if defined(CONFIG_X86_64) && defined (CONFIG_IEE) -+struct iee_free_slab_work { -+ struct work_struct work; -+ struct kmem_cache *s; -+ struct slab *slab; -+}; -+static void iee_free_task_struct_slab(struct work_struct *work) ++ /* Map allocated token pages to token addresses. */ ++ iee_set_token_page_valid(token_addr, alloc_token, order); ++ return slab; ++} ++ ++void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, pgd_t *pgd) ++{ ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); ++ ++ token->pgd = pgd; ++} ++ ++void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); ++ ++ token->pgd = NULL; ++ token->valid = false; ++} ++ ++void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) +{ -+ struct iee_free_slab_work *iee_free_slab_work = container_of(work, struct iee_free_slab_work, work); -+ struct kmem_cache *s = iee_free_slab_work->s; -+ struct slab *slab = iee_free_slab_work->slab; -+ struct folio *folio = slab_folio(slab); -+ int order = folio_order(folio); -+ // Free stack and tmp page. -+ int i; -+ void *start = fixup_red_left(s, page_address(folio_page(folio, 0))); -+ void *obj; -+ void *iee_stack; -+ void *tmp_page; -+ void *token; -+ for(i = 0; i < oo_objects(s->oo); i++) -+ { -+ obj = start + s->random_seq[i]; -+ iee_stack = (void *)iee_read_token_stack((struct task_struct *)obj); -+ if (iee_stack) { -+ free_iee_stack((void *)(iee_stack - PAGE_SIZE * 4)); -+ } -+ tmp_page = iee_read_tmp_page((struct task_struct *)obj); -+ free_pages((unsigned long)tmp_page, 0); -+ } -+ // Free token. -+ token = (void *)__phys_to_iee(page_to_phys(folio_page(folio, 0))); -+ iee_set_token_page_invalid(token, NULL, order); -+ __free_pages(&folio->page, order); -+ kfree(iee_free_slab_work); ++ struct task_token *token = (struct task_token *)__addr_to_iee(tsk); ++ ++ if (token->valid) ++ pr_err("IEE: validate token for multiple times."); ++ token->valid = true; +} ++ +#ifdef CONFIG_PTP -+static void iee_free_pgtable_slab(struct work_struct *work) ++void _iee_unset_token(unsigned long __unused, pte_t *token_ptep, ++ pte_t *token_page_ptep, unsigned long token, unsigned int order) +{ -+ struct iee_free_slab_work *iee_free_slab_work = container_of(work, struct iee_free_slab_work, work); -+ struct slab *slab = iee_free_slab_work->slab; -+ struct folio *folio = slab_folio(slab); -+ int order = folio_order(folio); -+ unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ __free_pages(&folio->page, order); -+ kfree(iee_free_slab_work); ++ token_ptep = (pte_t *)__addr_to_iee(token_ptep); ++ token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); ++ ++ for (int i = 0; i < (0x1 << order); i++) { ++ pte_t pte = READ_ONCE(*token_ptep); ++ ++ pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | ++ (__phys_to_pfn(__iee_pa(token + i * PAGE_SIZE)) << PAGE_SHIFT)); ++ WRITE_ONCE(*token_ptep, pte); ++ pte = READ_ONCE(*token_page_ptep); ++ pte = __pte(pte_val(pte) | ___D | __RW); ++ WRITE_ONCE(*token_page_ptep, pte); ++ token_ptep++; ++ token_page_ptep++; ++ } +} -+#endif + -+#ifdef CONFIG_CREDP -+static void iee_free_cred_slab(struct work_struct *work) ++void _iee_set_token(unsigned long __unused, pte_t *token_ptep, ++ pte_t *token_page_ptep, unsigned long token_page, unsigned int order) +{ -+ struct iee_free_slab_work *iee_free_slab_work = container_of(work, struct iee_free_slab_work, work); -+ struct slab *slab = iee_free_slab_work->slab; -+ struct folio *folio = slab_folio(slab); -+ int order = folio_order(folio); -+ unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ __free_pages(&folio->page, order); -+ kfree(iee_free_slab_work); ++ token_ptep = (pte_t *)__addr_to_iee(token_ptep); ++ token_page_ptep = (pte_t *)__addr_to_iee(token_page_ptep); ++ ++ for (int i = 0; i < (0x1 << order); i++) { ++ pte_t pte = READ_ONCE(*token_ptep); ++ ++ pte = __pte(((pte_val(pte) & ~PTE_PFN_MASK)) | ++ (__phys_to_pfn(__pa(token_page + i * PAGE_SIZE)) << PAGE_SHIFT)); ++ WRITE_ONCE(*token_ptep, pte); ++ pte = READ_ONCE(*token_page_ptep); ++ pte = __pte((pte_val(pte) & ~__RW) & ~___D); ++ WRITE_ONCE(*token_page_ptep, pte); ++ token_ptep++; ++ token_page_ptep++; ++ } +} +#endif -+static void iee_free_slab(struct kmem_cache *s, struct slab *slab, void (*do_free_slab)(struct work_struct *work)) +diff --git a/arch/x86/kernel/haoc/iee/iee.c b/arch/x86/kernel/haoc/iee/iee.c +new file mode 100644 +index 000000000000..40dea77dd0c9 +--- /dev/null ++++ b/arch/x86/kernel/haoc/iee/iee.c +@@ -0,0 +1,24 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++ ++void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) +{ -+ struct iee_free_slab_work *iee_free_slab_work = kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); -+ if (!iee_free_slab_work) -+ panic("No mem for struct iee_free_slab_work"); -+ iee_free_slab_work->s = s; -+ iee_free_slab_work->slab = slab; -+ INIT_WORK(&iee_free_slab_work->work, do_free_slab); -+ schedule_work(&iee_free_slab_work->work); ++ memcpy(dst, src, n); +} -+#else -+#ifdef CONFIG_KOI -+extern void koi_remove_page_mapping(unsigned long token, void *__unused, unsigned long order); -+#ifdef CONFIG_X86_64 -+struct koi_free_slab_work { -+ struct work_struct work; -+ struct kmem_cache *s; -+ struct slab *slab; -+}; -+static void koi_free_task_struct_slab(struct work_struct *work) ++ ++void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n) +{ -+ struct koi_free_slab_work *koi_free_slab_work = container_of(work, struct koi_free_slab_work, work); -+ struct slab *slab = koi_free_slab_work->slab; -+ struct folio *folio = slab_folio(slab); -+ int order = folio_order(folio); -+ void *token; -+ // Free token. -+ token = (void *)__phys_to_koi(page_to_phys(folio_page(folio, 0))); -+ koi_remove_page_mapping((unsigned long)token, NULL, order); -+ __free_pages(&folio->page, order); -+ kfree(koi_free_slab_work); ++ memset(ptr, data, n); +} + -+static void koi_free_slab(struct kmem_cache *s, struct slab *slab, void (*do_free_slab)(struct work_struct *work)) ++void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr) +{ -+ struct koi_free_slab_work *koi_free_slab_work = kmalloc(sizeof(struct koi_free_slab_work), GFP_ATOMIC); -+ if (!koi_free_slab_work) -+ panic("No mem for struct koi_free_slab_work"); -+ koi_free_slab_work->s = s; -+ koi_free_slab_work->slab = slab; -+ INIT_WORK(&koi_free_slab_work->work, do_free_slab); -+ schedule_work(&koi_free_slab_work->work); ++ *pptr = ptr; +} -+#endif /* CONFIG_X86_64 */ -+#endif /* CONFIG_KOI */ -+#endif + - static void __free_slab(struct kmem_cache *s, struct slab *slab) - { - struct folio *folio = slab_folio(slab); -@@ -2104,6 +2460,118 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) - __folio_clear_slab(folio); - mm_account_reclaimed_pages(pages); - unaccount_slab(slab, order, s); ++unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, volatile unsigned long *addr) ++{ ++ kcsan_mb(); ++ instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); ++ return arch_test_and_clear_bit(nr, addr); ++} +diff --git a/arch/x86/kernel/haoc/ptp/Makefile b/arch/x86/kernel/haoc/ptp/Makefile +new file mode 100644 +index 000000000000..ab30a58e0e73 +--- /dev/null ++++ b/arch/x86/kernel/haoc/ptp/Makefile +@@ -0,0 +1 @@ ++obj-y += ptp.o ptp-gate.o +diff --git a/arch/x86/kernel/haoc/ptp/ident_map.c b/arch/x86/kernel/haoc/ptp/ident_map.c +new file mode 100644 +index 000000000000..bd2e3e6c82b6 +--- /dev/null ++++ b/arch/x86/kernel/haoc/ptp/ident_map.c +@@ -0,0 +1,182 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Helper routines for building identity mapping page tables. This is ++ * included by both the compressed kernel and the regular kernel. ++ */ + -+ #ifdef CONFIG_IEE -+ // If the page containing this token is empty, free it and restore iee&lm va. -+ if(s == task_struct_cachep) -+ { -+ #ifdef CONFIG_X86_64 -+ iee_free_slab(s, slab, iee_free_task_struct_slab); -+ return; -+ #else -+ // Free stack. -+ int i; -+ void *start = fixup_red_left(s, page_address(folio_page(folio, 0))); -+ void *obj; -+ void *iee_stack; -+ void *tmp_page; -+ void *token_addr; -+ for(i = 0; i < oo_objects(s->oo); i++) -+ { -+ obj = start + s->random_seq[i]; -+ tmp_page = iee_read_tmp_page((struct task_struct *)obj); -+ free_pages((unsigned long)tmp_page, 0); -+ iee_stack = (void *)iee_read_token_stack((struct task_struct *)obj); -+ if (iee_stack) { -+ free_iee_stack((void *)(iee_stack - PAGE_SIZE * 4)); -+ } -+ } -+ // Free token. -+ token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(folio, 0))); -+ { -+ pgd_t *pgdir = swapper_pg_dir; -+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token_addr); -+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token_addr); -+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token_addr); -+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token_addr); -+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token_addr); -+ void *token_page = page_address(pte_page(*ptep)); -+ iee_set_token_page_invalid(token_addr, token_page, order); -+ free_pages((unsigned long)token_page, order); -+ } -+ #endif -+ } -+ if (s == iee_stack_jar) -+ { -+ unset_iee_stack_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ } ++static inline void ptp_set_pte_pre_init(pte_t *ptep, pte_t pte) ++{ ++ WRITE_ONCE(*ptep, pte); ++} + -+ #ifdef CONFIG_PTP -+ if(s == pgtable_jar || s == pgd_jar) -+ { -+ #ifdef CONFIG_X86_64 -+ iee_free_slab(s, slab, iee_free_pgtable_slab); -+ return; -+ #else -+ unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ #endif -+ } -+ #endif ++static inline void ptp_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) ++{ ++ WRITE_ONCE(*pmdp, pmd); ++} + -+ #ifdef CONFIG_CREDP -+ if (s == cred_jar) -+ { -+ #ifdef CONFIG_X86_64 -+ iee_free_slab(s, slab, iee_free_cred_slab); -+ return; -+ #else -+ unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); -+ #endif -+ } -+ #endif -+ -+ #ifdef CONFIG_KEYP -+ if(s == key_jar) -+ { -+ unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ } -+ #endif ++static inline void ptp_set_pud_pre_init(pud_t *pudp, pud_t pud) ++{ ++ WRITE_ONCE(*pudp, pud); ++} + -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(s == policy_jar) -+ { -+ unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); -+ } -+ #endif ++static inline void ptp_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) ++{ ++ pgd_t pgd; + -+ #else -+ #ifdef CONFIG_KOI -+ if(s == task_struct_cachep) -+ { -+ #ifdef CONFIG_X86_64 -+ koi_free_slab(s, slab, koi_free_task_struct_slab); -+ return; -+ #else -+ int i; -+ for(i = 0; i < (0x1 << order); i++) -+ { -+ unsigned long token_addr = __phys_to_virt(page_to_phys(folio_page(folio, i))) + (unsigned long)KOI_OFFSET; -+ unsigned long flags; -+ local_irq_save(flags); -+ asm volatile("at s1e1r, %0"::"r"(token_addr)); -+ isb(); -+ unsigned long res = read_sysreg(par_el1); -+ local_irq_restore(flags); -+ if(!(res & 0x1)) { -+ koi_remove_page_mapping(token_addr, NULL, 0); -+ free_page((unsigned long)__va(res & PTE_ADDR_MASK)); -+ } -+ } -+ #endif ++ if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { ++ WRITE_ONCE(*p4dp, p4d); ++ return; + } -+ #endif -+ #endif + - __free_pages(&folio->page, order); - } - -@@ -3447,6 +3915,38 @@ static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, - 0, sizeof(void *)); - } - -+#ifdef CONFIG_IEE -+#ifdef CONFIG_KFENCE -+static bool is_iee_kmem_cache(struct kmem_cache *s) -+{ -+ if (s == iee_stack_jar) -+ return true; -+#ifdef CONFIG_PTP -+ else if (s == pgtable_jar || s == pgd_jar) -+ return true; -+#endif -+#ifdef CONFIG_CREDP -+ else if (s == cred_jar) -+ return true; -+#endif -+#ifdef CONFIG_KEYP -+ else if (s == key_jar) -+ return true; -+#endif -+#ifdef CONFIG_IEE_SELINUX_P -+ else if (s == policy_jar) -+ return true; -+#endif -+ return false; ++ pgd = native_make_pgd(native_p4d_val(p4d)); ++ pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); ++ WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); +} -+#else -+static bool is_iee_kmem_cache(struct kmem_cache *s) ++ ++static inline void ptp_set_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) +{ -+ return false; ++ WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); +} -+#endif // CONFIG_KFENCE -+#endif // CONFIG_IEE + - /* - * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) - * have the fastpath folded into their functions. So no function call -@@ -3468,10 +3968,18 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list - if (!s) - return NULL; - -+#ifdef CONFIG_IEE -+ /* Skip kfence_alloc for iee kmem caches. */ -+ if(is_iee_kmem_cache(s)) -+ goto slab_alloc; -+#endif - object = kfence_alloc(s, orig_size, gfpflags); - if (unlikely(object)) - goto out; - -+#ifdef CONFIG_IEE -+slab_alloc: -+#endif - object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); - - maybe_wipe_obj_freeptr(s, object); -@@ -3493,6 +4001,8 @@ static __fastpath_inline void *slab_alloc(struct kmem_cache *s, struct list_lru - return slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, addr, orig_size); - } - -+// u64 pgtable_alloc_count = 0; ++static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, ++ unsigned long addr, unsigned long end) ++{ ++ addr &= PMD_MASK; ++ for (; addr < end; addr += PMD_SIZE) { ++ pmd_t *pmd = pmd_page + pmd_index(addr); + - static __fastpath_inline - void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, - gfp_t gfpflags) -@@ -3501,6 +4011,13 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, - - trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); - -+ // if (s == pgtable_jar && (pgtable_alloc_count % 100 == 0)) { -+ // printk("IEE TEST: object 0x%llx slab 0x%llx obj_to_page 0x%llx", -+ // (u64)ret, page_address((struct page*)virt_to_slab(ret)), -+ // page_address(virt_to_page(ret))); -+ // pgtable_alloc_count++; -+ // } ++ if (pmd_present(*pmd)) ++ continue; + - return ret; - } - -@@ -3951,6 +4468,11 @@ static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, - local_lock_irqsave(&s->cpu_slab->lock, irqflags); - - for (i = 0; i < size; i++) { -+ #ifdef CONFIG_IEE -+ /* Skip kfence_alloc for iee kmem caches. */ -+ if(is_iee_kmem_cache(s)) -+ goto slab_alloc; -+ #endif - void *object = kfence_alloc(s, s->object_size, flags); - - if (unlikely(object)) { -@@ -3958,6 +4480,9 @@ static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, - continue; - } - -+ #ifdef CONFIG_IEE -+ slab_alloc: -+ #endif - object = c->freelist; - if (unlikely(!object)) { - /* -@@ -4291,6 +4816,36 @@ static void early_kmem_cache_node_alloc(int node) - __add_partial(n, slab, DEACTIVATE_TO_HEAD); - } - -+#ifdef CONFIG_PTP -+void early_pgtable_jar_alloc(struct kmem_cache *pgtable_jar) ++ ptp_set_pmd_pre_init(pmd, __pmd((addr - info->offset) | info->page_flag)); ++ } ++} ++ ++static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, ++ unsigned long addr, unsigned long end) +{ -+ struct slab *slab; -+ int node = 0; -+ int i = 0; ++ unsigned long next; + -+ for(i = 0; i < nr_cpu_ids; i++) -+ { -+ node = cpupid_to_nid(i); -+ slab = new_slab(pgtable_jar, GFP_NOWAIT | __GFP_ZERO, node); ++ for (; addr < end; addr = next) { ++ pud_t *pud = pud_page + pud_index(addr); ++ pmd_t *pmd; + -+ BUG_ON(!slab); -+ if (slab_nid(slab) != node) { -+ pr_err("SLUB: Unable to allocate memory from node %d\n", node); -+ pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); -+ } ++ next = (addr & PUD_MASK) + PUD_SIZE; ++ if (next > end) ++ next = end; ++ ++ if (info->direct_gbpages) { ++ pud_t pudval; + -+ slab->inuse = 0; -+ inc_slabs_node(pgtable_jar, node, slab->objects); ++ if (pud_present(*pud)) ++ continue; + -+ /* -+ * No locks need to be taken here as it has just been -+ * initialized and there is no concurrent access. -+ */ -+ __add_partial(get_node(pgtable_jar, slab_nid(slab)), slab, DEACTIVATE_TO_HEAD); -+ } -+} -+#endif ++ addr &= PUD_MASK; ++ pudval = __pud((addr - info->offset) | info->page_flag); ++ ptp_set_pud_pre_init(pud, pudval); ++ continue; ++ } + - static void free_kmem_cache_nodes(struct kmem_cache *s) - { - int node; -@@ -4484,6 +5039,31 @@ static int calculate_sizes(struct kmem_cache *s) - s->size = size; - s->reciprocal_size = reciprocal_value(size); - order = calculate_order(size); -+ -+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) -+ if(strcmp(s->name, "task_struct") == 0) -+ order = HUGE_PMD_ORDER; -+ if(strcmp(s->name, "iee_stack_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ #endif -+ #ifdef CONFIG_PTP -+ if(strcmp(s->name, "pgtable_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ if(strcmp(s->name, "pgd_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ #endif -+ #ifdef CONFIG_CREDP -+ if(strcmp(s->name, "cred_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ #endif -+ #ifdef CONFIG_KEYP -+ if(strcmp(s->name, "key_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ #endif -+ #ifdef CONFIG_IEE_SELINUX_P -+ if(strcmp(s->name, "policy_jar") == 0) -+ order = HUGE_PMD_ORDER; -+ #endif - - if ((int)order < 0) - return 0; -@@ -4546,6 +5126,23 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) - s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); - s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); - -+ #ifdef CONFIG_IEE -+ if(strcmp(s->name, "task_struct") == 0) -+ s->min_partial *= (1 << TASK_ORDER); -+ if(strcmp(s->name, "iee_stack_jar") == 0) -+ s->min_partial *= (1 << TASK_ORDER); -+ #endif -+ #ifdef CONFIG_PTP -+ if(strcmp(s->name, "pgtable_jar") == 0) -+ s->min_partial = (1 << HUGE_PMD_ORDER); -+ if(strcmp(s->name, "pgd_jar") == 0) -+ s->min_partial = (1 << HUGE_PMD_ORDER); -+ #endif -+ #ifdef CONFIG_KEYP -+ if(strcmp(s->name, "key_jar") == 0) -+ s->min_partial = (1 << TASK_ORDER); -+ #endif ++ if (pud_present(*pud)) { ++ pmd = pmd_offset(pud, 0); ++ ident_pmd_init(info, pmd, addr, next); ++ continue; ++ } ++ pmd = (pmd_t *)info->alloc_pgt_page(info->context); ++ if (!pmd) ++ return -ENOMEM; ++ ident_pmd_init(info, pmd, addr, next); ++ ptp_set_pud_pre_init(pud, __pud(__pa(pmd) | info->kernpg_flag)); ++ } + - set_cpu_partial(s); - - #ifdef CONFIG_NUMA -@@ -5119,6 +5716,13 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags) - if (err) - return err; - -+ #ifdef CONFIG_PTP -+ if(strcmp(s->name, "pgtable_jar") == 0) -+ pgtable_jar_offset = s->offset; -+ if (strcmp(s->name, "pgd_jar") == 0) -+ pgd_jar_offset = s->offset; -+ #endif ++ return 0; ++} + - /* Mutex is not taken during early boot */ - if (slab_state <= UP) - return 0; -diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c -index 2628fc02be08..8601b88ab96f 100644 ---- a/mm/sparse-vmemmap.c -+++ b/mm/sparse-vmemmap.c -@@ -28,6 +28,10 @@ - #include - #include - -+#ifdef CONFIG_PTP -+#include -+#endif ++static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, ++ unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ int result; + - #include - #include - -@@ -146,6 +150,7 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, - struct page *reuse) - { - pte_t *pte = pte_offset_kernel(pmd, addr); ++ for (; addr < end; addr = next) { ++ p4d_t *p4d = p4d_page + p4d_index(addr); ++ pud_t *pud; + - if (pte_none(ptep_get(pte))) { - pte_t entry; - void *p; -@@ -167,6 +172,9 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, - get_page(reuse); - p = page_to_virt(reuse); - } -+ #ifdef CONFIG_PTP -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(p))); -+ #endif - entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); - set_pte_at(&init_mm, addr, pte, entry); - } -@@ -181,6 +189,10 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) - return NULL; - memset(p, 0, size); - -+ #ifdef CONFIG_PTP -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(p))); -+ #endif -+ - return p; - } - -diff --git a/mm/vmalloc.c b/mm/vmalloc.c -index cb0951fea238..6cefa902facb 100644 ---- a/mm/vmalloc.c -+++ b/mm/vmalloc.c -@@ -3458,7 +3458,7 @@ static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) - - if (WARN_ON_ONCE(pfn_valid(pfn))) - return -EINVAL; -- -+ - ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); - set_pte_at(&init_mm, addr, pte, ptent); - -diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c -index 4c6441536d55..2483f38d6eca 100644 ---- a/net/ceph/ceph_common.c -+++ b/net/ceph/ceph_common.c -@@ -385,7 +385,11 @@ static int get_secret(struct ceph_crypto_key *dst, const char *name, - goto out; - } - -+ #ifdef CONFIG_KEYP -+ ckey = ((union key_payload *)(ukey->name_link.next))->data[0]; -+ #else - ckey = ukey->payload.data[0]; -+ #endif - err = ceph_crypto_key_clone(dst, ckey); - if (err) - goto out_key; -diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c -index 051d22c0e4ad..cd5839ea3000 100644 ---- a/net/ceph/crypto.c -+++ b/net/ceph/crypto.c -@@ -336,7 +336,11 @@ static void ceph_key_free_preparse(struct key_preparsed_payload *prep) - - static void ceph_key_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ struct ceph_crypto_key *ckey = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - struct ceph_crypto_key *ckey = key->payload.data[0]; -+ #endif - - ceph_crypto_key_destroy(ckey); - kfree(ckey); -diff --git a/net/core/filter.c b/net/core/filter.c -index 48dd2896ee1d..4ef815d885d4 100644 ---- a/net/core/filter.c -+++ b/net/core/filter.c -@@ -569,7 +569,7 @@ static int bpf_convert_filter(struct sock_filter *prog, int len, - u8 bpf_src; - - BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK); -- BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); -+ BUILD_BUG_ON(BPF_REG_FP + 2 != MAX_BPF_REG); - - if (len <= 0 || len > BPF_MAXINSNS) - return -EINVAL; -@@ -9699,25 +9699,41 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, - break; - - case offsetof(struct __sk_buff, data): -+ #ifdef CONFIG_HIVE -+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sfi_data), -+ si->dst_reg, si->src_reg, -+ offsetof(struct sk_buff, sfi_data)); -+ #else - *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), - si->dst_reg, si->src_reg, - offsetof(struct sk_buff, data)); -+ #endif - break; - - case offsetof(struct __sk_buff, data_meta): - off = si->off; - off -= offsetof(struct __sk_buff, data_meta); -+ #ifdef CONFIG_HIVE -+ off += offsetof(struct sk_buff, sfi_data_meta); -+ *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, -+ si->src_reg, off); -+ #else - off += offsetof(struct sk_buff, cb); - off += offsetof(struct bpf_skb_data_end, data_meta); - *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, - si->src_reg, off); -+ #endif - break; - - case offsetof(struct __sk_buff, data_end): - off = si->off; - off -= offsetof(struct __sk_buff, data_end); -+ #ifdef CONFIG_HIVE -+ off += offsetof(struct sk_buff, sfi_data_end); -+ #else - off += offsetof(struct sk_buff, cb); - off += offsetof(struct bpf_skb_data_end, data_end); -+ #endif - *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, - si->src_reg, off); - break; -diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c -index c42ddd85ff1f..8450eb924b62 100644 ---- a/net/dns_resolver/dns_key.c -+++ b/net/dns_resolver/dns_key.c -@@ -34,6 +34,13 @@ - #include - #include "internal.h" - -+#ifdef CONFIG_CREDP -+#include -+#endif -+#ifdef CONFIG_KEYP -+#include -+#endif ++ next = (addr & P4D_MASK) + P4D_SIZE; ++ if (next > end) ++ next = end; + - MODULE_DESCRIPTION("DNS Resolver"); - MODULE_AUTHOR("Wang Lei"); - MODULE_LICENSE("GPL"); -@@ -295,7 +302,11 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) - { - seq_puts(m, key->description); - if (key_is_positive(key)) { -+ #ifdef CONFIG_KEYP -+ int err = PTR_ERR(((union key_payload *)(key->name_link.next))->data[dns_key_error]); -+ #else - int err = PTR_ERR(key->payload.data[dns_key_error]); -+ #endif - - if (err) - seq_printf(m, ": %d", err); -@@ -311,7 +322,11 @@ static void dns_resolver_describe(const struct key *key, struct seq_file *m) - static long dns_resolver_read(const struct key *key, - char *buffer, size_t buflen) - { -+ #ifdef CONFIG_KEYP -+ int err = PTR_ERR(((union key_payload *)(key->name_link.next))->data[dns_key_error]); -+ #else - int err = PTR_ERR(key->payload.data[dns_key_error]); -+ #endif - - if (err) - return err; -@@ -364,9 +379,18 @@ static int __init init_dns_resolver(void) - - /* instruct request_key() to use this special keyring as a cache for - * the results it looks up */ -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(keyring, KEY_FLAG_ROOT_CAN_CLEAR, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); -+ #endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(cred,keyring); -+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); -+ #else - cred->thread_keyring = keyring; - cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; -+ #endif - dns_resolver_cache = cred; - - kdebug("DNS resolver keyring: %d\n", key_serial(keyring)); -diff --git a/net/dns_resolver/dns_query.c b/net/dns_resolver/dns_query.c -index 82b084cc1cc6..2bebc6eef501 100644 ---- a/net/dns_resolver/dns_query.c -+++ b/net/dns_resolver/dns_query.c -@@ -47,6 +47,10 @@ - - #include "internal.h" - -+#ifdef CONFIG_KEYP -+#include -+#endif ++ if (p4d_present(*p4d)) { ++ pud = pud_offset(p4d, 0); ++ result = ident_pud_init(info, pud, addr, next); ++ if (result) ++ return result; + - /** - * dns_query - Query the DNS - * @net: The network namespace to operate in. -@@ -133,16 +137,26 @@ int dns_query(struct net *net, - goto out; - } - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(rkey)); -+ iee_set_key_flag_bit(rkey, KEY_FLAG_ROOT_CAN_INVAL, SET_BIT_OP); -+ iee_set_key_perm(rkey, rkey->perm | KEY_USR_VIEW); -+ #else - down_read(&rkey->sem); - set_bit(KEY_FLAG_ROOT_CAN_INVAL, &rkey->flags); - rkey->perm |= KEY_USR_VIEW; -+ #endif - - ret = key_validate(rkey); - if (ret < 0) - goto put; - - /* If the DNS server gave an error, return that to the caller */ -+ #ifdef CONFIG_KEYP -+ ret = PTR_ERR(((union key_payload *)(rkey->name_link.next))->data[dns_key_error]); -+ #else - ret = PTR_ERR(rkey->payload.data[dns_key_error]); -+ #endif - if (ret) - goto put; - -@@ -161,7 +175,11 @@ int dns_query(struct net *net, - - ret = len; - put: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(rkey)); -+ #else - up_read(&rkey->sem); -+ #endif - if (invalidate) - key_invalidate(rkey); - key_put(rkey); -diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c -index fa8aec78f63d..ba9d2feace7a 100644 ---- a/net/rxrpc/af_rxrpc.c -+++ b/net/rxrpc/af_rxrpc.c -@@ -307,7 +307,11 @@ struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock, - - if (!key) - key = rx->key; -+ #ifdef CONFIG_KEYP -+ if (key && !((union key_payload *)(key->name_link.next))->data[0]) -+ #else - if (key && !key->payload.data[0]) -+ #endif - key = NULL; /* a no-security key */ - - memset(&p, 0, sizeof(p)); -diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c -index 598b4ee389fc..1d9574406c16 100644 ---- a/net/rxrpc/conn_event.c -+++ b/net/rxrpc/conn_event.c -@@ -247,8 +247,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, - if (ret < 0) - return ret; - -+ #ifdef CONFIG_KEYP -+ ret = conn->security->init_connection_security( -+ conn, ((union key_payload *)(conn->key->name_link.next))->data[0]); -+ #else - ret = conn->security->init_connection_security( - conn, conn->key->payload.data[0]); -+ #endif - if (ret < 0) - return ret; - -diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c -index 33e8302a79e3..c4b755733ed2 100644 ---- a/net/rxrpc/key.c -+++ b/net/rxrpc/key.c -@@ -413,7 +413,11 @@ static void rxrpc_free_preparse(struct key_preparsed_payload *prep) - */ - static void rxrpc_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ rxrpc_free_token_list(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - rxrpc_free_token_list(key->payload.data[0]); -+ #endif - } - - /* -@@ -426,7 +430,11 @@ static void rxrpc_describe(const struct key *key, struct seq_file *m) - - seq_puts(m, key->description); - -+ #ifdef CONFIG_KEYP -+ for (token = ((union key_payload *)(key->name_link.next))->data[0]; token; token = token->next) { -+ #else - for (token = key->payload.data[0]; token; token = token->next) { -+ #endif - seq_puts(m, sep); - - switch (token->security_index) { -@@ -584,7 +592,11 @@ static long rxrpc_read(const struct key *key, - size += 1 * 4; /* token count */ - - ntoks = 0; -+ #ifdef CONFIG_KEYP -+ for (token = ((union key_payload *)(key->name_link.next))->data[0]; token; token = token->next) { -+ #else - for (token = key->payload.data[0]; token; token = token->next) { -+ #endif - toksize = 4; /* sec index */ - - switch (token->security_index) { -@@ -654,7 +666,11 @@ static long rxrpc_read(const struct key *key, - ENCODE(ntoks); - - tok = 0; -+ #ifdef CONFIG_KEYP -+ for (token = ((union key_payload *)(key->name_link.next))->data[0]; token; token = token->next) { -+ #else - for (token = key->payload.data[0]; token; token = token->next) { -+ #endif - toksize = toksizes[tok++]; - ENCODE(toksize); - oldxdr = xdr; -diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c -index ad6c57a9f27c..3161cbf4a958 100644 ---- a/net/rxrpc/rxkad.c -+++ b/net/rxrpc/rxkad.c -@@ -21,6 +21,10 @@ - #include - #include "ar-internal.h" - -+#ifdef CONFIG_KEYP -+#include -+#endif ++ continue; ++ } ++ pud = (pud_t *)info->alloc_pgt_page(info->context); ++ if (!pud) ++ return -ENOMEM; + - #define RXKAD_VERSION 2 - #define MAXKRB5TICKETLEN 1024 - #define RXKAD_TKT_TYPE_KERBEROS_V5 256 -@@ -88,10 +92,18 @@ static void rxkad_free_preparse_server_key(struct key_preparsed_payload *prep) - - static void rxkad_destroy_server_key(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ if (((union key_payload *)(key->name_link.next))->data[0]) { -+ crypto_free_skcipher(((union key_payload *)(key->name_link.next))->data[0]); -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[0] = NULL; ++ result = ident_pud_init(info, pud, addr, next); ++ if (result) ++ return result; ++ ++ ptp_set_p4d_pre_init(p4d, __p4d(__pa(pud) | info->kernpg_flag)); + } -+ #else - if (key->payload.data[0]) { - crypto_free_skcipher(key->payload.data[0]); - key->payload.data[0] = NULL; - } -+ #endif - } - - /* -@@ -205,7 +217,11 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, - return -ENOMEM; - } - -+ #ifdef CONFIG_KEYP -+ token = ((union key_payload *)(conn->key->name_link.next))->data[0]; -+ #else - token = conn->key->payload.data[0]; -+ #endif - memcpy(&iv, token->kad->session_key, sizeof(iv)); - - tmpbuf[0] = htonl(conn->proto.epoch); -@@ -317,7 +333,11 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, - } - - /* encrypt from the session key */ -+ #ifdef CONFIG_KEYP -+ token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; -+ #else - token = call->conn->key->payload.data[0]; -+ #endif - memcpy(&iv, token->kad->session_key, sizeof(iv)); - - sg_init_one(&sg, txb->data, txb->len); -@@ -507,7 +527,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, - } - - /* decrypt from the session key */ -+ #ifdef CONFIG_KEYP -+ token = ((union key_payload *)(call->conn->key->name_link.next))->data[0]; -+ #else - token = call->conn->key->payload.data[0]; -+ #endif - memcpy(&iv, token->kad->session_key, sizeof(iv)); - - skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); -@@ -824,7 +848,11 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, - return rxrpc_abort_conn(conn, skb, RXKADLEVELFAIL, -EACCES, - rxkad_abort_chall_level); - -+ #ifdef CONFIG_KEYP -+ token = ((union key_payload *)(conn->key->name_link.next))->data[0]; -+ #else - token = conn->key->payload.data[0]; -+ #endif - - /* build the response packet */ - resp = kzalloc(sizeof(struct rxkad_response), GFP_NOFS); -@@ -876,12 +904,24 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, - - *_expiry = 0; - -+ #ifdef CONFIG_KEYP -+ ASSERT(((union key_payload *)(server_key->name_link.next))->data[0] != NULL); -+ #else - ASSERT(server_key->payload.data[0] != NULL); -+ #endif - ASSERTCMP((unsigned long) ticket & 7UL, ==, 0); - -+ #ifdef CONFIG_KEYP -+ memcpy(&iv, &((union key_payload *)(server_key->name_link.next))->data[2], sizeof(iv)); -+ #else - memcpy(&iv, &server_key->payload.data[2], sizeof(iv)); -+ #endif - -+ #ifdef CONFIG_KEYP -+ req = skcipher_request_alloc(((union key_payload *)(server_key->name_link.next))->data[0], GFP_NOFS); -+ #else - req = skcipher_request_alloc(server_key->payload.data[0], GFP_NOFS); -+ #endif - if (!req) - return -ENOMEM; - -diff --git a/net/rxrpc/security.c b/net/rxrpc/security.c -index cb8dd1d3b1d4..6bffe9965040 100644 ---- a/net/rxrpc/security.c -+++ b/net/rxrpc/security.c -@@ -79,7 +79,11 @@ int rxrpc_init_client_call_security(struct rxrpc_call *call) - if (ret < 0) - return ret; - -+ #ifdef CONFIG_KEYP -+ for (token = ((union key_payload *)(key->name_link.next))->data[0]; token; token = token->next) { -+ #else - for (token = key->payload.data[0]; token; token = token->next) { -+ #endif - sec = rxrpc_security_lookup(token->security_index); - if (sec) - goto found; -@@ -103,7 +107,11 @@ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn) - - _enter("{%d},{%x}", conn->debug_id, key_serial(key)); - -+ #ifdef CONFIG_KEYP -+ for (token = ((union key_payload *)(key->name_link.next))->data[0]; token; token = token->next) { -+ #else - for (token = key->payload.data[0]; token; token = token->next) { -+ #endif - if (token->security_index == conn->security->security_index) - goto found; - } -diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c -index 24f765d243db..4369ed7bf34b 100644 ---- a/net/rxrpc/sendmsg.c -+++ b/net/rxrpc/sendmsg.c -@@ -585,7 +585,11 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, - } - - key = rx->key; -+ #ifdef CONFIG_KEYP -+ if (key && !((union key_payload *)(rx->key->name_link.next))->data[0]) -+ #else - if (key && !rx->key->payload.data[0]) -+ #endif - key = NULL; - - memset(&cp, 0, sizeof(cp)); -diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c -index e51940589ee5..1ea7e51b71f0 100644 ---- a/net/rxrpc/server_key.c -+++ b/net/rxrpc/server_key.c -@@ -100,7 +100,11 @@ static void rxrpc_free_preparse_s(struct key_preparsed_payload *prep) - - static void rxrpc_destroy_s(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; -+ #else - const struct rxrpc_security *sec = key->payload.data[1]; -+ #endif - - if (sec && sec->destroy_server_key) - sec->destroy_server_key(key); -@@ -108,7 +112,11 @@ static void rxrpc_destroy_s(struct key *key) - - static void rxrpc_describe_s(const struct key *key, struct seq_file *m) - { -+ #ifdef CONFIG_KEYP -+ const struct rxrpc_security *sec = ((union key_payload *)(key->name_link.next))->data[1]; -+ #else - const struct rxrpc_security *sec = key->payload.data[1]; -+ #endif - - seq_puts(m, key->description); - if (sec && sec->describe_server_key) -diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c -index 382c7a71f81f..9416a3c0ac66 100644 ---- a/net/sched/cls_bpf.c -+++ b/net/sched/cls_bpf.c -@@ -16,6 +16,10 @@ - #include - #include - -+#ifdef CONFIG_HIVE -+#include -+#endif + - #include - #include - #include -@@ -29,6 +33,10 @@ MODULE_DESCRIPTION("TC BPF based classifier"); - #define CLS_BPF_SUPPORTED_GEN_FLAGS \ - (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW) - -+#ifdef CONFIG_HIVE -+extern pte_t *bpf_sfi_get_ptep(u64 addr); -+#endif ++ return 0; ++} + - struct cls_bpf_head { - struct list_head plist; - struct idr handle_idr; -@@ -78,6 +86,70 @@ static int cls_bpf_exec_opcode(int code) - } - } - -+#ifdef CONFIG_HIVE -+static inline void bpf_sfi_map_skb(struct bpf_prog *prog, struct sk_buff *skb) ++int ptp_kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, ++ unsigned long pstart, unsigned long pend) +{ -+ u32 skb_data_offset, skb_meta_offset, skb_end_offset, reserved_skb_data_size; -+ u64 addr_to_map, map_length, map_page_cnt; -+ u64 start_time, end_time; -+ pte_t pte; -+ struct bpf_skb_data_end *cb; ++ unsigned long addr = pstart + info->offset; ++ unsigned long end = pend + info->offset; ++ unsigned long next; ++ int result; + -+ // skb_data_size = (u64)((struct bpf_skb_data_end *)skb->cb)->data_end - (u64)skb->head; -+ // skb_page_cnt = PAGE_ALIGN(skb_data_size) >> PAGE_SHIFT; -+ // pr_err("skb page cnt = %d\n", skb_page_cnt); -+ -+#ifdef CONFIG_ARM64 -+ isb(); -+#elif CONFIG_X86_64 -+ asm volatile("mfence":::"memory"); -+#endif -+ start_time = sched_clock(); -+ -+ // 1. get skb data size -+ cb = (struct bpf_skb_data_end *)skb->cb; -+ skb_data_offset = (u64)skb->data & ~PAGE_MASK; -+ skb_meta_offset = cb->data_meta - (void *)skb->data; -+ skb_end_offset = cb->data_end - (void *)skb->sfi_data_end; -+ map_length = skb->len; -+ // 2. ensure bpf_sfi reserved size is enough -+ reserved_skb_data_size = prog->shadow_skb_page_cnt * PAGE_SIZE; -+ BUG_ON(unlikely(reserved_skb_data_size < map_length)); -+ // 3. double map -+ map_page_cnt = PAGE_ALIGN(map_length) >> PAGE_SHIFT; -+ addr_to_map = (u64)prog->shadow_skb_addr; -+ // printk("skb %llx, %d page, map to %llx\n", (u64)skb->data, skb_page_cnt, addr_to_map); -+ for (int i = 0; i < map_page_cnt; i++) { -+ pte_t *origin_ptep = bpf_sfi_get_ptep((u64)skb->data + i * PAGE_SIZE); -+ if (unlikely(IS_ERR(origin_ptep))) { -+ pr_err("map pkt %llx failed\n", (u64)skb->data + i * PAGE_SIZE); -+ return; ++ /* Set the default pagetable flags if not supplied */ ++ if (!info->kernpg_flag) ++ info->kernpg_flag = _KERNPG_TABLE; ++ ++ /* Filter out unsupported __PAGE_KERNEL_* bits: */ ++ info->kernpg_flag &= __default_kernel_pte_mask; ++ ++ for (; addr < end; addr = next) { ++ pgd_t *pgd = pgd_page + pgd_index(addr); ++ p4d_t *p4d; ++ ++ next = (addr & PGDIR_MASK) + PGDIR_SIZE; ++ if (next > end) ++ next = end; ++ ++ if (pgd_present(*pgd)) { ++ p4d = p4d_offset(pgd, 0); ++ result = ident_p4d_init(info, p4d, addr, next); ++ if (result) ++ return result; ++ continue; + } -+ pte_t *sfi_ptep = bpf_sfi_get_ptep(addr_to_map + i * PAGE_SIZE); -+ if (unlikely(IS_ERR(sfi_ptep))) { -+ pr_err("map pkt %llx failed\n", addr_to_map + i * PAGE_SIZE); -+ return; ++ ++ p4d = (p4d_t *)info->alloc_pgt_page(info->context); ++ if (!p4d) ++ return -ENOMEM; ++ result = ident_p4d_init(info, p4d, addr, next); ++ if (result) ++ return result; ++ if (pgtable_l5_enabled()) { ++ ptp_set_pgd_pre_init(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); ++ } else { ++ /* ++ * With p4d folded, pgd is equal to p4d. ++ * The pgd entry has to point to the pud page table in this case. ++ */ ++ pud_t *pud = pud_offset(p4d, 0); ++ ++ ptp_set_pgd_pre_init(pgd, __pgd(__pa(pud) | info->kernpg_flag)); + } -+ pte = __pte(pte_val(*origin_ptep)); -+ set_pte(sfi_ptep, pte); + } -+ flush_tlb_kernel_range(addr_to_map, addr_to_map + map_page_cnt * PAGE_SIZE); -+ // skb->sfi_bpf_mapped = true; -+ skb->sfi_data = (void *)addr_to_map + skb_data_offset; -+ skb->sfi_data_meta = skb->sfi_data + skb_meta_offset; -+ skb->sfi_data_end = skb->sfi_data + skb_end_offset; + -+#ifdef CONFIG_ARM64 -+ isb(); -+#elif CONFIG_X86_64 -+ asm volatile("mfence":::"memory"); -+#endif -+ end_time = sched_clock(); -+ // pr_err("shadow packet region: %llx, %llx", addr_to_map, addr_to_map + map_page_cnt * PAGE_SIZE); -+ pr_err("shadow packet time = %lldns", end_time - start_time); -+} -+#endif ++ return 0; ++} +diff --git a/arch/x86/kernel/haoc/ptp/ptp-gate.S b/arch/x86/kernel/haoc/ptp/ptp-gate.S +new file mode 100644 +index 000000000000..4272574620d6 +--- /dev/null ++++ b/arch/x86/kernel/haoc/ptp/ptp-gate.S +@@ -0,0 +1,133 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#include ++#include ++#include ++#include ++#include ++ ++/* scratch_reg would be changed, ++caller should dertimine if scratch_reg should be saved and restored */ ++.macro DISABLE_WP scratch_reg:req ++ /* Disable write protection*/ ++ movq %cr0, %\scratch_reg ++ andq $(~X86_CR0_WP), %\scratch_reg ++ movq %\scratch_reg, %cr0 ++.endm ++ ++.macro ENABLE_WP scratch_reg:req ++ /* Enable write protection */ ++ movq %cr0, %\scratch_reg ++1: ++ orq $X86_CR0_WP, %\scratch_reg ++ movq %\scratch_reg, %cr0 ++ testq $X86_CR0_WP, %\scratch_reg ++ je 1b ++.endm ++ ++SYM_FUNC_START(ptp_xchg) ++ /* save RFLAGS, close irq */ ++ pushfq ++ cli ++ ++ pushq %r12 ++ ++#ifdef CONFIG_PTP ++ movq PER_CPU_VAR(iee_disables) + IEE_DISABLE, %r12 ++ testq %r12, %r12 ++ jz xchg_wp_enabled ++ ++ xchg %rsi, (%rdi) ++ movq %rsi, %rax + - TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb, - const struct tcf_proto *tp, - struct tcf_result *res) -@@ -98,10 +170,16 @@ TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb, - /* It is safe to push/pull even if skb_shared() */ - __skb_push(skb, skb->mac_len); - bpf_compute_data_pointers(skb); -+ #ifdef CONFIG_HIVE -+ bpf_sfi_map_skb(prog->filter, skb); -+ #endif - filter_res = bpf_prog_run(prog->filter, skb); - __skb_pull(skb, skb->mac_len); - } else { - bpf_compute_data_pointers(skb); -+ #ifdef CONFIG_HIVE -+ bpf_sfi_map_skb(prog->filter, skb); -+ #endif - filter_res = bpf_prog_run(prog->filter, skb); - } - if (unlikely(!skb->tstamp && skb->mono_delivery_time)) -diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c -index ec41b26af76e..c429a92dc3ef 100644 ---- a/net/sunrpc/auth.c -+++ b/net/sunrpc/auth.c -@@ -38,9 +38,13 @@ static const struct rpc_authops __rcu *auth_flavors[RPC_AUTH_MAXFLAVOR] = { - static LIST_HEAD(cred_unused); - static unsigned long number_cred_unused; - -+#ifdef CONFIG_CREDP -+struct cred* machine_cred; -+#else - static struct cred machine_cred = { - .usage = ATOMIC_INIT(1), - }; -+#endif - - /* - * Return the machine_cred pointer to be used whenever -@@ -48,7 +52,11 @@ static struct cred machine_cred = { - */ - const struct cred *rpc_machine_cred(void) - { -+ #ifdef CONFIG_CREDP -+ return machine_cred; -+ #else - return &machine_cred; -+ #endif - } - EXPORT_SYMBOL_GPL(rpc_machine_cred); - -@@ -871,6 +879,9 @@ static struct shrinker rpc_cred_shrinker = { - - int __init rpcauth_init_module(void) - { -+ #ifdef CONFIG_CREDP -+ machine_cred = prepare_kernel_cred(&init_task); -+ #endif - int err; - - err = rpc_init_authunix(); -@@ -888,6 +899,9 @@ int __init rpcauth_init_module(void) - - void rpcauth_remove_module(void) - { -+ #ifdef CONFIG_CREDP -+ abort_creds(machine_cred); -+ #endif - rpc_destroy_authunix(); - unregister_shrinker(&rpc_cred_shrinker); - } -diff --git a/security/commoncap.c b/security/commoncap.c -index bc0521104197..d7d3b7cc13e8 100644 ---- a/security/commoncap.c -+++ b/security/commoncap.c -@@ -26,6 +26,10 @@ - #include - #include - -+#ifdef CONFIG_CREDP -+#include ++ popq %r12 ++ popfq ++ jmp __x86_return_thunk ++ ++xchg_wp_enabled: +#endif ++ DISABLE_WP r12 + - /* - * If a non-root user executes a setuid-root binary in - * !secure(SECURE_NOROOT) mode, then we raise capabilities. -@@ -266,6 +270,15 @@ int cap_capset(struct cred *new, - if (!cap_issubset(*effective, *permitted)) - return -EPERM; - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,*effective); -+ iee_set_cred_cap_inheritable(new,*inheritable); -+ iee_set_cred_cap_permitted(new,*permitted); ++ xchg %rsi, (%rdi) ++ movq %rsi, %rax + -+ iee_set_cred_cap_ambient(new,cap_intersect(new->cap_ambient, -+ cap_intersect(*permitted, -+ *inheritable))); -+ #else - new->cap_effective = *effective; - new->cap_inheritable = *inheritable; - new->cap_permitted = *permitted; -@@ -277,6 +290,7 @@ int cap_capset(struct cred *new, - new->cap_ambient = cap_intersect(new->cap_ambient, - cap_intersect(*permitted, - *inheritable)); -+ #endif - if (WARN_ON(!cap_ambient_invariant_ok(new))) - return -EINVAL; - return 0; -@@ -601,9 +615,16 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps, - * pP' = (X & fP) | (pI & fI) - * The addition of pA' is handled later. - */ -+#ifdef CONFIG_CREDP -+ kernel_cap_t temp = new->cap_permitted; -+ temp.val = (new->cap_bset.val & caps->permitted.val) | -+ (new->cap_inheritable.val & caps->inheritable.val); -+ iee_set_cred_cap_permitted(new,temp); -+#else - new->cap_permitted.val = - (new->cap_bset.val & caps->permitted.val) | - (new->cap_inheritable.val & caps->inheritable.val); -+#endif - - if (caps->permitted.val & ~new->cap_permitted.val) - /* insufficient to execute correctly */ -@@ -726,7 +747,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, - int rc = 0; - struct cpu_vfs_cap_data vcaps; - -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = bprm->cred->cap_permitted; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_permitted(bprm->cred, tmp_cap); -+ } while (0); -+ #else - cap_clear(bprm->cred->cap_permitted); -+ #endif - - if (!file_caps_enabled) - return 0; -@@ -757,7 +786,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, - - out: - if (rc) -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = bprm->cred->cap_permitted; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_permitted(bprm->cred, tmp_cap); -+ } while (0); -+ #else - cap_clear(bprm->cred->cap_permitted); -+ #endif - - return rc; - } -@@ -809,8 +846,13 @@ static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap, - */ - if (__is_eff(root_uid, new) || __is_real(root_uid, new)) { - /* pP' = (cap_bset & ~0) | (pI & ~0) */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_permitted(new,cap_combine(old->cap_bset, -+ old->cap_inheritable)); -+ #else - new->cap_permitted = cap_combine(old->cap_bset, - old->cap_inheritable); -+ #endif - } - /* - * If only the real uid is 0, we do not set the effective bit. -@@ -919,34 +961,71 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) - /* downgrade; they get no more than they had, and maybe less */ - if (!ns_capable(new->user_ns, CAP_SETUID) || - (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_euid(new,new->uid); -+ iee_set_cred_egid(new,new->gid); -+ #else - new->euid = new->uid; - new->egid = new->gid; -+ #endif - } -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_permitted(new,cap_intersect(new->cap_permitted, -+ old->cap_permitted)); -+ #else - new->cap_permitted = cap_intersect(new->cap_permitted, - old->cap_permitted); -+ #endif - } - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_fsuid(new,new->euid); -+ iee_set_cred_suid(new,new->euid); -+ iee_set_cred_fsgid(new,new->egid); -+ iee_set_cred_sgid(new,new->egid); -+ #else - new->suid = new->fsuid = new->euid; - new->sgid = new->fsgid = new->egid; -+ #endif - - /* File caps or setid cancels ambient. */ - if (has_fcap || is_setid) -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = new->cap_ambient; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_ambient(new, tmp_cap); -+ } while (0); -+ #else - cap_clear(new->cap_ambient); -+ #endif - - /* - * Now that we've computed pA', update pP' to give: - * pP' = (X & fP) | (pI & fI) | pA' - */ -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_permitted(new,cap_combine(new->cap_permitted, new->cap_ambient)); -+ #else - new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient); -+ #endif - - /* - * Set pE' = (fE ? pP' : pA'). Because pA' is zero if fE is set, - * this is the same as pE' = (fE ? pP' : 0) | pA'. - */ - if (effective) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,new->cap_permitted); -+ #else - new->cap_effective = new->cap_permitted; -+ #endif - else -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,new->cap_ambient); -+ #else - new->cap_effective = new->cap_ambient; -+ #endif - - if (WARN_ON(!cap_ambient_invariant_ok(new))) - return -EPERM; -@@ -957,7 +1036,11 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) - return ret; - } - -+ #ifdef CONFIG_CREDP -+ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); -+ #else - new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); -+ #endif - - if (WARN_ON(!cap_ambient_invariant_ok(new))) - return -EPERM; -@@ -1092,8 +1175,21 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) - !uid_eq(new->euid, root_uid) && - !uid_eq(new->suid, root_uid))) { - if (!issecure(SECURE_KEEP_CAPS)) { -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = new->cap_permitted; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_permitted(new, tmp_cap); -+ } while (0); -+ do { -+ kernel_cap_t tmp_cap = new->cap_effective; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_effective(new, tmp_cap); -+ } while (0); -+ #else - cap_clear(new->cap_permitted); - cap_clear(new->cap_effective); -+ #endif - } - - /* -@@ -1101,12 +1197,32 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) - * by exec to drop capabilities. We should make sure that - * this remains the case. - */ -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = new->cap_ambient; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_ambient(new, tmp_cap); -+ } while (0); -+ #else - cap_clear(new->cap_ambient); -+ #endif - } - if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid)) -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = new->cap_effective; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_effective(new, tmp_cap); -+ } while (0); -+ #else - cap_clear(new->cap_effective); -+ #endif - if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,new->cap_permitted); -+ #else - new->cap_effective = new->cap_permitted; -+ #endif - } - - /** -@@ -1142,13 +1258,22 @@ int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags) - if (!issecure(SECURE_NO_SETUID_FIXUP)) { - kuid_t root_uid = make_kuid(old->user_ns, 0); - if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,cap_drop_fs_set(new->cap_effective)); -+ #else - new->cap_effective = - cap_drop_fs_set(new->cap_effective); -+ #endif - - if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid)) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_cap_effective(new,cap_raise_fs_set(new->cap_effective, -+ new->cap_permitted)); -+ #else - new->cap_effective = - cap_raise_fs_set(new->cap_effective, - new->cap_permitted); -+ #endif - } - break; - -@@ -1243,7 +1368,15 @@ static int cap_prctl_drop(unsigned long cap) - new = prepare_creds(); - if (!new) - return -ENOMEM; -+ #ifdef CONFIG_CREDP -+ { -+ kernel_cap_t tmp = new->cap_bset; -+ cap_lower(tmp, cap); -+ iee_set_cred_cap_bset(new, tmp); -+ } -+ #else - cap_lower(new->cap_bset, cap); -+ #endif - return commit_creds(new); - } - -@@ -1319,7 +1452,11 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, - new = prepare_creds(); - if (!new) - return -ENOMEM; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_securebits(new,arg2); -+ #else - new->securebits = arg2; -+ #endif - return commit_creds(new); - - case PR_GET_SECUREBITS: -@@ -1338,9 +1475,17 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, - if (!new) - return -ENOMEM; - if (arg2) -+ #ifdef CONFIG_CREDP -+ iee_set_cred_securebits(new,new->securebits | issecure_mask(SECURE_KEEP_CAPS)); -+ #else - new->securebits |= issecure_mask(SECURE_KEEP_CAPS); -+ #endif - else -+ #ifdef CONFIG_CREDP -+ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); -+ #else - new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); -+ #endif - return commit_creds(new); - - case PR_CAP_AMBIENT: -@@ -1351,7 +1496,15 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, - new = prepare_creds(); - if (!new) - return -ENOMEM; -+ #ifdef CONFIG_CREDP -+ do { -+ kernel_cap_t tmp_cap = new->cap_ambient; -+ tmp_cap.val = 0; -+ iee_set_cred_cap_ambient(new, tmp_cap); -+ } while (0); -+ #else - cap_clear(new->cap_ambient); -+ #endif - return commit_creds(new); - } - -@@ -1375,9 +1528,25 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, - if (!new) - return -ENOMEM; - if (arg2 == PR_CAP_AMBIENT_RAISE) -+ #ifdef CONFIG_CREDP -+ { -+ kernel_cap_t tmp = new->cap_ambient; -+ cap_raise(tmp, arg3); -+ iee_set_cred_cap_ambient(new, tmp); -+ } -+ #else - cap_raise(new->cap_ambient, arg3); -+ #endif - else -+ #ifdef CONFIG_CREDP -+ { -+ kernel_cap_t tmp = new->cap_ambient; -+ cap_lower(tmp, arg3); -+ iee_set_cred_cap_ambient(new, tmp); -+ } -+ #else - cap_lower(new->cap_ambient, arg3); -+ #endif - return commit_creds(new); - } - -diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c -index 720e5913832f..41ecdd75961a 100644 ---- a/security/integrity/evm/evm_crypto.c -+++ b/security/integrity/evm/evm_crypto.c -@@ -466,14 +466,26 @@ int evm_init_key(void) - if (IS_ERR(evm_key)) - return -ENOENT; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(evm_key)); -+ #else - down_read(&evm_key->sem); -+ #endif -+ #ifdef CONFIG_KEYP -+ ekp = ((union key_payload *)(evm_key->name_link.next))->data[0]; -+ #else - ekp = evm_key->payload.data[0]; -+ #endif - - rc = evm_set_key(ekp->decrypted_data, ekp->decrypted_datalen); - - /* burn the original key contents */ - memset(ekp->decrypted_data, 0, ekp->decrypted_datalen); -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(evm_key)); -+ #else - up_read(&evm_key->sem); -+ #endif - key_put(evm_key); - return rc; - } -diff --git a/security/keys/big_key.c b/security/keys/big_key.c -index c3367622c683..bc6a097f2f87 100644 ---- a/security/keys/big_key.c -+++ b/security/keys/big_key.c -@@ -164,7 +164,11 @@ void big_key_free_preparse(struct key_preparsed_payload *prep) - */ - void big_key_revoke(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ struct big_key_payload *payload = to_big_key_payload(*((union key_payload *)(key->name_link.next))); -+ #else - struct big_key_payload *payload = to_big_key_payload(key->payload); -+ #endif - - /* clear the quota */ - key_payload_reserve(key, 0); -@@ -177,7 +181,11 @@ void big_key_revoke(struct key *key) - */ - void big_key_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ struct big_key_payload *payload = to_big_key_payload(*((union key_payload *)(key->name_link.next))); -+ #else - struct big_key_payload *payload = to_big_key_payload(key->payload); -+ #endif - - if (payload->length > BIG_KEY_FILE_THRESHOLD) { - path_put(&payload->path); -@@ -210,7 +218,11 @@ int big_key_update(struct key *key, struct key_preparsed_payload *prep) - */ - void big_key_describe(const struct key *key, struct seq_file *m) - { -+ #ifdef CONFIG_KEYP -+ struct big_key_payload *payload = to_big_key_payload(*((union key_payload *)(key->name_link.next))); -+ #else - struct big_key_payload *payload = to_big_key_payload(key->payload); -+ #endif - - seq_puts(m, key->description); - -@@ -226,7 +238,11 @@ void big_key_describe(const struct key *key, struct seq_file *m) - */ - long big_key_read(const struct key *key, char *buffer, size_t buflen) - { -+ #ifdef CONFIG_KEYP -+ struct big_key_payload *payload = to_big_key_payload(*((union key_payload *)(key->name_link.next))); -+ #else - struct big_key_payload *payload = to_big_key_payload(key->payload); -+ #endif - size_t datalen = payload->length; - long ret; - -diff --git a/security/keys/dh.c b/security/keys/dh.c -index da64c358474b..f00a3e0c2c87 100644 ---- a/security/keys/dh.c -+++ b/security/keys/dh.c -@@ -32,7 +32,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) - - ret = -EOPNOTSUPP; - if (key->type == &key_type_user) { -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - status = key_validate(key); - if (status == 0) { - const struct user_key_payload *payload; -@@ -49,7 +53,11 @@ static ssize_t dh_data_from_key(key_serial_t keyid, const void **data) - ret = -ENOMEM; - } - } -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - } - - key_put(key); -diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c -index 1e313982af02..ef5e8f4f03f0 100644 ---- a/security/keys/encrypted-keys/encrypted.c -+++ b/security/keys/encrypted-keys/encrypted.c -@@ -314,11 +314,19 @@ static struct key *request_user_key(const char *master_desc, const u8 **master_k - if (IS_ERR(ukey)) - goto error; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(ukey)); -+ #else - down_read(&ukey->sem); -+ #endif - upayload = user_key_payload_locked(ukey); - if (!upayload) { - /* key was revoked before we acquired its semaphore */ -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(ukey)); -+ #else - up_read(&ukey->sem); -+ #endif - key_put(ukey); - ukey = ERR_PTR(-EKEYREVOKED); - goto error; -@@ -729,7 +737,11 @@ static int encrypted_key_decrypt(struct encrypted_key_payload *epayload, - if (ret < 0) - pr_err("encrypted_key: failed to decrypt key (%d)\n", ret); - out: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(mkey)); -+ #else - up_read(&mkey->sem); -+ #endif - key_put(mkey); - memzero_explicit(derived_key, sizeof(derived_key)); - return ret; -@@ -874,7 +886,11 @@ static void encrypted_rcu_free(struct rcu_head *rcu) - */ - static int encrypted_update(struct key *key, struct key_preparsed_payload *prep) - { -+ #ifdef CONFIG_KEYP -+ struct encrypted_key_payload *epayload = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - struct encrypted_key_payload *epayload = key->payload.data[0]; -+ #endif - struct encrypted_key_payload *new_epayload; - char *buf; - char *new_master_desc = NULL; -@@ -974,7 +990,11 @@ static long encrypted_read(const struct key *key, char *buffer, - goto out; - } - -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(mkey)); -+ #else - up_read(&mkey->sem); -+ #endif - key_put(mkey); - memzero_explicit(derived_key, sizeof(derived_key)); - -@@ -983,7 +1003,11 @@ static long encrypted_read(const struct key *key, char *buffer, - - return asciiblob_len; - out: -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(mkey)); -+ #else - up_read(&mkey->sem); -+ #endif - key_put(mkey); - memzero_explicit(derived_key, sizeof(derived_key)); - return ret; -@@ -994,7 +1018,11 @@ static long encrypted_read(const struct key *key, char *buffer, - */ - static void encrypted_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - kfree_sensitive(key->payload.data[0]); -+ #endif - } - - struct key_type key_type_encrypted = { -diff --git a/security/keys/encrypted-keys/masterkey_trusted.c b/security/keys/encrypted-keys/masterkey_trusted.c -index e6d22ce77e98..13803e0f1a8f 100644 ---- a/security/keys/encrypted-keys/masterkey_trusted.c -+++ b/security/keys/encrypted-keys/masterkey_trusted.c -@@ -34,8 +34,13 @@ struct key *request_trusted_key(const char *trusted_desc, - if (IS_ERR(tkey)) - goto error; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(tkey)); -+ tpayload = ((union key_payload *)(tkey->name_link.next))->data[0]; -+ #else - down_read(&tkey->sem); - tpayload = tkey->payload.data[0]; -+ #endif - *master_key = tpayload->key; - *master_keylen = tpayload->key_len; - error: -diff --git a/security/keys/gc.c b/security/keys/gc.c -index eaddaceda14e..4038e7b95395 100644 ---- a/security/keys/gc.c -+++ b/security/keys/gc.c -@@ -10,6 +10,11 @@ - #include - #include "internal.h" - -+#ifdef CONFIG_KEYP -+#include -+#include -+#endif ++ ENABLE_WP r12 + - /* - * Delay between key revocation/expiry in seconds - */ -@@ -135,18 +140,30 @@ void key_gc_keytype(struct key_type *ktype) - static noinline void key_gc_unused_keys(struct list_head *keys) - { - while (!list_empty(keys)) { -+ #ifdef CONFIG_KEYP -+ struct key *key = list_entry(keys->next, struct key_union, graveyard_link)->key; -+ #else - struct key *key = - list_entry(keys->next, struct key, graveyard_link); -+ #endif - short state = key->state; - -+ #ifdef CONFIG_KEYP -+ list_del(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link)); -+ #else - list_del(&key->graveyard_link); -+ #endif - - kdebug("- %u", key->serial); - key_check(key); - - #ifdef CONFIG_KEY_NOTIFICATIONS - remove_watch_list(key->watchers, key->serial); -+ #ifdef CONFIG_KEYP -+ iee_set_key_watchers(key, NULL); -+ #else - key->watchers = NULL; -+ #endif - #endif - - /* Throw away the key data if the key is instantiated */ -@@ -171,7 +188,15 @@ static noinline void key_gc_unused_keys(struct list_head *keys) - key_put_tag(key->domain_tag); - kfree(key->description); - -+ #ifdef CONFIG_KEYP -+ kmem_cache_free(key_union_jar,(struct key_union *)(key->graveyard_link.next)); -+ kmem_cache_free(key_struct_jar, (struct key_struct *)(key->name_link.prev)); -+ kmem_cache_free(key_payload_jar, (union key_payload *)(key->name_link.next)); -+ iee_memset(key, 0, sizeof(*key)); -+ barrier_data(key); -+ #else - memzero_explicit(key, sizeof(*key)); -+ #endif - kmem_cache_free(key_jar, key); - } - } -@@ -223,7 +248,11 @@ static void key_garbage_collector(struct work_struct *work) - - continue_scanning: - while (cursor) { -+ #ifdef CONFIG_KEYP -+ key = rb_entry(cursor, struct key_union, serial_node)->key; -+ #else - key = rb_entry(cursor, struct key, serial_node); -+ #endif - cursor = rb_next(cursor); - - if (refcount_read(&key->usage) == 0) -@@ -232,8 +261,13 @@ static void key_garbage_collector(struct work_struct *work) - if (unlikely(gc_state & KEY_GC_REAPING_DEAD_1)) { - if (key->type == key_gc_dead_keytype) { - gc_state |= KEY_GC_FOUND_DEAD_KEY; -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(key, KEY_FLAG_DEAD, SET_BIT_OP); -+ iee_set_key_perm(key, 0); -+ #else - set_bit(KEY_FLAG_DEAD, &key->flags); - key->perm = 0; -+ #endif - goto skip_dead_key; - } else if (key->type == &key_type_keyring && - key->restrict_link) { -@@ -339,10 +373,18 @@ static void key_garbage_collector(struct work_struct *work) - */ - found_unreferenced_key: - kdebug("unrefd key %d", key->serial); -+ #ifdef CONFIG_KEYP -+ rb_erase(&(((struct key_union *)(key->graveyard_link.next))->serial_node), &key_serial_tree); -+ #else - rb_erase(&key->serial_node, &key_serial_tree); -+ #endif - spin_unlock(&key_serial_lock); - -+ #ifdef CONFIG_KEYP -+ list_add_tail(&(((struct key_union *)(key->graveyard_link.next))->graveyard_link), &graveyard); -+ #else - list_add_tail(&key->graveyard_link, &graveyard); -+ #endif - gc_state |= KEY_GC_REAP_AGAIN; - goto maybe_resched; - -@@ -370,11 +412,21 @@ static void key_garbage_collector(struct work_struct *work) - destroy_dead_key: - spin_unlock(&key_serial_lock); - kdebug("destroy key %d", key->serial); -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ iee_set_key_type(key, &key_type_dead); -+ #else - down_write(&key->sem); - key->type = &key_type_dead; -+ #endif - if (key_gc_dead_keytype->destroy) - key_gc_dead_keytype->destroy(key); -+ #ifdef CONFIG_KEYP -+ iee_memset((key->name_link.next), KEY_DESTROY, sizeof(key->payload)); -+ up_write(&KEY_SEM(key)); -+ #else - memset(&key->payload, KEY_DESTROY, sizeof(key->payload)); - up_write(&key->sem); -+ #endif - goto maybe_resched; - } -diff --git a/security/keys/internal.h b/security/keys/internal.h -index ec2ec335b613..00d76f89179c 100644 ---- a/security/keys/internal.h -+++ b/security/keys/internal.h -@@ -83,12 +83,20 @@ extern unsigned key_quota_maxbytes; - - - extern struct kmem_cache *key_jar; -+#ifdef CONFIG_KEYP -+extern struct kmem_cache *key_union_jar; -+extern struct kmem_cache *key_struct_jar; -+extern struct kmem_cache *key_payload_jar; -+#endif - extern struct rb_root key_serial_tree; - extern spinlock_t key_serial_lock; - extern struct mutex key_construction_mutex; - extern wait_queue_head_t request_key_conswq; - - extern void key_set_index_key(struct keyring_index_key *index_key); -+#ifdef CONFIG_KEYP -+extern void iee_key_set_index_key(struct keyring_index_key *index_key); -+#endif - extern struct key_type *key_type_lookup(const char *type); - extern void key_type_put(struct key_type *ktype); - -diff --git a/security/keys/key.c b/security/keys/key.c -index 35db23d05302..92b8840bed69 100644 ---- a/security/keys/key.c -+++ b/security/keys/key.c -@@ -17,7 +17,18 @@ - #include - #include "internal.h" - -+#ifdef CONFIG_KEYP -+#include -+#include -+#include -+#endif ++ popq %r12 ++ ++ /* restore RFLAGS*/ ++ popfq ++ jmp __x86_return_thunk /* ret */ ++SYM_FUNC_END(ptp_xchg) ++ ++SYM_FUNC_START(ptp_try_cmpxchg) ++ /* save RFLAGS, close irq */ ++ pushfq ++ cli ++ ++ pushq %r12 ++ ++#ifdef CONFIG_PTP ++ movq PER_CPU_VAR(iee_disables) + IEE_DISABLE, %r12 ++ testq %r12, %r12 ++ jz cmpxchg_wp_enabled ++ ++ movq %rsi, %rax ++ lock cmpxchgq %rdx, (%rdi) ++ ++ popq %r12 ++ popfq ++ jmp __x86_return_thunk + - struct kmem_cache *key_jar; -+#ifdef CONFIG_KEYP -+struct kmem_cache *key_union_jar; -+struct kmem_cache *key_struct_jar; -+struct kmem_cache *key_payload_jar; ++cmpxchg_wp_enabled: +#endif - struct rb_root key_serial_tree; /* tree of keys indexed by serial */ - DEFINE_SPINLOCK(key_serial_lock); - -@@ -139,9 +150,16 @@ static inline void key_alloc_serial(struct key *key) - /* propose a random serial number and look for a hole for it in the - * serial number tree */ - do { -+ #ifdef CONFIG_KEYP -+ key_serial_t tmp; -+ get_random_bytes(&tmp, sizeof(key->serial)); ++ DISABLE_WP r12 + -+ iee_set_key_serial(key, tmp >> 1); -+ #else - get_random_bytes(&key->serial, sizeof(key->serial)); - - key->serial >>= 1; /* negative numbers are not permitted */ -+ #endif - } while (key->serial < 3); - - spin_lock(&key_serial_lock); -@@ -152,7 +170,11 @@ static inline void key_alloc_serial(struct key *key) - - while (*p) { - parent = *p; -+ #ifdef CONFIG_KEYP -+ xkey = rb_entry(parent, struct key_union, serial_node)->key; -+ #else - xkey = rb_entry(parent, struct key, serial_node); -+ #endif - - if (key->serial < xkey->serial) - p = &(*p)->rb_left; -@@ -163,8 +185,13 @@ static inline void key_alloc_serial(struct key *key) - } - - /* we've found a suitable hole - arrange for this key to occupy it */ -+ #ifdef CONFIG_KEYP -+ rb_link_node(&(((struct key_union *)(key->graveyard_link.next))->serial_node), parent, p); -+ rb_insert_color(&(((struct key_union *)(key->graveyard_link.next))->serial_node), &key_serial_tree); -+ #else - rb_link_node(&key->serial_node, parent, p); - rb_insert_color(&key->serial_node, &key_serial_tree); -+ #endif - - spin_unlock(&key_serial_lock); - return; -@@ -173,9 +200,18 @@ static inline void key_alloc_serial(struct key *key) - * that point looking for the next unused serial number */ - serial_exists: - for (;;) { -+ #ifdef CONFIG_KEYP -+ key_serial_t tmp = key->serial + 1; -+ iee_set_key_serial(key, tmp); -+ #else - key->serial++; -+ #endif - if (key->serial < 3) { -+ #ifdef CONFIG_KEYP -+ iee_set_key_serial(key, 3); -+ #else - key->serial = 3; -+ #endif - goto attempt_insertion; - } - -@@ -183,7 +219,11 @@ static inline void key_alloc_serial(struct key *key) - if (!parent) - goto attempt_insertion; - -+ #ifdef CONFIG_KEYP -+ xkey = rb_entry(parent, struct key_union, serial_node)->key; -+ #else - xkey = rb_entry(parent, struct key, serial_node); -+ #endif - if (key->serial < xkey->serial) - goto attempt_insertion; - } -@@ -231,6 +271,9 @@ struct key *key_alloc(struct key_type *type, const char *desc, - struct key *key; - size_t desclen, quotalen; - int ret; -+ #ifdef CONFIG_KEYP -+ unsigned long kflags; -+ #endif - - key = ERR_PTR(-EINVAL); - if (!desc || !*desc) -@@ -274,17 +317,72 @@ struct key *key_alloc(struct key_type *type, const char *desc, - } - - /* allocate and initialise the key and its description */ -+ #ifdef CONFIG_KEYP -+ key = kmem_cache_alloc(key_jar, GFP_KERNEL); -+ #else - key = kmem_cache_zalloc(key_jar, GFP_KERNEL); -+ #endif - if (!key) - goto no_memory_2; -- -+ #ifdef CONFIG_KEYP -+ struct key_union *key_union = kmem_cache_zalloc(key_union_jar, GFP_KERNEL); -+ key_union->key = key; -+ struct key_struct *key_struct = kmem_cache_zalloc(key_struct_jar, GFP_KERNEL); -+ key_struct->key = key; -+ iee_set_key_union(key, key_union); -+ iee_set_key_struct(key, key_struct); -+ iee_set_key_payload(key, kmem_cache_alloc(key_payload_jar, GFP_KERNEL)); -+ #endif ++ movq %rsi, %rax ++ lock cmpxchgq %rdx, (%rdi) + -+ #ifdef CONFIG_KEYP -+ struct keyring_index_key tmp = key->index_key; -+ tmp.desc_len = desclen; -+ tmp.description = kmemdup(desc, desclen + 1, GFP_KERNEL); -+ iee_set_key_index_key(key, &tmp); -+ #else - key->index_key.desc_len = desclen; - key->index_key.description = kmemdup(desc, desclen + 1, GFP_KERNEL); -+ #endif - if (!key->index_key.description) - goto no_memory_3; -+ #ifdef CONFIG_KEYP -+ tmp = key->index_key; -+ tmp.type = type; -+ iee_set_key_index_key(key, &tmp); -+ iee_key_set_index_key(&key->index_key); -+ #else - key->index_key.type = type; - key_set_index_key(&key->index_key); -+ #endif ++ ENABLE_WP r12 + -+ #ifdef CONFIG_KEYP -+ iee_set_key_usage(key, 1, REFCOUNT_SET); -+ init_rwsem(&KEY_SEM(key)); -+ lockdep_set_class(&KEY_SEM(key), &type->lock_class); -+ iee_set_key_user(key, user); -+ iee_set_key_quotalen(key, quotalen); -+ iee_set_key_datalen(key, type->def_datalen); -+ iee_set_key_uid(key, uid); -+ iee_set_key_gid(key, gid); -+ iee_set_key_perm(key, perm); -+ iee_set_key_restrict_link(key, restrict_link); -+ iee_set_key_last_used_at(key, ktime_get_real_seconds()); -+ -+ kflags = key->flags; -+ if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) -+ kflags |= 1 << KEY_FLAG_IN_QUOTA; -+ if (flags & KEY_ALLOC_BUILT_IN) -+ kflags |= 1 << KEY_FLAG_BUILTIN; -+ if (flags & KEY_ALLOC_UID_KEYRING) -+ kflags |= 1 << KEY_FLAG_UID_KEYRING; -+ if (flags & KEY_ALLOC_SET_KEEP) -+ kflags |= 1 << KEY_FLAG_KEEP; -+ iee_set_key_flags(key, kflags); - -+#ifdef KEY_DEBUGGING -+ iee_set_key_magic(key, KEY_DEBUG_MAGIC); ++ popq %r12 ++ /* restore RFLAGS*/ ++ popfq ++ jmp __x86_return_thunk /* ret */ ++SYM_FUNC_END(ptp_try_cmpxchg) ++ ++SYM_FUNC_START(ptp_rw_gate) ++ /* save Interrupt flag */ ++ pushfq ++ /* close irq*/ ++ cli ++ ++ pushq %r12 ++ ++#ifdef CONFIG_PTP ++ movq PER_CPU_VAR(iee_disables) + IEE_DISABLE, %r12 ++ testq %r12, %r12 ++ jz ptprw_wp_enabled ++ ++ leaq iee_funcs(%rip), %rax ++ call *(%rax, %rdi, 8) ++ ++ popq %r12 ++ popfq ++ jmp __x86_return_thunk ++ ++ptprw_wp_enabled: +#endif -+ #else - refcount_set(&key->usage, 1); - init_rwsem(&key->sem); - lockdep_set_class(&key->sem, &type->lock_class); -@@ -310,6 +408,7 @@ struct key *key_alloc(struct key_type *type, const char *desc, - #ifdef KEY_DEBUGGING - key->magic = KEY_DEBUG_MAGIC; - #endif -+ #endif - - /* let the security module know about the key */ - ret = security_key_alloc(key, cred, flags); -@@ -326,6 +425,12 @@ struct key *key_alloc(struct key_type *type, const char *desc, - - security_error: - kfree(key->description); -+ #ifdef CONFIG_KEYP -+ kmem_cache_free(key_union_jar,(struct key_union *)(key->graveyard_link.next)); -+ kmem_cache_free(key_struct_jar, (struct key_struct *)(key->name_link.prev)); -+ kmem_cache_free(key_payload_jar, (union key_payload *)(key->name_link.next)); -+ iee_memset(key, 0, sizeof(struct key)); -+ #endif - kmem_cache_free(key_jar, key); - if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { - spin_lock(&user->lock); -@@ -338,6 +443,12 @@ struct key *key_alloc(struct key_type *type, const char *desc, - goto error; - - no_memory_3: -+ #ifdef CONFIG_KEYP -+ kmem_cache_free(key_union_jar,(struct key_union *)(key->graveyard_link.next)); -+ kmem_cache_free(key_struct_jar, (struct key_struct *)(key->name_link.prev)); -+ kmem_cache_free(key_payload_jar, (union key_payload *)(key->name_link.next)); -+ iee_memset(key, 0, sizeof(struct key)); ++ DISABLE_WP r12 ++ ++ /* call iee func */ ++ leaq iee_funcs(%rip), %rax ++ call *(%rax, %rdi, 8) ++ ++ ENABLE_WP r12 ++ ++ popq %r12 ++ ++ /* restore irq*/ ++ popfq ++ ++ jmp __x86_return_thunk /* ret */ ++SYM_FUNC_END(ptp_rw_gate) ++EXPORT_SYMBOL(ptp_rw_gate) +diff --git a/arch/x86/kernel/haoc/ptp/ptp.c b/arch/x86/kernel/haoc/ptp/ptp.c +new file mode 100644 +index 000000000000..673dfcec2d54 +--- /dev/null ++++ b/arch/x86/kernel/haoc/ptp/ptp.c +@@ -0,0 +1,358 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define __pte_to_phys(pte) (pte_pfn(pte) << PAGE_SHIFT) ++#define __pmd_to_phys(pmd) (__pte_to_phys(__pte(pmd_val(pmd)))) ++#define __pud_to_phys(pud) (__pte_to_phys(__pte(pud_val(pud)))) ++#define __p4d_to_phys(p4d) (__pte_to_phys(__pte(p4d_val(p4d)))) ++#define __pgd_to_phys(pgd) (__pte_to_phys(__pte(pgd_val(pgd)))) ++ ++static void __init _ptp_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) ++{ ++ pmd_t pmd = READ_ONCE(*pmdp); ++ unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); ++ ++ set_iee_page(logical_addr, 0); ++} ++ ++static void __init _ptp_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); ++ pmd_t *pmdp; ++ pmd_t pmd; ++ unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); ++ ++ set_iee_page(logical_addr, 0); ++ pmdp = pmd_offset(pudp, addr); ++ do { ++ next = pmd_addr_end(addr, end); ++ pmd = READ_ONCE(*pmdp); ++ if (pmd_val(pmd) & _PSE) ++ continue; ++ else ++ _ptp_set_pte_table_ro(pmdp, addr, next); ++ } while (pmdp++, addr = next, addr != end); ++} ++ ++static void __init _ptp_set_pud_table_ro(p4d_t *p4dp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ p4d_t p4d = READ_ONCE(*p4dp); ++ pud_t *pudp; ++ pud_t pud; ++ unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); ++ ++ set_iee_page(logical_addr, 0); ++ pudp = pud_offset(p4dp, addr); ++ do { ++ next = pud_addr_end(addr, end); ++ pud = READ_ONCE(*pudp); ++ if (pud_val(pud) & _PSE) { ++ // _PSE = 1 means a page, not a table ++ continue; ++ } else { ++ _ptp_set_pmd_table_ro(pudp, addr, next); ++ } ++ } while (pudp++, addr = next, addr != end); ++} ++ ++static void __init _ptp_set_p4d_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ pgd_t pgd = READ_ONCE(*pgdp); ++ p4d_t *p4dp; ++ p4d_t p4d; ++ unsigned long logical_addr = (unsigned long)__va(__pgd_to_phys(pgd)); ++ ++ set_iee_page(logical_addr, 0); ++ p4dp = p4d_offset(pgdp, addr); ++ do { ++ next = p4d_addr_end(addr, end); ++ p4d = READ_ONCE(*p4dp); ++ /* No 512 GiB huge pages yet */ ++ _ptp_set_pud_table_ro(p4dp, addr, next); ++ } while (p4dp++, addr = next, addr != end); ++} ++ ++static void __init _ptp_mark_iee_pgtable_for_one_region_ro(pgd_t *pgdir, ++ unsigned long va_start, unsigned long va_end) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); ++ ++ addr = va_start & PAGE_MASK; ++ end = PAGE_ALIGN(va_end); ++ ++ do { ++ next = pgd_addr_end(addr, end); ++ _ptp_set_p4d_table_ro(pgdp, addr, next); ++ } while (pgdp++, addr = next, addr != end); ++} ++ ++void __init ptp_mark_all_pgtable_ro(void) ++{ ++ unsigned long logical_addr; ++ phys_addr_t start, end; ++ u64 i; ++ pgd_t *pgdp; ++ ++ // handing 1-level page table swapper_pg_dir ++ pgdp = swapper_pg_dir; ++ set_iee_page((unsigned long)swapper_pg_dir, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); ++ set_iee_page(logical_addr, 0); ++ ++ // handling 2/3/4/5-level page table for kernel ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)_text, (unsigned long)_etext); ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)__start_rodata, (unsigned long)__end_rodata); ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)_sdata, (unsigned long)_edata); ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)__bss_start, (unsigned long)__bss_stop); ++ ++ // handling 2/3/4/5-level statically allocated page table ++ #ifdef CONFIG_X86_5LEVEL ++ set_iee_page((unsigned long)level4_kernel_pgt, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(level4_kernel_pgt)); ++ set_iee_page(logical_addr, 0); + #endif - kmem_cache_free(key_jar, key); - no_memory_2: - if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) { -@@ -391,14 +502,22 @@ int key_payload_reserve(struct key *key, size_t datalen) - } - else { - key->user->qnbytes += delta; -+ #ifdef CONFIG_KEYP -+ iee_set_key_quotalen(key, key->quotalen + delta); -+ #else - key->quotalen += delta; -+ #endif - } - spin_unlock(&key->user->lock); ++ ++ set_iee_page((unsigned long)level3_kernel_pgt, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(level3_kernel_pgt)); ++ set_iee_page(logical_addr, 0); ++ ++ set_iee_page((unsigned long)level2_kernel_pgt, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(level2_kernel_pgt)); ++ set_iee_page(logical_addr, 0); ++ ++ for (int i = 0; i < FIXMAP_PMD_NUM; i++) { ++ set_iee_page((unsigned long)level2_fixmap_pgt, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(level2_fixmap_pgt)); ++ set_iee_page(logical_addr, 0); ++ } ++ ++ for (int i = 0; i < FIXMAP_PMD_NUM; i++) { ++ set_iee_page((unsigned long)level1_fixmap_pgt, 0); ++ logical_addr = (unsigned long)__va(__pa_symbol(level2_fixmap_pgt)); ++ set_iee_page(logical_addr, 0); ++ } ++ ++ // handling 2/3/4-level page table for logical mem and iee ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; ++ /* ++ * The linear map must allow allocation tags reading/writing ++ * if MTE is present. Otherwise, it has the same attributes as ++ * PAGE_KERNEL. ++ */ ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)__va(start), (unsigned long)__va(end)); ++ _ptp_mark_iee_pgtable_for_one_region_ro(pgdp, ++ (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); ++ } ++ pr_info("PTP: mark all kernel page tables ro done"); ++} ++ ++static inline void check_addr_range_ro(unsigned long addr_start, unsigned long addr_end, ++ unsigned long start, unsigned long end, pte_t pte, const char *msg) ++{ ++ if (unlikely((addr_start >= start && addr_start < end) ++ || (addr_end > start && addr_end < end))) ++ if (unlikely(pte_write(pte))) ++ panic("IEE Error: Are you trying to write %s: [0x%lx, 0x%lx]?", ++ msg, addr_start, addr_end); ++} ++ ++static inline void check_dep_and_ro(unsigned long image_addr, unsigned long size, pte_t pte) ++{ ++ if (unlikely(!kernel_set_to_readonly)) ++ return; ++ ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)_stext, (unsigned long)_etext, pte, "text"); ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)__iee_si_text_start, (unsigned long)__iee_si_text_end, pte, "iee_si_text"); ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); ++} ++ ++void _iee_set_pte(unsigned long __unused, pte_t *ptep, pte_t pte) ++{ ++ if (!(pte_val(pte) & _PAGE_PRESENT)) { ++ WRITE_ONCE(*ptep, pte); ++ return; ++ } ++ ++ phys_addr_t phys_addr = __pte_to_phys(pte); ++ unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; ++ ++ check_dep_and_ro(image_addr, PAGE_SIZE, pte); ++ WRITE_ONCE(*ptep, pte); ++} ++ ++void _iee_set_pmd(unsigned long __unused, pmd_t *pmdp, pmd_t pmd) ++{ ++ phys_addr_t phys_addr; ++ unsigned long image_addr; ++ ++ if (!(pmd_val(pmd) & _PAGE_PRESENT)) { ++ WRITE_ONCE(*pmdp, pmd); ++ return; ++ } ++ if (pmd_leaf(pmd)) { ++ phys_addr = __pmd_to_phys(pmd); ++ image_addr = phys_addr + __START_KERNEL_map - phys_base; ++ check_dep_and_ro(image_addr, PMD_SIZE, __pte(pmd_val(pmd))); ++ } ++ ++ WRITE_ONCE(*pmdp, pmd); ++} ++ ++void _iee_set_pud(unsigned long __unused, pud_t *pudp, pud_t pud) ++{ ++ phys_addr_t phys_addr; ++ unsigned long image_addr; ++ ++ if (!(pud_val(pud) & _PAGE_PRESENT)) { ++ WRITE_ONCE(*pudp, pud); ++ return; ++ } ++ if (pud_leaf(pud)) { ++ phys_addr = __pud_to_phys(pud); ++ image_addr = phys_addr + __START_KERNEL_map - phys_base; ++ check_dep_and_ro(image_addr, PUD_SIZE, __pte(pud_val(pud))); ++ } ++ ++ WRITE_ONCE(*pudp, pud); ++} ++ ++void _iee_set_p4d(unsigned long __unused, p4d_t *p4dp, p4d_t p4d) ++{ ++ WRITE_ONCE(*p4dp, p4d); ++} ++ ++void _iee_set_pgd(unsigned long __unused, pgd_t *pgdp, pgd_t pgd) ++{ ++ WRITE_ONCE(*pgdp, pgd); ++} ++ ++static inline void check_text_poke_ro(unsigned long image_addr, unsigned long size, pte_t pte) ++{ ++ if (unlikely(!kernel_set_to_readonly)) ++ return; ++ ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)__start_rodata, (unsigned long)__end_rodata, pte, "rodata"); ++ check_addr_range_ro(image_addr, image_addr + size, ++(unsigned long)__iee_si_data_start, (unsigned long)__iee_si_data_end, pte, "iee_si_data"); ++} ++ ++void _iee_set_pte_text_poke(unsigned long __unused, pte_t *ptep, pte_t pte) ++{ ++ if (!(pte_val(pte) & _PAGE_PRESENT)) { ++ WRITE_ONCE(*ptep, pte); ++ return; ++ } ++ ++ phys_addr_t phys_addr = __pte_to_phys(pte); ++ unsigned long image_addr = phys_addr + __START_KERNEL_map - phys_base; ++ ++ check_text_poke_ro(image_addr, PAGE_SIZE, pte); ++ WRITE_ONCE(*ptep, pte); ++} ++ ++DEFINE_PER_CPU(struct iee_disable_t, iee_disables); ++ ++void __init ptp_iee_disable_init(void) ++{ ++ int cpu; ++ struct iee_disable_t *iee_disable; ++ ++ for_each_possible_cpu(cpu) { ++ iee_disable = per_cpu_ptr(&iee_disables, cpu); ++ iee_disable->disabled_cnt = 0; ++ } ++} ++ ++void ptp_disable_iee(unsigned long *reg) ++{ ++ unsigned long irq_flags; ++ int cpu; ++ struct iee_disable_t *iee_disable; ++ ++ local_irq_save(irq_flags); ++ cpu = get_cpu(); ++ iee_disable = per_cpu_ptr(&iee_disables, cpu); ++ if (iee_disable->disabled_cnt == 0) { ++ *reg = read_cr0(); ++ asm volatile("mov %0, %%cr0" : : "r"(*reg & ~X86_CR0_WP)); ++ } ++ iee_disable->disabled_cnt++; ++ put_cpu(); ++ local_irq_restore(irq_flags); ++} ++ ++void ptp_enable_iee(unsigned long reg) ++{ ++ unsigned long irq_flags; ++ int cpu; ++ struct iee_disable_t *iee_disable; ++ ++ local_irq_save(irq_flags); ++ cpu = get_cpu(); ++ iee_disable = per_cpu_ptr(&iee_disables, cpu); ++ iee_disable->disabled_cnt--; ++ if (iee_disable->disabled_cnt == 0) ++ asm volatile("mov %0, %%cr0" : : "r"(reg | X86_CR0_WP)); ++ put_cpu(); ++ local_irq_restore(irq_flags); ++} ++ ++void ptp_context_enable_iee(int *disabled_cnt, unsigned long *reg) ++{ ++ unsigned long irq_flags; ++ int cpu; ++ struct iee_disable_t *iee_disable; ++ ++ local_irq_save(irq_flags); ++ cpu = get_cpu(); ++ iee_disable = per_cpu_ptr(&iee_disables, cpu); ++ *disabled_cnt = iee_disable->disabled_cnt; ++ if (*disabled_cnt > 0) { ++ *reg = read_cr0(); ++ iee_disable->disabled_cnt = 0; ++ asm volatile("mov %0, %%cr0" : : "r"(*reg | X86_CR0_WP)); ++ } ++ put_cpu(); ++ local_irq_restore(irq_flags); ++} ++ ++void ptp_context_restore_iee(int disabled_cnt, unsigned long reg) ++{ ++ unsigned long irq_flags; ++ int cpu; ++ struct iee_disable_t *iee_disable; ++ ++ local_irq_save(irq_flags); ++ if (disabled_cnt > 0) { ++ cpu = get_cpu(); ++ iee_disable = per_cpu_ptr(&iee_disables, cpu); ++ asm volatile("mov %0, %%cr0" : : "r"(reg & ~X86_CR0_WP)); ++ iee_disable->disabled_cnt = disabled_cnt; ++ put_cpu(); ++ } ++ local_irq_restore(irq_flags); ++} +diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c +index 1defe865de67..86c167d5a093 100644 +--- a/arch/x86/kernel/head64.c ++++ b/arch/x86/kernel/head64.c +@@ -720,7 +720,11 @@ static void startup_64_load_idt(unsigned long physbase) } - /* change the recorded data length if that didn't generate an error */ - if (ret == 0) -+ #ifdef CONFIG_KEYP -+ iee_set_key_datalen(key, datalen); -+ #else - key->datalen = datalen; -+ #endif - - return ret; - } -@@ -412,8 +531,14 @@ static void mark_key_instantiated(struct key *key, int reject_error) - /* Commit the payload before setting the state; barrier versus - * key_read_state(). - */ -+ #ifdef CONFIG_KEYP -+ compiletime_assert_atomic_type(key->state); -+ barrier(); -+ iee_set_key_state(key, (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); + desc->address = (unsigned long)idt; ++ #ifdef CONFIG_IEE_SIP ++ iee_load_idt_early(desc); + #else - smp_store_release(&key->state, - (reject_error < 0) ? reject_error : KEY_IS_POSITIVE); + native_load_idt(desc); + #endif } - /* -@@ -449,13 +574,22 @@ static int __key_instantiate_and_link(struct key *key, - mark_key_instantiated(key, 0); - notify_key(key, NOTIFY_KEY_INSTANTIATED, 0); - -+ #ifdef CONFIG_KEYP -+ if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) -+ awaken = 1; -+ #else - if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) - awaken = 1; -+ #endif + /* This is used when running on kernel addresses */ +diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c +index 234851fe0ef8..51aab86e48b9 100644 +--- a/arch/x86/kernel/paravirt.c ++++ b/arch/x86/kernel/paravirt.c +@@ -81,7 +81,11 @@ void __init native_pv_lock_init(void) - /* and link it into the destination keyring */ - if (keyring) { - if (test_bit(KEY_FLAG_KEEP, &keyring->flags)) -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(key, KEY_FLAG_KEEP, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_KEEP, &key->flags); -+ #endif - - __key_link(keyring, key, _edit); - } -@@ -464,8 +598,14 @@ static int __key_instantiate_and_link(struct key *key, - if (authkey) - key_invalidate(authkey); - -- if (prep->expiry != TIME64_MAX) -+ if (prep->expiry != TIME64_MAX) { -+ #ifdef CONFIG_KEYP -+ iee_set_key_expiry(key, prep->expiry); -+ #else - key_set_expiry(key, prep->expiry); -+ #endif -+ key_schedule_gc(prep->expiry + key_gc_delay); -+ } - } - } + static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) + { ++#ifdef CONFIG_PTP ++ ptp_tlb_remove_page(tlb, table); ++#else + tlb_remove_page(tlb, table); ++#endif + } -@@ -605,10 +745,20 @@ int key_reject_and_link(struct key *key, - atomic_inc(&key->user->nikeys); - mark_key_instantiated(key, -error); - notify_key(key, NOTIFY_KEY_INSTANTIATED, -error); -+ #ifdef CONFIG_KEYP -+ iee_set_key_expiry(key, ktime_get_real_seconds() + timeout); -+ #else - key_set_expiry(key, ktime_get_real_seconds() + timeout); -+ #endif -+ key_schedule_gc(key->expiry + key_gc_delay); + unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, +diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S +index 0b98dcc93d90..d444b8f136f3 100644 +--- a/arch/x86/kernel/vmlinux.lds.S ++++ b/arch/x86/kernel/vmlinux.lds.S +@@ -99,6 +99,22 @@ jiffies = jiffies_64; -+ #ifdef CONFIG_KEYP -+ if (iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, TEST_AND_CLEAR_BIT)) -+ awaken = 1; -+ #else - if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) - awaken = 1; -+ #endif + #endif - ret = 0; ++#ifdef CONFIG_IEE_SIP ++#define IEE_SI_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_text_start = .; \ ++ *(.iee.si_text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_text_end = .; ++ ++#define IEE_SI_DATA \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_data_start = .; \ ++ *(.iee.si_data) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_data_end = .; ++#endif ++ + PHDRS { + text PT_LOAD FLAGS(5); /* R_E */ + data PT_LOAD FLAGS(6); /* RW_ */ +@@ -131,6 +147,9 @@ SECTIONS + SCHED_TEXT + LOCK_TEXT + KPROBES_TEXT ++#ifdef CONFIG_IEE_SIP ++ IEE_SI_TEXT ++#endif + SOFTIRQENTRY_TEXT + #ifdef CONFIG_RETPOLINE + *(.text..__x86.indirect_thunk) +@@ -181,6 +200,9 @@ SECTIONS + CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES) -@@ -647,8 +797,13 @@ void key_put(struct key *key) - if (key) { - key_check(key); + DATA_DATA ++#ifdef CONFIG_IEE_SIP ++ IEE_SI_DATA ++#endif + CONSTRUCTORS -+ #ifdef CONFIG_KEYP -+ if (iee_set_key_usage(key, 0, REFCOUNT_DEC_AND_TEST)) -+ schedule_work(&key_gc_work); -+ #else - if (refcount_dec_and_test(&key->usage)) - schedule_work(&key_gc_work); -+ #endif - } - } - EXPORT_SYMBOL(key_put); -@@ -666,7 +821,11 @@ struct key *key_lookup(key_serial_t id) - /* search the tree for the specified key */ - n = key_serial_tree.rb_node; - while (n) { -+ #ifdef CONFIG_KEYP -+ key = rb_entry(n, struct key_union, serial_node)->key; -+ #else - key = rb_entry(n, struct key, serial_node); -+ #endif + /* rarely changed data like cpu maps */ +diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c +index 8c421b8be133..d16dbe1552ff 100644 +--- a/arch/x86/mm/init_64.c ++++ b/arch/x86/mm/init_64.c +@@ -56,6 +56,9 @@ + #include + #include - if (id < key->serial) - n = n->rb_left; -@@ -684,8 +843,13 @@ struct key *key_lookup(key_serial_t id) - /* A key is allowed to be looked up only if someone still owns a - * reference to it - otherwise it's awaiting the gc. - */ -+ #ifdef CONFIG_KEYP -+ if (!iee_set_key_usage(key, 0, REFCOUNT_INC_NOT_ZERO)) -+ goto not_found; -+ #else - if (!refcount_inc_not_zero(&key->usage)) - goto not_found; -+ #endif ++#ifdef CONFIG_IEE ++#include ++#endif + #include "mm_internal.h" - error: - spin_unlock(&key_serial_lock); -@@ -723,13 +887,27 @@ void key_set_timeout(struct key *key, unsigned timeout) - time64_t expiry = TIME64_MAX; + #include "ident_map.c" +@@ -1352,6 +1355,14 @@ static void __init preallocate_vmalloc_pages(void) - /* make the changes with the locks held to prevent races */ -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif + void __init mem_init(void) + { ++#ifdef CONFIG_PTP ++ ptp_pg_cache_init(&pgd_cache, PGD_ALLOCATION_ORDER, 1, "pgd_cache"); ++#ifdef CONFIG_X86_5LEVEL ++ ptp_pg_cache_init(&pg_cache, 0, 4, "pg_cache"); ++#else ++ ptp_pg_cache_init(&pg_cache, 0, 3, "pg_cache"); ++#endif ++#endif + pci_iommu_alloc(); - if (timeout > 0) - expiry = ktime_get_real_seconds() + timeout; -+ -+ #ifdef CONFIG_KEYP -+ iee_set_key_expiry(key, expiry); -+ #else - key_set_expiry(key, expiry); -+ #endif -+ key_schedule_gc(key->expiry + key_gc_delay); + /* clear_bss() already clear the empty_zero_page */ +@@ -1373,6 +1384,15 @@ void __init mem_init(void) + if (get_gate_vma(&init_mm)) + kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif ++#ifdef CONFIG_IEE ++ iee_init(); ++#endif ++#ifdef CONFIG_PTP ++ ptp_set_iee_reserved(&pg_cache); ++ ptp_set_iee_reserved(&pgd_cache); ++ ptp_mark_all_pgtable_ro(); ++ ptp_iee_disable_init(); ++ #endif + preallocate_vmalloc_pages(); } - EXPORT_SYMBOL_GPL(key_set_timeout); - -@@ -762,7 +940,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, - if (!key->type->update) - goto error; -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif +diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c +index 8e1ef5345b7a..2aaa93feed6e 100644 +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -6,6 +6,10 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#include ++#endif - ret = key->type->update(key, prep); - if (ret == 0) { -@@ -771,7 +953,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, - notify_key(key, NOTIFY_KEY_UPDATED, 0); - } + #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK + phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; +@@ -415,15 +419,27 @@ static inline void _pgd_free(pgd_t *pgd) + } + #else -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif ++#ifdef CONFIG_PTP ++struct pg_cache pgd_cache; ++#endif ++ + static inline pgd_t *_pgd_alloc(void) + { ++#ifdef CONFIG_PTP ++ return (pgd_t *)ptp_pg_alloc(&pgd_cache, GFP_PGTABLE_USER); ++#else + return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, + PGD_ALLOCATION_ORDER); ++#endif + } - if (ret < 0) - goto error; -@@ -1087,7 +1273,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) - goto error; - } + static inline void _pgd_free(pgd_t *pgd) + { ++#ifdef CONFIG_PTP ++ ptp_pg_free(&pgd_cache, pgd); ++#else + free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); ++#endif + } + #endif /* CONFIG_X86_PAE */ -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif +@@ -557,8 +573,13 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, + int ret = 0; - ret = key->type->update(key, &prep); - if (ret == 0) { -@@ -1096,7 +1286,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) - notify_key(key, NOTIFY_KEY_UPDATED, 0); - } + if (pte_young(*ptep)) ++#ifdef CONFIG_PTP ++ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, ++ (unsigned long *) &ptep->pte); ++#else + ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *) &ptep->pte); ++#endif -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif + return ret; + } +@@ -570,8 +591,13 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, + int ret = 0; - error: - if (key->type->preparse) -@@ -1125,6 +1319,23 @@ void key_revoke(struct key *key) - * authorisation key whilst holding the sem on a key we've just - * instantiated - */ -+ #ifdef CONFIG_KEYP -+ down_write_nested(&KEY_SEM(key), 1); -+ if (!iee_set_key_flag_bit(key, KEY_FLAG_REVOKED, TEST_AND_SET_BIT)) { -+ notify_key(key, NOTIFY_KEY_REVOKED, 0); -+ if (key->type->revoke) -+ key->type->revoke(key); -+ -+ /* set the death time to no more than the expiry time */ -+ time = ktime_get_real_seconds(); -+ if (key->revoked_at == 0 || key->revoked_at > time) { -+ iee_set_key_revoked_at(key, time); -+ key_schedule_gc(key->revoked_at + key_gc_delay); -+ } -+ } -+ -+ up_write(&KEY_SEM(key)); -+ #else - down_write_nested(&key->sem, 1); - if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags)) { - notify_key(key, NOTIFY_KEY_REVOKED, 0); -@@ -1140,6 +1351,7 @@ void key_revoke(struct key *key) - } + if (pmd_young(*pmdp)) ++#ifdef CONFIG_PTP ++ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, ++ (unsigned long *)pmdp); ++#else + ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pmdp); ++#endif - up_write(&key->sem); -+ #endif + return ret; } - EXPORT_SYMBOL(key_revoke); +@@ -584,8 +610,13 @@ int pudp_test_and_clear_young(struct vm_area_struct *vma, + int ret = 0; -@@ -1157,12 +1369,21 @@ void key_invalidate(struct key *key) - key_check(key); + if (pud_young(*pudp)) ++#ifdef CONFIG_PTP ++ ret = iee_test_and_clear_bit(_PAGE_BIT_ACCESSED, ++ (unsigned long *)pudp); ++#else + ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pudp); ++#endif - if (!test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { -+ #ifdef CONFIG_KEYP -+ down_write_nested(&KEY_SEM(key), 1); -+ if (!iee_set_key_flag_bit(key, KEY_FLAG_INVALIDATED, TEST_AND_SET_BIT)) { -+ notify_key(key, NOTIFY_KEY_INVALIDATED, 0); -+ key_schedule_gc_links(); -+ } -+ up_write(&KEY_SEM(key)); -+ #else - down_write_nested(&key->sem, 1); - if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) { - notify_key(key, NOTIFY_KEY_INVALIDATED, 0); - key_schedule_gc_links(); + return ret; + } +@@ -833,14 +864,22 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) + for (i = 0; i < PTRS_PER_PMD; i++) { + if (!pmd_none(pmd_sv[i])) { + pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); ++#ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, pte); ++#else + free_page((unsigned long)pte); ++#endif } - up_write(&key->sem); -+ #endif } - } - EXPORT_SYMBOL(key_invalidate); -@@ -1186,9 +1407,16 @@ int generic_key_instantiate(struct key *key, struct key_preparsed_payload *prep) - ret = key_payload_reserve(key, prep->quotalen); - if (ret == 0) { - rcu_assign_keypointer(key, prep->payload.data[0]); -+ #ifdef CONFIG_KEYP -+ union key_payload *key_payload = ((union key_payload *)(key->name_link.next)); -+ key_payload->data[1] = prep->payload.data[1]; -+ key_payload->data[2] = prep->payload.data[2]; -+ key_payload->data[3] = prep->payload.data[3]; -+ #else - key->payload.data[1] = prep->payload.data[1]; - key->payload.data[2] = prep->payload.data[2]; - key->payload.data[3] = prep->payload.data[3]; -+ #endif - prep->payload.data[0] = NULL; - prep->payload.data[1] = NULL; - prep->payload.data[2] = NULL; -@@ -1262,6 +1490,11 @@ void __init key_init(void) - /* allocate a slab in which we can store keys */ - key_jar = kmem_cache_create("key_jar", sizeof(struct key), - 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); -+ #ifdef CONFIG_KEYP -+ key_union_jar = kmem_cache_create("key_union_jar", sizeof(struct key_union), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); -+ key_struct_jar = kmem_cache_create("key_struct_jar", sizeof(struct key_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); -+ key_payload_jar = kmem_cache_create("key_payload_jar", sizeof(union key_payload)*2, 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); -+ #endif - /* add the special key types */ - list_add_tail(&key_type_keyring.link, &key_types_list); -diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c -index aa1dc43b16dd..e92df31642d5 100644 ---- a/security/keys/keyctl.c -+++ b/security/keys/keyctl.c -@@ -23,6 +23,12 @@ - #include - #include - #include "internal.h" -+#ifdef CONFIG_CREDP -+#include -+#endif -+#ifdef CONFIG_KEYP -+#include + free_page((unsigned long)pmd_sv); + + pagetable_pmd_dtor(virt_to_ptdesc(pmd)); ++#ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, pmd); ++#else + free_page((unsigned long)pmd); +#endif - #define KEY_MAX_DESC_SIZE 4096 + return 1; + } +@@ -863,7 +902,11 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) + /* INVLPG to clear all paging-structure caches */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); -@@ -804,11 +810,19 @@ static long __keyctl_read_key(struct key *key, char *buffer, size_t buflen) - { - long ret; ++#ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, pte); ++#else + free_page((unsigned long)pte); ++#endif -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(key)); -+ #else - down_read(&key->sem); -+ #endif - ret = key_validate(key); - if (ret == 0) - ret = key->type->read(key, buffer, buflen); -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(key)); -+ #else - up_read(&key->sem); -+ #endif - return ret; + return 1; } +diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c +index 916247c1d2a3..a47af6f9d677 100644 +--- a/arch/x86/mm/pti.c ++++ b/arch/x86/mm/pti.c +@@ -139,7 +139,11 @@ pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) + * The user page tables get the full PGD, accessible from + * userspace: + */ ++#ifdef CONFIG_PTP ++ ptp_set_pgd(kernel_to_user_pgdp(pgdp), pgd); ++#else + kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; ++#endif -@@ -978,7 +992,11 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) - - /* make the changes with the locks held to prevent chown/chown races */ - ret = -EACCES; -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif + /* + * If this is normal user memory, make it NX in the kernel +@@ -292,7 +296,11 @@ static void __init pti_setup_vsyscall(void) + if (WARN_ON(!target_pte)) + return; - { - bool is_privileged_op = false; -@@ -1036,19 +1054,32 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) - } ++#ifdef CONFIG_PTP ++ set_pte(target_pte, *pte); ++#else + *target_pte = *pte; ++#endif + set_vsyscall_pgtable_user_bits(kernel_to_user_pgdp(swapper_pg_dir)); + } + #else +@@ -370,14 +378,22 @@ pti_clone_pgtable(unsigned long start, unsigned long end, + * code that only set this bit when supported. + */ + if (boot_cpu_has(X86_FEATURE_PGE)) ++#ifdef CONFIG_PTP ++ set_pmd(pmd, pmd_set_flags(*pmd, _PAGE_GLOBAL)); ++#else + *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); ++#endif - zapowner = key->user; -+ #ifdef CONFIG_KEYP -+ iee_set_key_user(key, newowner); -+ iee_set_key_uid(key, uid); -+ #else - key->user = newowner; - key->uid = uid; -+ #endif - } + /* + * Copy the PMD. That is, the kernelmode and usermode + * tables will share the last-level page tables of this + * address range + */ ++#ifdef CONFIG_PTP ++ set_pmd(target_pmd, *pmd); ++#else + *target_pmd = *pmd; ++#endif - /* change the GID */ - if (group != (gid_t) -1) -+ #ifdef CONFIG_KEYP -+ iee_set_key_gid(key, gid); -+ #else - key->gid = gid; -+ #endif + addr = round_up(addr + 1, PMD_SIZE); - notify_key(key, NOTIFY_KEY_SETATTR, 0); - ret = 0; +@@ -401,10 +417,18 @@ pti_clone_pgtable(unsigned long start, unsigned long end, - error_put: -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif - key_put(key); - if (zapowner) - key_user_put(zapowner); -@@ -1090,16 +1121,28 @@ long keyctl_setperm_key(key_serial_t id, key_perm_t perm) - - /* make the changes with the locks held to prevent chown/chmod races */ - ret = -EACCES; -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif + /* Set GLOBAL bit in both PTEs */ + if (boot_cpu_has(X86_FEATURE_PGE)) ++#ifdef CONFIG_PTP ++ set_pte(pte, pte_set_flags(*pte, _PAGE_GLOBAL)); ++#else + *pte = pte_set_flags(*pte, _PAGE_GLOBAL); ++#endif - /* if we're not the sysadmin, we can only change a key that we own */ - if (uid_eq(key->uid, current_fsuid()) || capable(CAP_SYS_ADMIN)) { -+ #ifdef CONFIG_KEYP -+ iee_set_key_perm(key, perm); -+ #else - key->perm = perm; -+ #endif - notify_key(key, NOTIFY_KEY_SETATTR, 0); - ret = 0; - } + /* Clone the PTE */ ++#ifdef CONFIG_PTP ++ set_pte(target_pte, *pte); ++#else + *target_pte = *pte; ++#endif -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif - key_put(key); - error: - return ret; -@@ -1155,7 +1198,11 @@ static int keyctl_change_reqkey_auth(struct key *key) - return -ENOMEM; + addr = round_up(addr + 1, PAGE_SIZE); - key_put(new->request_key_auth); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_request_key_auth(new,key_get(key)); -+ #else - new->request_key_auth = key_get(key); -+ #endif +@@ -430,7 +454,11 @@ static void __init pti_clone_p4d(unsigned long addr) - return commit_creds(new); + kernel_pgd = pgd_offset_k(addr); + kernel_p4d = p4d_offset(kernel_pgd, addr); ++#ifdef CONFIG_PTP ++ set_p4d(user_p4d, *kernel_p4d); ++#else + *user_p4d = *kernel_p4d; ++#endif } -@@ -1196,7 +1243,11 @@ static long keyctl_instantiate_key_common(key_serial_t id, - if (!instkey) - goto error; - -+ #ifdef CONFIG_KEYP -+ rka = ((union key_payload *)(instkey->name_link.next))->data[0]; -+ #else - rka = instkey->payload.data[0]; -+ #endif - if (rka->target_key->serial != id) - goto error; - -@@ -1358,7 +1409,11 @@ long keyctl_reject_key(key_serial_t id, unsigned timeout, unsigned error, - if (!instkey) - goto error; -+ #ifdef CONFIG_KEYP -+ rka = ((union key_payload *)(instkey->name_link.next))->data[0]; -+ #else - rka = instkey->payload.data[0]; -+ #endif - if (rka->target_key->serial != id) - goto error; + /* +@@ -461,7 +489,11 @@ static void __init pti_clone_user_shared(void) + if (WARN_ON(!target_pte)) + return; -@@ -1432,7 +1487,11 @@ long keyctl_set_reqkey_keyring(int reqkey_defl) ++#ifdef CONFIG_PTP ++ set_pte(target_pte, pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL)); ++#else + *target_pte = pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL); ++#endif } + } - set: -+ #ifdef CONFIG_CREDP -+ iee_set_cred_jit_keyring(new,reqkey_defl); -+ #else - new->jit_keyring = reqkey_defl; -+ #endif - commit_creds(new); - return old_setting; - error: -@@ -1644,9 +1703,17 @@ long keyctl_session_to_parent(void) - cred = cred_alloc_blank(); - if (!cred) - goto error_keyring; -+ #ifdef CONFIG_CREDP -+ newwork = (struct rcu_head *)(cred->rcu.func); -+ #else - newwork = &cred->rcu; -+ #endif +diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c +index 91d31ac422d6..b9c707a58507 100644 +--- a/arch/x86/platform/efi/efi_64.c ++++ b/arch/x86/platform/efi/efi_64.c +@@ -48,6 +48,10 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#include ++#endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_session_keyring(cred,key_ref_to_ptr(keyring_r)); -+ #else - cred->session_keyring = key_ref_to_ptr(keyring_r); -+ #endif - keyring_r = NULL; - init_task_work(newwork, key_change_session_keyring); - -@@ -1705,7 +1772,11 @@ long keyctl_session_to_parent(void) - write_unlock_irq(&tasklist_lock); - rcu_read_unlock(); - if (oldwork) -+ #ifdef CONFIG_CREDP -+ put_cred(*(struct cred **)(oldwork + 1)); -+ #else - put_cred(container_of(oldwork, struct cred, rcu)); -+ #endif - if (newwork) - put_cred(cred); - return ret; -@@ -1814,25 +1885,45 @@ long keyctl_watch_key(key_serial_t id, int watch_queue_fd, int watch_id) - if (ret < 0) - goto err_watch; + /* + * We allocate runtime services regions top-down, starting from -4G, i.e. +@@ -73,7 +77,11 @@ int __init efi_alloc_page_tables(void) + gfp_t gfp_mask; -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif - if (!key->watchers) { -+ #ifdef CONFIG_KEYP -+ iee_set_key_watchers(key, wlist); -+ #else - key->watchers = wlist; -+ #endif - wlist = NULL; - } + gfp_mask = GFP_KERNEL | __GFP_ZERO; ++#ifdef CONFIG_PTP ++ efi_pgd = ptp_pg_alloc(&pgd_cache, gfp_mask); ++#else + efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); ++#endif + if (!efi_pgd) + goto fail; - ret = add_watch_to_object(watch, key->watchers); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif +@@ -116,7 +124,11 @@ void efi_sync_low_kernel_mappings(void) + pgd_k = pgd_offset_k(PAGE_OFFSET); - if (ret == 0) - watch = NULL; - } else { - ret = -EBADSLT; - if (key->watchers) { -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(key)); -+ #else - down_write(&key->sem); -+ #endif - ret = remove_watch_from_object(key->watchers, - wqueue, key_serial(key), - false); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(key)); -+ #else - up_write(&key->sem); -+ #endif - } - } + num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); ++#ifdef CONFIG_PTP ++ iee_memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); ++#else + memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); ++#endif -diff --git a/security/keys/keyring.c b/security/keys/keyring.c -index f331725d5a37..b8bb07a55130 100644 ---- a/security/keys/keyring.c -+++ b/security/keys/keyring.c -@@ -21,6 +21,11 @@ - #include - #include "internal.h" + pgd_efi = efi_pgd + pgd_index(EFI_VA_END); + pgd_k = pgd_offset_k(EFI_VA_END); +@@ -124,7 +136,11 @@ void efi_sync_low_kernel_mappings(void) + p4d_k = p4d_offset(pgd_k, 0); -+#ifdef CONFIG_KEYP -+#include -+#include + num_entries = p4d_index(EFI_VA_END); ++#ifdef CONFIG_PTP ++ iee_memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); ++#else + memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); +#endif -+ - /* - * When plumbing the depths of the key tree, this sets a hard limit - * set on how deep we're willing to go. -@@ -112,7 +117,11 @@ static void keyring_publish_name(struct key *keyring) - keyring->description[0] && - keyring->description[0] != '.') { - write_lock(&keyring_name_lock); -+ #ifdef CONFIG_KEYP -+ list_add_tail(&(((struct key_struct *)(keyring->name_link.prev))->name_link), &ns->keyring_name_list); -+ #else - list_add_tail(&keyring->name_link, &ns->keyring_name_list); -+ #endif - write_unlock(&keyring_name_lock); - } - } -@@ -140,7 +149,11 @@ static void keyring_free_preparse(struct key_preparsed_payload *prep) - static int keyring_instantiate(struct key *keyring, - struct key_preparsed_payload *prep) - { -+ #ifdef CONFIG_KEYP -+ assoc_array_init(&((struct key_struct *)(keyring->name_link.prev))->keys); -+ #else - assoc_array_init(&keyring->keys); -+ #endif - /* make the keyring available by name if it has one */ - keyring_publish_name(keyring); - return 0; -@@ -207,13 +220,70 @@ static void hash_key_type_and_desc(struct keyring_index_key *index_key) - index_key->hash = hash; - } -+#ifdef CONFIG_KEYP -+static void iee_hash_key_type_and_desc(struct keyring_index_key *index_key) -+{ -+ const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP; -+ const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK; -+ const char *description = index_key->description; -+ unsigned long hash, type; -+ u32 piece; -+ u64 acc; -+ int n, desc_len = index_key->desc_len; -+ -+ type = (unsigned long)index_key->type; -+ acc = mult_64x32_and_fold(type, desc_len + 13); -+ acc = mult_64x32_and_fold(acc, 9207); -+ piece = (unsigned long)index_key->domain_tag; -+ acc = mult_64x32_and_fold(acc, piece); -+ acc = mult_64x32_and_fold(acc, 9207); -+ -+ for (;;) { -+ n = desc_len; -+ if (n <= 0) -+ break; -+ if (n > 4) -+ n = 4; -+ piece = 0; -+ memcpy(&piece, description, n); -+ description += n; -+ desc_len -= n; -+ acc = mult_64x32_and_fold(acc, piece); -+ acc = mult_64x32_and_fold(acc, 9207); -+ } -+ -+ /* Fold the hash down to 32 bits if need be. */ -+ hash = acc; -+ if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32) -+ hash ^= acc >> 32; -+ -+ /* Squidge all the keyrings into a separate part of the tree to -+ * ordinary keys by making sure the lowest level segment in the hash is -+ * zero for keyrings and non-zero otherwise. -+ */ -+ if (index_key->type != &key_type_keyring && (hash & fan_mask) == 0) -+ hash |= (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1; -+ else if (index_key->type == &key_type_keyring && (hash & fan_mask) != 0) -+ hash = (hash + (hash << level_shift)) & ~fan_mask; -+ struct keyring_index_key tmp = *index_key; -+ tmp.hash = hash; -+ iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); -+} -+#endif -+ -+#ifdef CONFIG_KEYP -+static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; + /* + * We share all the PUD entries apart from those that map the +@@ -139,13 +155,21 @@ void efi_sync_low_kernel_mappings(void) + pud_k = pud_offset(p4d_k, 0); + + num_entries = pud_index(EFI_VA_END); ++#ifdef CONFIG_PTP ++ iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); ++#else + memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif -+ - /* - * Finalise an index key to include a part of the description actually in the - * index key, to set the domain tag and to calculate the hash. - */ - void key_set_index_key(struct keyring_index_key *index_key) - { -+ #ifndef CONFIG_KEYP - static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), }; -+ #endif - size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); - memcpy(index_key->desc, index_key->description, n); -@@ -228,6 +298,33 @@ void key_set_index_key(struct keyring_index_key *index_key) - hash_key_type_and_desc(index_key); - } + pud_efi = pud_offset(p4d_efi, EFI_VA_START); + pud_k = pud_offset(p4d_k, EFI_VA_START); -+#ifdef CONFIG_KEYP -+void iee_key_set_index_key(struct keyring_index_key *index_key) -+{ -+ size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc)); -+ struct keyring_index_key tmp; -+ -+ iee_memcpy(index_key->desc, index_key->description, n); -+ -+ if (!index_key->domain_tag) { -+ if (index_key->type->flags & KEY_TYPE_NET_DOMAIN) -+ { -+ tmp = *index_key; -+ tmp.domain_tag = current->nsproxy->net_ns->key_domain; -+ iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); -+ } -+ else -+ { -+ tmp = *index_key; -+ tmp.domain_tag = &default_domain_tag; -+ iee_set_key_index_key(container_of(index_key, struct key, index_key), &tmp); -+ } -+ } -+ -+ iee_hash_key_type_and_desc(index_key); -+} + num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); ++#ifdef CONFIG_PTP ++ iee_memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); ++#else + memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); +#endif -+ - /** - * key_put_tag - Release a ref on a tag. - * @tag: The tag to release. -@@ -414,9 +511,15 @@ static void keyring_destroy(struct key *keyring) - if (keyring->description) { - write_lock(&keyring_name_lock); - -+ #ifdef CONFIG_KEYP -+ if(((struct key_struct *)(keyring->name_link.prev))->name_link.next != NULL && -+ !list_empty(&(((struct key_struct *)(keyring->name_link.prev))->name_link))) -+ list_del(&(((struct key_struct *)(keyring->name_link.prev))->name_link)); -+ #else - if (keyring->name_link.next != NULL && - !list_empty(&keyring->name_link)) - list_del(&keyring->name_link); -+ #endif + } - write_unlock(&keyring_name_lock); - } -@@ -428,7 +531,11 @@ static void keyring_destroy(struct key *keyring) - kfree(keyres); + /* +diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c +index 83092d93f36a..fb12e7d0660a 100644 +--- a/drivers/firmware/efi/arm-runtime.c ++++ b/drivers/firmware/efi/arm-runtime.c +@@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void) + return 0; } -+ #ifdef CONFIG_KEYP -+ assoc_array_destroy(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops); ++ #ifdef CONFIG_PTP ++ efi_memmap_unmap_after_init(); + #else - assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops); + efi_memmap_unmap(); + #endif - } - - /* -@@ -442,8 +549,13 @@ static void keyring_describe(const struct key *keyring, struct seq_file *m) - seq_puts(m, "[anon]"); - if (key_is_positive(keyring)) { -+ #ifdef CONFIG_KEYP -+ if (((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree != 0) -+ seq_printf(m, ": %lu", ((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree); -+ #else - if (keyring->keys.nr_leaves_on_tree != 0) - seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree); -+ #endif - else - seq_puts(m, ": empty"); - } -@@ -494,8 +606,13 @@ static long keyring_read(const struct key *keyring, - ctx.buffer = (key_serial_t *)buffer; - ctx.buflen = buflen; - ctx.count = 0; -+ #ifdef CONFIG_KEYP -+ ret = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, -+ keyring_read_iterator, &ctx); -+ #else - ret = assoc_array_iterate(&keyring->keys, - keyring_read_iterator, &ctx); -+ #endif - if (ret < 0) { - kleave(" = %ld [iterate]", ret); - return ret; -@@ -503,7 +620,11 @@ static long keyring_read(const struct key *keyring, - } + mapsize = efi.memmap.desc_size * efi.memmap.nr_map; - /* Return the size of the buffer needed */ -+ #ifdef CONFIG_KEYP -+ ret = ((struct key_struct *)(keyring->name_link.prev))->keys.nr_leaves_on_tree * sizeof(key_serial_t); +diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c +index a5fbb6ed38ae..93f80a19c194 100644 +--- a/drivers/tty/serial/earlycon.c ++++ b/drivers/tty/serial/earlycon.c +@@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size) + { + void __iomem *base; + #ifdef CONFIG_FIX_EARLYCON_MEM ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO); + #else - ret = keyring->keys.nr_leaves_on_tree * sizeof(key_serial_t); + set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK); + #endif - if (ret <= buflen) - kleave("= %ld [ok]", ret); - else -@@ -648,12 +769,22 @@ static int search_keyring(struct key *keyring, struct keyring_search_context *ct - if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_DIRECT) { - const void *object; - -+ #ifdef CONFIG_KEYP -+ object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, -+ &keyring_assoc_array_ops, -+ &ctx->index_key); -+ #else - object = assoc_array_find(&keyring->keys, - &keyring_assoc_array_ops, - &ctx->index_key); -+ #endif - return object ? ctx->iterator(object, ctx) : 0; - } -+ #ifdef CONFIG_KEYP -+ return assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, ctx->iterator, ctx); + base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE); + base += paddr & ~PAGE_MASK; + #else +diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c +index 45b42d8f6453..ac3b1d268969 100644 +--- a/drivers/usb/early/ehci-dbgp.c ++++ b/drivers/usb/early/ehci-dbgp.c +@@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s) + * FIXME I don't have the bar size so just guess PAGE_SIZE is more + * than enough. 1K is the biggest I have seen. + */ ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE); + #else - return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx); + set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK); + #endif - } + ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE); + ehci_bar += bar_val & ~PAGE_MASK; + dbgp_printk("ehci_bar: %p\n", ehci_bar); +diff --git a/fs/exec.c b/fs/exec.c +index 936c59235ef7..5a2ea88ebd13 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -79,6 +79,12 @@ + #include + #endif - /* -@@ -729,7 +860,11 @@ static bool search_nested_keyrings(struct key *keyring, - if (!(ctx->flags & KEYRING_SEARCH_RECURSE)) - goto not_this_keyring; ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include ++#endif + static int bprm_creds_from_file(struct linux_binprm *bprm); -+ #ifdef CONFIG_KEYP -+ ptr = READ_ONCE(((struct key_struct *)(keyring->name_link.prev))->keys.root); -+ #else - ptr = READ_ONCE(keyring->keys.root); -+ #endif - if (!ptr) - goto not_this_keyring; - -@@ -856,10 +991,17 @@ static bool search_nested_keyrings(struct key *keyring, - key = key_ref_to_ptr(ctx->result); - key_check(key); - if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) { -+ #ifdef CONFIG_KEYP -+ iee_set_key_last_used_at(key, ctx->now); -+ iee_set_key_last_used_at(keyring, ctx->now); -+ while (sp > 0) -+ iee_set_key_last_used_at(stack[--sp].keyring, ctx->now); -+ #else - key->last_used_at = ctx->now; - keyring->last_used_at = ctx->now; - while (sp > 0) - stack[--sp].keyring->last_used_at = ctx->now; -+ #endif - } - kleave(" = true"); - return true; -@@ -1056,7 +1198,11 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, - goto error; + int suid_dumpable = 0; +@@ -735,7 +741,11 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + free_pgd_range(&tlb, old_start, old_end, new_end, + next ? next->vm_start : USER_PGTABLES_CEILING); } ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_tlb_finish_mmu(&tlb); ++#else + tlb_finish_mmu(&tlb); ++#endif -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); -+ #else - down_write(&keyring->sem); -+ #endif - down_write(&keyring_serialise_restrict_sem); + vma_prev(&vmi); + /* Shrink the vma to just the new range */ +@@ -1025,6 +1035,9 @@ static int exec_mmap(struct mm_struct *mm) + if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + activate_mm(active_mm, mm); ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(tsk, mm->pgd); ++#endif + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + lru_gen_add_mm(mm); +@@ -1926,10 +1939,17 @@ static int do_execveat_common(int fd, struct filename *filename, + { + struct linux_binprm *bprm; + int retval; ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; ++#endif - if (keyring->restrict_link) { -@@ -1064,12 +1210,20 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, - } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) { - ret = -EDEADLK; - } else { -+ #ifdef CONFIG_KEYP -+ iee_set_key_restrict_link(keyring, restrict_link); -+ #else - keyring->restrict_link = restrict_link; -+ #endif - notify_key(keyring, NOTIFY_KEY_SETATTR, 0); - } + if (IS_ERR(filename)) + return PTR_ERR(filename); - up_write(&keyring_serialise_restrict_sem); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(keyring)); -+ #else - up_write(&keyring->sem); -+ #endif ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_disable_iee(®); ++#endif ++ + /* + * We move the actual failure in case of RLIMIT_NPROC excess from + * set*uid() to execve() because too many poorly written programs +@@ -2001,6 +2021,9 @@ static int do_execveat_common(int fd, struct filename *filename, - if (ret < 0) { - key_put(restrict_link->key); -@@ -1109,8 +1263,13 @@ key_ref_t find_key_to_update(key_ref_t keyring_ref, - kenter("{%d},{%s,%s}", - keyring->serial, index_key->type->name, index_key->description); + out_ret: + putname(filename); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_enable_iee(reg); ++#endif + return retval; + } -+ #ifdef CONFIG_KEYP -+ object = assoc_array_find(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops, -+ index_key); -+ #else - object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops, - index_key); -+ #endif +diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h +index 22384baee10e..8ad81b7209f0 100644 +--- a/include/asm-generic/tlb.h ++++ b/include/asm-generic/tlb.h +@@ -209,6 +209,9 @@ struct mmu_table_batch { + ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) - if (object) - goto found; -@@ -1154,7 +1313,13 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) - /* Search this hash bucket for a keyring with a matching name that - * grants Search permission and that hasn't been revoked - */ -+ #ifdef CONFIG_KEYP -+ for(keyring = list_first_entry(&ns->keyring_name_list, struct key_struct, name_link)->key; -+ !(&(((struct key_struct *)(keyring->name_link.prev))->name_link) == (&ns->keyring_name_list)); -+ keyring = list_entry(((struct key_struct *)(keyring->name_link.prev))->name_link.next, struct key_struct, name_link)->key) { -+ #else - list_for_each_entry(keyring, &ns->keyring_name_list, name_link) { -+ #endif - if (!kuid_has_mapping(ns, keyring->user->uid)) - continue; - -@@ -1177,9 +1342,15 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) - /* we've got a match but we might end up racing with - * key_cleanup() if the keyring is currently 'dead' - * (ie. it has a zero usage count) */ -+ #ifdef CONFIG_KEYP -+ if (!iee_set_key_usage(keyring, 0, REFCOUNT_INC_NOT_ZERO)) -+ continue; -+ iee_set_key_last_used_at(keyring, ktime_get_real_seconds()); -+ #else - if (!refcount_inc_not_zero(&keyring->usage)) - continue; - keyring->last_used_at = ktime_get_real_seconds(); -+ #endif - goto out; - } + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++extern void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table); ++#endif -@@ -1238,13 +1409,21 @@ static int keyring_detect_cycle(struct key *A, struct key *B) - */ - int __key_link_lock(struct key *keyring, - const struct keyring_index_key *index_key) -+ #ifdef CONFIG_KEYP -+ __acquires(&KEY_SEM(keyring)) -+ #else - __acquires(&keyring->sem) -+ #endif - __acquires(&keyring_serialise_link_lock) - { - if (keyring->type != &key_type_keyring) - return -ENOTDIR; + #else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */ -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); -+ #else - down_write(&keyring->sem); -+ #endif +@@ -361,6 +364,9 @@ struct mmu_gather { + }; - /* Serialise link/link calls to prevent parallel calls causing a cycle - * when linking two keyring in opposite orders. -@@ -1260,8 +1439,13 @@ int __key_link_lock(struct key *keyring, - */ - int __key_move_lock(struct key *l_keyring, struct key *u_keyring, - const struct keyring_index_key *index_key) -+ #ifdef CONFIG_KEYP -+ __acquires(&KEY_SEM(l_keyring)) -+ __acquires(&KEY_SEM(u_keyring)) -+ #else - __acquires(&l_keyring->sem) - __acquires(&u_keyring->sem) -+ #endif - __acquires(&keyring_serialise_link_lock) - { - if (l_keyring->type != &key_type_keyring || -@@ -1273,11 +1457,21 @@ int __key_move_lock(struct key *l_keyring, struct key *u_keyring, - * move operation. - */ - if (l_keyring < u_keyring) { -+ #ifdef CNOFIG_KEYP -+ down_write(&KEY_SEM(l_keyring)); -+ down_write_nested(&KEY_SEM(u_keyring), 1); -+ #else - down_write(&l_keyring->sem); - down_write_nested(&u_keyring->sem, 1); -+ #endif - } else { -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(u_keyring)); -+ down_write_nested(&KEY_SEM(l_keyring), 1); -+ #else - down_write(&u_keyring->sem); - down_write_nested(&l_keyring->sem, 1); -+ #endif - } + void tlb_flush_mmu(struct mmu_gather *tlb); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++void ptp_tlb_flush_mmu(struct mmu_gather *tlb); ++#endif - /* Serialise link/link calls to prevent parallel calls causing a cycle -@@ -1314,10 +1508,17 @@ int __key_link_begin(struct key *keyring, - /* Create an edit script that will insert/replace the key in the - * keyring tree. - */ -+ #ifdef CONFIG_KEYP -+ edit = assoc_array_insert(&((struct key_struct *)(keyring->name_link.prev))->keys, -+ &keyring_assoc_array_ops, -+ index_key, -+ NULL); -+ #else - edit = assoc_array_insert(&keyring->keys, - &keyring_assoc_array_ops, - index_key, - NULL); -+ #endif - if (IS_ERR(edit)) { - ret = PTR_ERR(edit); - goto error; -@@ -1385,7 +1586,11 @@ void __key_link(struct key *keyring, struct key *key, - void __key_link_end(struct key *keyring, - const struct keyring_index_key *index_key, - struct assoc_array_edit *edit) -+ #ifdef CONFIG_KEYP -+ __releases(&KEY_SEM(keyring)) -+ #else - __releases(&keyring->sem) -+ #endif - __releases(&keyring_serialise_link_lock) - { - BUG_ON(index_key->type == NULL); -@@ -1398,7 +1603,11 @@ void __key_link_end(struct key *keyring, - } - assoc_array_cancel_edit(edit); - } -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(keyring)); -+ #else - up_write(&keyring->sem); -+ #endif + static inline void __tlb_adjust_range(struct mmu_gather *tlb, + unsigned long address, +@@ -476,6 +482,15 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb, + tlb_flush_mmu(tlb); + } - if (index_key->type == &key_type_keyring) - mutex_unlock(&keyring_serialise_link_lock); -@@ -1411,8 +1620,13 @@ static int __key_link_check_restriction(struct key *keyring, struct key *key) ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static inline void ptp_tlb_remove_page_size(struct mmu_gather *tlb, ++ struct page *page, int page_size) ++{ ++ if (__tlb_remove_page_size(tlb, page, false, page_size)) ++ ptp_tlb_flush_mmu(tlb); ++} ++#endif ++ + static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb, + struct page *page, bool delay_rmap) { - if (!keyring->restrict_link || !keyring->restrict_link->check) - return 0; -+ #ifdef CONFIG_KEYP -+ return keyring->restrict_link->check(keyring, key->type, ((union key_payload *)(key->name_link.next)), -+ keyring->restrict_link->key); -+ #else - return keyring->restrict_link->check(keyring, key->type, &key->payload, - keyring->restrict_link->key); -+ #endif +@@ -491,9 +506,19 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) + return tlb_remove_page_size(tlb, page, PAGE_SIZE); } - /** -@@ -1472,12 +1686,20 @@ EXPORT_SYMBOL(key_link); - * Lock a keyring for unlink. - */ - static int __key_unlink_lock(struct key *keyring) -+ #ifdef CONFIG_KEYP -+ __acquires(&KEY_SEM(keyring)) -+ #else - __acquires(&keyring->sem) -+ #endif ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static inline void ptp_tlb_remove_page(struct mmu_gather *tlb, struct page *page) ++{ ++ return ptp_tlb_remove_page_size(tlb, page, PAGE_SIZE); ++} ++#endif + static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) { - if (keyring->type != &key_type_keyring) - return -ENOTDIR; - -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); ++ #if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_tlb_remove_table(tlb, pt); + #else - down_write(&keyring->sem); + tlb_remove_table(tlb, pt); + #endif - return 0; } -@@ -1491,8 +1713,13 @@ static int __key_unlink_begin(struct key *keyring, struct key *key, - - BUG_ON(*_edit != NULL); + /* Like tlb_remove_ptdesc, but for page-like page directories. */ +diff --git a/include/linux/efi.h b/include/linux/efi.h +index 01032318202f..618ab6d06a6a 100644 +--- a/include/linux/efi.h ++++ b/include/linux/efi.h +@@ -27,6 +27,9 @@ -+ #ifdef CONFIG_KEYP -+ edit = assoc_array_delete(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops, -+ &key->index_key); -+ #else - edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops, - &key->index_key); -+ #endif - if (IS_ERR(edit)) - return PTR_ERR(edit); - -@@ -1521,11 +1748,19 @@ static void __key_unlink(struct key *keyring, struct key *key, - static void __key_unlink_end(struct key *keyring, - struct key *key, - struct assoc_array_edit *edit) -+ #ifdef CONFIG_KEYP -+ __releases(&KEY_SEM(keyring)) -+ #else - __releases(&keyring->sem) -+ #endif - { - if (edit) - assoc_array_cancel_edit(edit); -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(keyring)); -+ #else - up_write(&keyring->sem); -+ #endif - } + #include + #include ++#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++#include ++#endif - /** -@@ -1655,9 +1890,15 @@ int keyring_clear(struct key *keyring) - if (keyring->type != &key_type_keyring) - return -ENOTDIR; + struct screen_info; -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); -+ -+ edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops); -+ #else - down_write(&keyring->sem); +diff --git a/include/linux/mm.h b/include/linux/mm.h +index c0040a2014c4..2b5d37174346 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -31,6 +31,9 @@ + #include + #include + #include ++#ifdef CONFIG_PTP ++#include ++#endif - edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); -+ #endif - if (IS_ERR(edit)) { - ret = PTR_ERR(edit); - } else { -@@ -1668,7 +1909,11 @@ int keyring_clear(struct key *keyring) - ret = 0; - } + struct mempolicy; + struct anon_vma; +@@ -2975,7 +2978,11 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) + */ + static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) + { ++#ifdef CONFIG_PTP ++ struct page *page = virt_to_page((unsigned long)ptp_pg_alloc(&pg_cache, gfp)); ++#else + struct page *page = alloc_pages(gfp | __GFP_COMP, order); ++#endif -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(keyring)); -+ #else - up_write(&keyring->sem); -+ #endif - return ret; + return page_ptdesc(page); } - EXPORT_SYMBOL(keyring_clear); -@@ -1682,7 +1927,11 @@ static void keyring_revoke(struct key *keyring) +@@ -2991,7 +2998,11 @@ static inline void pagetable_free(struct ptdesc *pt) { - struct assoc_array_edit *edit; + struct page *page = ptdesc_page(pt); -+ #ifdef CONFIG_KEYP -+ edit = assoc_array_clear(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops); -+ #else - edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops); -+ #endif - if (!IS_ERR(edit)) { - if (edit) - assoc_array_apply_edit(edit); -@@ -1728,8 +1977,13 @@ void keyring_gc(struct key *keyring, time64_t limit) - - /* scan the keyring looking for dead keys */ - rcu_read_lock(); -+ #ifdef CONFIG_KEYP -+ result = assoc_array_iterate(&((struct key_struct *)(keyring->name_link.prev))->keys, -+ keyring_gc_check_iterator, &limit); -+ #else - result = assoc_array_iterate(&keyring->keys, - keyring_gc_check_iterator, &limit); -+ #endif - rcu_read_unlock(); - if (result == true) - goto do_gc; -@@ -1739,10 +1993,17 @@ void keyring_gc(struct key *keyring, time64_t limit) - return; - - do_gc: -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); -+ assoc_array_gc(&((struct key_struct *)(keyring->name_link.prev))->keys, &keyring_assoc_array_ops, -+ keyring_gc_select_iterator, &limit); -+ up_write(&KEY_SEM(keyring)); ++ #ifdef CONFIG_PTP ++ ptp_pg_free(&pg_cache, page_address(page)); + #else - down_write(&keyring->sem); - assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops, - keyring_gc_select_iterator, &limit); - up_write(&keyring->sem); + __free_pages(page, compound_order(page)); + #endif - kleave(" [gc]"); } -@@ -1781,7 +2042,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) - } + #if USE_SPLIT_PTE_PTLOCKS +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index b25cc93103c3..8acd55114f63 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -1227,6 +1227,9 @@ struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_finish_mmu(struct mmu_gather *tlb); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++extern void ptp_tlb_finish_mmu(struct mmu_gather *tlb); ++#endif - /* Lock the keyring to ensure that a link is not in progress */ -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(keyring)); -+ #else - down_write(&keyring->sem); -+ #endif + struct vm_fault; - keyres = keyring->restrict_link; +diff --git a/include/linux/ptp-cache.h b/include/linux/ptp-cache.h +new file mode 100644 +index 000000000000..57d3c232b0c2 +--- /dev/null ++++ b/include/linux/ptp-cache.h +@@ -0,0 +1,37 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_PTP_CACHE_H ++#define _LINUX_PTP_CACHE_H ++#include ++union freelist_aba_t { ++ struct { ++ void **freelist; ++ unsigned long counter; ++ }; ++ u128 full; ++}; ++ ++struct pg_cache { ++ union { ++ struct { ++ void **freelist; ++ unsigned long tid; ++ }; ++ union freelist_aba_t freelist_tid; ++ }; ++ unsigned long reserve_order; ++ unsigned long reserve_start_addr; ++ unsigned long reserve_end_addr; ++ unsigned long object_order; ++ int levels; ++ const char *name; ++ atomic_t count; ++ atomic_t fail_count; ++}; ++ ++extern struct pg_cache pg_cache; ++extern void ptp_pg_cache_init(struct pg_cache *cache, unsigned long object_order, ++ int levels, const char *name); ++extern void ptp_set_iee_reserved(struct pg_cache *cache); ++extern void *ptp_pg_alloc(struct pg_cache *cache, gfp_t gfp); ++extern void ptp_pg_free(struct pg_cache *cache, void *object); ++#endif +diff --git a/kernel/exit.c b/kernel/exit.c +index 4c7a08ad7456..55439e7dede5 100644 +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -73,6 +73,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif -@@ -1791,7 +2056,11 @@ void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type) - keyres->key = NULL; - keyres->keytype = NULL; + /* + * The default value should be high enough to not crash a system that randomly +@@ -557,6 +560,9 @@ static void exit_mm(void) + smp_mb__after_spinlock(); + local_irq_disable(); + current->mm = NULL; ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(current, NULL); ++#endif + membarrier_update_current_mm(NULL); + enter_lazy_tlb(mm, current); + local_irq_enable(); +diff --git a/kernel/fork.c b/kernel/fork.c +index f3c663602345..6dbaaa70ac12 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -121,6 +121,12 @@ -+ #ifdef CONFIG_KEYP -+ up_write(&KEY_SEM(keyring)); -+ #else - up_write(&keyring->sem); -+ #endif + #define CREATE_TRACE_POINTS + #include ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include ++#endif - kleave(" [restriction gc]"); + /* + * Minimum number of threads to boot the kernel +@@ -177,7 +183,11 @@ void __weak arch_release_task_struct(struct task_struct *tsk) } -diff --git a/security/keys/proc.c b/security/keys/proc.c -index 4f4e2c1824f1..5f72240ddd96 100644 ---- a/security/keys/proc.c -+++ b/security/keys/proc.c -@@ -67,7 +67,11 @@ static struct rb_node *key_serial_next(struct seq_file *p, struct rb_node *n) - - n = rb_next(n); - while (n) { -+ #ifdef CONFIG_KEYP -+ struct key *key = rb_entry(n, struct key_union, serial_node)->key; -+ #else - struct key *key = rb_entry(n, struct key, serial_node); -+ #endif - if (kuid_has_mapping(user_ns, key->user->uid)) - break; - n = rb_next(n); -@@ -82,7 +86,11 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) - struct key *minkey = NULL; - - while (n) { -+ #ifdef CONFIG_KEYP -+ struct key *key = rb_entry(n, struct key_union, serial_node)->key; -+ #else - struct key *key = rb_entry(n, struct key, serial_node); -+ #endif - if (id < key->serial) { - if (!minkey || minkey->serial > key->serial) - minkey = key; -@@ -102,10 +110,18 @@ static struct key *find_ge_key(struct seq_file *p, key_serial_t id) - for (;;) { - if (kuid_has_mapping(user_ns, minkey->user->uid)) - return minkey; -+ #ifdef CONFIG_KEYP -+ n = rb_next(&(((struct key_union *)(minkey->graveyard_link.next))->serial_node)); -+ #else - n = rb_next(&minkey->serial_node); -+ #endif - if (!n) - return NULL; -+ #ifdef CONFIG_KEYP -+ minkey = rb_entry(n, struct key_union, serial_node)->key; -+ #else - minkey = rb_entry(n, struct key, serial_node); -+ #endif + + #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR ++#ifdef CONFIG_IEE_PTRP ++struct kmem_cache *task_struct_cachep; ++#else + static struct kmem_cache *task_struct_cachep; ++#endif + + static inline struct task_struct *alloc_task_struct_node(int node) + { +@@ -643,6 +653,9 @@ void free_task(struct task_struct *tsk) + #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY + sched_prefer_cpus_free(tsk); + #endif ++#ifdef CONFIG_IEE_PTRP ++ iee_invalidate_token(tsk); ++#endif + #ifdef CONFIG_QOS_SCHED_SMART_GRID + if (smart_grid_enabled()) + sched_grid_qos_free(tsk); +@@ -1812,6 +1825,9 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) + #endif + + tsk->mm = NULL; ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(tsk, NULL); ++#endif + tsk->active_mm = NULL; + + /* +@@ -1843,6 +1859,9 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) } - } -@@ -123,12 +139,20 @@ static void *proc_keys_start(struct seq_file *p, loff_t *_pos) - if (!key) - return NULL; - *_pos = key->serial; -+ #ifdef CONFIG_KEYP -+ return &(((struct key_union *)(key->graveyard_link.next))->serial_node); -+ #else - return &key->serial_node; -+ #endif - } + tsk->mm = mm; ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(tsk, mm->pgd); ++#endif + tsk->active_mm = mm; + sched_mm_cid_fork(tsk); + return 0; +@@ -2430,6 +2449,9 @@ __latent_entropy struct task_struct *copy_process( + p = dup_task_struct(current, node); + if (!p) + goto fork_out; ++#ifdef CONFIG_IEE_PTRP ++ iee_validate_token(p); ++#endif + p->flags &= ~PF_KTHREAD; + if (args->kthread) + p->flags |= PF_KTHREAD; +@@ -2997,6 +3019,9 @@ pid_t kernel_clone(struct kernel_clone_args *args) + struct task_struct *p; + int trace = 0; + pid_t nr; ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; ++#endif + + /* + * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument +@@ -3030,7 +3055,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) + trace = 0; + } - static inline key_serial_t key_node_serial(struct rb_node *n) - { -+ #ifdef CONFIG_KEYP -+ struct key *key = rb_entry(n, struct key_union, serial_node)->key; -+ #else - struct key *key = rb_entry(n, struct key, serial_node); -+ #endif - return key->serial; - } ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_disable_iee(®); ++#endif + p = copy_process(NULL, trace, NUMA_NO_NODE, args); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_enable_iee(reg); ++#endif + add_latent_entropy(); -@@ -153,7 +177,11 @@ static void proc_keys_stop(struct seq_file *p, void *v) - static int proc_keys_show(struct seq_file *m, void *v) - { - struct rb_node *_p = v; -+ #ifdef CONFIG_KEYP -+ struct key *key = rb_entry(_p, struct key_union, serial_node)->key; -+ #else - struct key *key = rb_entry(_p, struct key, serial_node); -+ #endif - unsigned long flags; - key_ref_t key_ref, skey_ref; - time64_t now, expiry; -diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c -index b5d5333ab330..bff026c9abbf 100644 ---- a/security/keys/process_keys.c -+++ b/security/keys/process_keys.c -@@ -19,6 +19,13 @@ - #include - #include "internal.h" + if (IS_ERR(p)) +diff --git a/kernel/kthread.c b/kernel/kthread.c +index 980e6b325b7d..36346f1595ae 100644 +--- a/kernel/kthread.c ++++ b/kernel/kthread.c +@@ -29,6 +29,9 @@ + #include + #include + #include ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif -+#ifdef CONFIG_CREDP -+#include + + static DEFINE_SPINLOCK(kthread_create_lock); +@@ -1457,6 +1460,9 @@ void kthread_use_mm(struct mm_struct *mm) + tsk->active_mm = mm; + tsk->mm = mm; + membarrier_update_current_mm(mm); ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(tsk, mm->pgd); +#endif -+#ifdef CONFIG_KEYP -+#include + switch_mm_irqs_off(active_mm, mm, tsk); + local_irq_enable(); + task_unlock(tsk); +@@ -1501,6 +1507,9 @@ void kthread_unuse_mm(struct mm_struct *mm) + local_irq_disable(); + tsk->mm = NULL; + membarrier_update_current_mm(NULL); ++#ifdef CONFIG_IEE_PTRP ++ iee_set_token_pgd(tsk, NULL); +#endif -+ - /* Session keyring create vs join semaphore */ - static DEFINE_MUTEX(key_session_mutex); + mmgrab_lazy_tlb(mm); + /* active_mm is still 'mm' */ + enter_lazy_tlb(mm, tsk); +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index d9afdd7e588e..02dafd6e85da 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -98,6 +98,12 @@ + #include "../smpboot.h" + + #include ++#ifdef CONFIG_IEE_PTRP ++#include ++#endif ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include ++#endif + + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); +@@ -5447,6 +5453,9 @@ context_switch(struct rq *rq, struct task_struct *prev, + * case 'prev->active_mm == next->mm' through + * finish_task_switch()'s mmdrop(). + */ ++#ifdef CONFIG_IEE_PTRP ++ iee_verify_token_pgd(next); ++#endif + switch_mm_irqs_off(prev->active_mm, next->mm, next); + lru_gen_use_mm(next->mm); -@@ -232,7 +239,11 @@ int install_thread_keyring_to_cred(struct cred *new) - if (IS_ERR(keyring)) - return PTR_ERR(keyring); +@@ -5462,9 +5471,18 @@ context_switch(struct rq *rq, struct task_struct *prev, -+ #ifdef CONFIG_CREDP -+ iee_set_cred_thread_keyring(new,keyring); -+ #else - new->thread_keyring = keyring; -+ #endif - return 0; - } + prepare_lock_switch(rq, next, rf); -@@ -279,7 +290,11 @@ int install_process_keyring_to_cred(struct cred *new) - if (IS_ERR(keyring)) - return PTR_ERR(keyring); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ int disabled_cnt = 0; ++ unsigned long reg = 0; ++ ++ ptp_context_enable_iee(&disabled_cnt, ®); ++#endif + /* Here we just switch the register state and the stack. */ + switch_to(prev, next, prev); + barrier(); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_context_restore_iee(disabled_cnt, reg); ++#endif -+ #ifdef CONFIG_CREDP -+ iee_set_cred_process_keyring(new,keyring); + return finish_task_switch(prev); + } +diff --git a/mm/Makefile b/mm/Makefile +index e45cdeda47b7..1ba7f1a008d6 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -150,3 +150,4 @@ obj-$(CONFIG_RECLAIM_NOTIFY) += reclaim_notify.o + obj-$(CONFIG_DYNAMIC_POOL) += dynamic_pool.o + obj-$(CONFIG_MEM_SAMPLING) += mem_sampling.o + obj-$(CONFIG_THP_CONTROL) += thp_ctl.o ++obj-$(CONFIG_IEE) += haoc/ +diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c +index 68af76ca8bc9..36766e6f2099 100644 +--- a/mm/debug_vm_pgtable.c ++++ b/mm/debug_vm_pgtable.c +@@ -437,7 +437,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args) + * X86 defined pmd_set_huge() verifies that the given + * PMD is not a populated non-leaf entry. + */ ++ #ifdef CONFIG_PTP ++ set_pmd(args->pmdp, __pmd(0)); + #else - new->process_keyring = keyring; + WRITE_ONCE(*args->pmdp, __pmd(0)); + #endif - return 0; - } - -@@ -338,7 +353,11 @@ int install_session_keyring_to_cred(struct cred *cred, struct key *keyring) - - /* install the keyring */ - old = cred->session_keyring; -+ #ifdef CONFIG_CREDP -+ iee_set_cred_session_keyring(cred,keyring); + WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); + WARN_ON(!pmd_clear_huge(args->pmdp)); + pmd = READ_ONCE(*args->pmdp); +@@ -457,7 +461,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args) + * X86 defined pud_set_huge() verifies that the given + * PUD is not a populated non-leaf entry. + */ ++ #ifdef CONFIG_PTP ++ set_pud(args->pudp, __pud(0)); + #else - cred->session_keyring = keyring; + WRITE_ONCE(*args->pudp, __pud(0)); + #endif + WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); + WARN_ON(!pud_clear_huge(args->pudp)); + pud = READ_ONCE(*args->pudp); +diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c +index ce06b2884789..3ee71ba8e782 100644 +--- a/mm/early_ioremap.c ++++ b/mm/early_ioremap.c +@@ -20,7 +20,11 @@ + #include "internal.h" - if (old) - key_put(old); -@@ -378,9 +397,15 @@ void key_fsuid_changed(struct cred *new_cred) + #ifdef CONFIG_MMU ++#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++int early_ioremap_debug __initdata; ++#else + static int early_ioremap_debug __initdata; ++#endif + + static int __init early_ioremap_debug_setup(char *str) { - /* update the ownership of the thread keyring */ - if (new_cred->thread_keyring) { -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(new_cred->thread_keyring)); -+ iee_set_key_uid(new_cred->thread_keyring, new_cred->fsuid); -+ up_write(&KEY_SEM(new_cred->thread_keyring)); -+ #else - down_write(&new_cred->thread_keyring->sem); - new_cred->thread_keyring->uid = new_cred->fsuid; - up_write(&new_cred->thread_keyring->sem); -+ #endif - } +@@ -30,7 +34,11 @@ static int __init early_ioremap_debug_setup(char *str) } + early_param("early_ioremap_debug", early_ioremap_debug_setup); -@@ -391,9 +416,15 @@ void key_fsgid_changed(struct cred *new_cred) - { - /* update the ownership of the thread keyring */ - if (new_cred->thread_keyring) { -+ #ifdef CONFIG_KEYP -+ down_write(&KEY_SEM(new_cred->thread_keyring)); -+ iee_set_key_gid(new_cred->thread_keyring, new_cred->fsgid); -+ up_write(&KEY_SEM(new_cred->thread_keyring)); -+ #else - down_write(&new_cred->thread_keyring->sem); - new_cred->thread_keyring->gid = new_cred->fsgid; - up_write(&new_cred->thread_keyring->sem); -+ #endif - } ++#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++int after_paging_init __initdata; ++#else + static int after_paging_init __initdata; ++#endif + + pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, + unsigned long size, +@@ -64,9 +72,15 @@ static inline void __init __late_clear_fixmap(enum fixed_addresses idx) } + #endif -@@ -557,7 +588,11 @@ key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx) - const struct cred *cred = ctx->cred; ++#if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; ++unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; ++unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; ++#else + static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; + static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; + static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; ++#endif - if (key_validate(cred->request_key_auth) == 0) { -+ #ifdef CONFIG_KEYP -+ rka = ((union key_payload *)(ctx->cred->request_key_auth->name_link.next))->data[0]; + void __init early_ioremap_setup(void) + { +@@ -147,7 +161,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) + if (after_paging_init) + __late_set_fixmap(idx, phys_addr, prot); + else ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ __iee_set_fixmap_pre_init(idx, phys_addr, prot); + #else - rka = ctx->cred->request_key_auth->payload.data[0]; + __early_set_fixmap(idx, phys_addr, prot); + #endif - - //// was search_process_keyrings() [ie. recursive] - ctx->cred = rka->cred; -@@ -725,17 +760,29 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, - if (!ctx.cred->request_key_auth) - goto error; - -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(ctx.cred->request_key_auth)); -+ #else - down_read(&ctx.cred->request_key_auth->sem); -+ #endif - if (test_bit(KEY_FLAG_REVOKED, - &ctx.cred->request_key_auth->flags)) { - key_ref = ERR_PTR(-EKEYREVOKED); - key = NULL; - } else { -+ #ifdef CONFIG_KEYP -+ rka = ((union key_payload *)(ctx.cred->request_key_auth->name_link.next))->data[0]; + phys_addr += PAGE_SIZE; + --idx; + --nrpages; +@@ -199,7 +217,11 @@ void __init early_iounmap(void __iomem *addr, unsigned long size) + if (after_paging_init) + __late_clear_fixmap(idx); + else ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR); + #else - rka = ctx.cred->request_key_auth->payload.data[0]; + __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); + #endif - key = rka->dest_keyring; - __key_get(key); - } -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(ctx.cred->request_key_auth)); -+ #else - up_read(&ctx.cred->request_key_auth->sem); -+ #endif - if (!key) - goto error; - key_ref = make_key_ref(key, 1); -@@ -804,7 +851,11 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, - if (ret < 0) - goto invalid_key; - -+ #ifdef CONFIG_KEYP -+ iee_set_key_last_used_at(key, ktime_get_real_seconds()); -+ #else - key->last_used_at = ktime_get_real_seconds(); -+ #endif - - error: - put_cred(ctx.cred); -@@ -911,7 +962,11 @@ long join_session_keyring(const char *name) - void key_change_session_keyring(struct callback_head *twork) - { - const struct cred *old = current_cred(); -+ #ifdef CONFIG_CREDP -+ struct cred *new = *(struct cred **)(twork + 1); -+ #else - struct cred *new = container_of(twork, struct cred, rcu); -+ #endif - - if (unlikely(current->flags & PF_EXITING)) { - put_cred(new); -@@ -925,6 +980,38 @@ void key_change_session_keyring(struct callback_head *twork) - return; + --idx; + --nrpages; } - -+ /* If get_ucounts fails more bits are needed in the refcount */ -+ if (unlikely(!get_ucounts(old->ucounts))) { -+ WARN_ONCE(1, "In %s get_ucounts failed\n", __func__); -+ put_cred(new); -+ return; +diff --git a/mm/haoc/Makefile b/mm/haoc/Makefile +new file mode 100644 +index 000000000000..ad37f99f562d +--- /dev/null ++++ b/mm/haoc/Makefile +@@ -0,0 +1 @@ ++obj-$(CONFIG_PTP) += ptp-pg_cache.o +diff --git a/mm/haoc/ptp-pg_cache.c b/mm/haoc/ptp-pg_cache.c +new file mode 100644 +index 000000000000..3e0b10c98151 +--- /dev/null ++++ b/mm/haoc/ptp-pg_cache.c +@@ -0,0 +1,251 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++struct pg_cache pg_cache; ++ ++static inline void *__get_freepointer(void *object) ++{ ++ return *((void **)object); ++}; ++ ++static inline void __set_freepointer(void *object, void *next_object) ++{ ++ *(void **)object = next_object; ++}; ++ ++static inline void __iee_set_freepointer(void *object, void *next_object) ++{ ++ iee_set_freeptr(object, next_object); ++}; ++ ++static inline bool __update_freelist(struct pg_cache *cache, void *freelist_old, void *freelist_new, ++ unsigned long tid) ++{ ++ union freelist_aba_t old = { .freelist = freelist_old, .counter = tid }; ++ union freelist_aba_t new = { .freelist = freelist_new, .counter = tid + 1 }; ++ ++ return try_cmpxchg128(&(cache->freelist_tid.full), &old.full, new.full); ++} ++ ++#ifdef CONFIG_ARM64 ++static void __ptp_set_iee_pages(unsigned long start_addr, unsigned long end_addr, ++ struct pg_cache *cache) ++{ ++ unsigned long addr; ++ ++ if (start_addr != ALIGN(start_addr, PMD_SIZE)) ++ panic("IEE: %s pool not aligned.", cache->name); ++ ++ addr = start_addr; ++ while (addr < end_addr) { ++ set_iee_page(addr, PMD_ORDER, IEE_PGTABLE); ++ addr += PMD_SIZE; + } ++ flush_tlb_kernel_range(start_addr, end_addr); ++} ++#endif + -+ #ifdef CONFIG_CREDP -+ iee_set_cred_uid(new,old-> uid); -+ iee_set_cred_euid(new,old-> euid); -+ iee_set_cred_suid(new,old-> suid); -+ iee_set_cred_fsuid(new,old->fsuid); -+ iee_set_cred_gid(new,old-> gid); -+ iee_set_cred_egid(new,old-> egid); -+ iee_set_cred_sgid(new,old-> sgid); -+ iee_set_cred_fsgid(new,old->fsgid); -+ iee_set_cred_user(new,get_uid(old->user)); -+ iee_set_cred_ucounts(new, old->ucounts); -+ iee_set_cred_user_ns(new,get_user_ns(old->user_ns)); -+ iee_set_cred_group_info(new,get_group_info(old->group_info)); -+ -+ iee_set_cred_securebits(new,old->securebits); -+ iee_set_cred_cap_inheritable(new,old->cap_inheritable); -+ iee_set_cred_cap_permitted(new,old->cap_permitted); -+ iee_set_cred_cap_effective(new,old->cap_effective); -+ iee_set_cred_cap_ambient(new,old->cap_ambient); -+ iee_set_cred_cap_bset(new,old->cap_bset); -+ -+ iee_set_cred_jit_keyring(new,old->jit_keyring); -+ iee_set_cred_thread_keyring(new,key_get(old->thread_keyring)); -+ iee_set_cred_process_keyring(new,key_get(old->process_keyring)); -+ #else - new-> uid = old-> uid; - new-> euid = old-> euid; - new-> suid = old-> suid; -@@ -948,6 +1035,7 @@ void key_change_session_keyring(struct callback_head *twork) - new->jit_keyring = old->jit_keyring; - new->thread_keyring = key_get(old->thread_keyring); - new->process_keyring = key_get(old->process_keyring); -+ #endif - - security_transfer_creds(new, old); - -diff --git a/security/keys/request_key.c b/security/keys/request_key.c -index a7673ad86d18..d281459651af 100644 ---- a/security/keys/request_key.c -+++ b/security/keys/request_key.c -@@ -17,6 +17,10 @@ - #include "internal.h" - #include - -+#ifdef CONFIG_KEYP -+#include -+#endif -+ - #define key_negative_timeout 60 /* default timeout on a negative key's existence */ - - static struct key *check_cached_key(struct keyring_search_context *ctx) -@@ -285,13 +289,21 @@ static int construct_get_dest_keyring(struct key **_dest_keyring) - case KEY_REQKEY_DEFL_REQUESTOR_KEYRING: - if (cred->request_key_auth) { - authkey = cred->request_key_auth; -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(authkey)); -+ #else - down_read(&authkey->sem); -+ #endif - rka = get_request_key_auth(authkey); - if (!test_bit(KEY_FLAG_REVOKED, - &authkey->flags)) - dest_keyring = - key_get(rka->dest_keyring); -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(authkey)); -+ #else - up_read(&authkey->sem); -+ #endif - if (dest_keyring) { - do_perm_check = false; - break; -@@ -398,7 +410,11 @@ static int construct_alloc_key(struct keyring_search_context *ctx, - if (IS_ERR(key)) - goto alloc_failed; - -+ #ifdef CONFIG_KEYP -+ iee_set_key_flag_bit(key, KEY_FLAG_USER_CONSTRUCT, SET_BIT_OP); -+ #else - set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags); -+ #endif - - if (dest_keyring) { - ret = __key_link_lock(dest_keyring, &key->index_key); -diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c -index 8f33cd170e42..ddf86b3a2ecb 100644 ---- a/security/keys/request_key_auth.c -+++ b/security/keys/request_key_auth.c -@@ -145,7 +145,11 @@ static void request_key_auth_revoke(struct key *key) - */ - static void request_key_auth_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ struct request_key_auth *rka = rcu_access_pointer(((union key_payload *)(key->name_link.next))->rcu_data0); -+ #else - struct request_key_auth *rka = rcu_access_pointer(key->payload.rcu_data0); ++void __init ptp_pg_cache_init(struct pg_cache *cache, unsigned long object_order, ++ int levels, const char *name) ++{ ++ unsigned long addr; ++ unsigned long addr_next; ++ unsigned long reserve_order; ++ unsigned long reserve_pages; ++ unsigned long start_addr; ++ unsigned long end_addr; ++ unsigned long object_size; ++ struct page *page; ++ ++ object_size = (1 << object_order) * PAGE_SIZE; ++ reserve_order = CONFIG_PTP_RESERVE_ORDER; ++ while (1) { ++ reserve_pages = (1 << reserve_order) * levels; ++ start_addr = (unsigned long)memblock_alloc(reserve_pages * PAGE_SIZE, ++ reserve_pages * PAGE_SIZE); ++ if (start_addr) ++ break; ++ reserve_order--; ++ /* Allocate pages in pmd blocks to reduce the mapping cost. */ ++ if (reserve_order < PMD_ORDER) ++ panic("IEE: fail to reserve pages for %s", name); ++ } ++ end_addr = start_addr + reserve_pages * PAGE_SIZE; ++ pr_err("IEE: reserve %ld pages for %s, range[0x%lx, 0x%lx]", ++ reserve_pages, name, start_addr, end_addr); ++ ++ addr = start_addr; ++ addr_next = addr + object_size; ++ while (addr < end_addr) { ++ page = virt_to_page(addr); ++ set_page_count(page, 1); ++ __set_freepointer((void *)addr, (void *)addr_next); ++ addr += object_size; ++ addr_next += object_size; ++ } ++ __set_freepointer((void *)(addr - object_size), NULL); ++ ++ cache->object_order = object_order; ++ cache->reserve_order = reserve_order; ++ cache->reserve_start_addr = start_addr; ++ cache->reserve_end_addr = end_addr; ++ cache->levels = levels; ++ cache->freelist = (void *)start_addr; ++ cache->tid = 0; ++ cache->levels = levels; ++ cache->name = name; ++ ++#ifdef CONFIG_ARM64 ++ /* IEE for ARM64 needs to access these data by IEE addresses. */ ++ __ptp_set_iee_pages(start_addr, end_addr, cache); ++#endif ++#ifdef DEBUG ++ atomic_set(&cache->count, reserve_pages); ++ atomic_set(&cache->fail_count, 0); ++ pr_info("IEE: %s ready. object size 0x%lx, count %d.", ++ name, object_size, atomic_read(&cache->count)); ++#endif ++} ++ ++void __init ptp_set_iee_reserved(struct pg_cache *cache) ++{ ++ #ifdef CONFIG_X86_64 ++ unsigned long addr = cache->reserve_start_addr; ++ ++ for (int i = 0; i < cache->levels; i++) { ++ set_iee_page(addr,cache->reserve_order); ++ addr += (1 << cache->reserve_order) * PAGE_SIZE; ++ } + #endif - - kenter("{%d}", key->serial); - if (rka) { -@@ -184,22 +188,38 @@ struct key *request_key_auth_new(struct key *target, const char *op, - * another process */ - if (cred->request_key_auth) { - /* it is - use that instantiation context here too */ -+ #ifdef CONFIG_KEYP -+ down_read(&KEY_SEM(cred->request_key_auth)); -+ #else - down_read(&cred->request_key_auth->sem); -+ #endif - - /* if the auth key has been revoked, then the key we're - * servicing is already instantiated */ - if (test_bit(KEY_FLAG_REVOKED, - &cred->request_key_auth->flags)) { -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(cred->request_key_auth)); -+ #else - up_read(&cred->request_key_auth->sem); -+ #endif - ret = -EKEYREVOKED; - goto error_free_rka; - } - -+ #ifdef CONFIG_KEYP -+ irka = ((union key_payload *)(cred->request_key_auth->name_link.next))->data[0]; -+ #else - irka = cred->request_key_auth->payload.data[0]; ++} ++ ++#ifdef CONFIG_ARM64 ++/* Expand the cache pool with PMD_SIZE for each time. */ ++static int __ref ptp_pg_cache_expand(struct pg_cache *cache, gfp_t gfp) ++{ ++ unsigned long addr; ++ unsigned long addr_next; ++ unsigned long start_addr; ++ unsigned long end_addr; ++ unsigned long object_size; ++ unsigned long tid; ++ void **freelist; ++ struct page *page; ++ ++ if (slab_is_available()) ++ start_addr = __get_free_pages(gfp, PMD_ORDER); ++ else ++ start_addr = (unsigned long)memblock_alloc(PMD_SIZE, PMD_SIZE); ++ ++ if (!start_addr) ++ return 0; ++ ++ addr = start_addr; ++ object_size = (1 << cache->object_order) * PAGE_SIZE; ++ addr_next = addr + object_size; ++ end_addr = start_addr + PMD_SIZE; ++ while (addr < end_addr) { ++ page = virt_to_page(addr); ++ set_page_count(page, 1); ++ __set_freepointer((void *)addr, (void *)addr_next); ++ addr += object_size; ++ addr_next += object_size; ++ } ++ __set_freepointer((void *)(addr - object_size), NULL); ++ ++ /* IEE for ARM64 needs to access these data by IEE addresses. */ ++ __ptp_set_iee_pages(start_addr, end_addr, cache); ++ ++#ifdef DEBUG ++ atomic_add(1 << PMD_ORDER, &cache->count); ++ pr_info("IEE: %s expand to count %d. Curr failed: %d", cache->name, ++ atomic_read(&cache->count), atomic_read(&cache->fail_count)); ++#endif ++ ++/* Fill the new allocated pages into the cache freelist. */ ++redo: ++ tid = READ_ONCE(cache->tid); ++ barrier(); ++ freelist = READ_ONCE(cache->freelist); ++ __iee_set_freepointer((void *)(end_addr - object_size), freelist); ++ if (unlikely(!__update_freelist(cache, freelist, (void *)start_addr, tid))) ++ goto redo; ++ ++ return 1; ++} ++#endif ++ ++void *ptp_pg_alloc(struct pg_cache *cache, gfp_t gfp) ++{ ++ unsigned long tid; ++ void *object; ++ void *next_object; ++redo: ++ tid = READ_ONCE(cache->tid); ++ barrier(); ++ object = READ_ONCE(cache->freelist); ++ if (unlikely(!object)) { ++ // slow path alloc ++ #ifdef CONFIG_ARM64 ++ if (ptp_pg_cache_expand(cache, gfp) || READ_ONCE(cache->freelist)) ++ goto redo; ++ ++ /* If the expandsion failed, alloc a singel object without RO protection to ++ * avoid block spliting. ++ */ ++ object = (void *)__get_free_pages(gfp, cache->object_order); ++ set_iee_address_valid((unsigned long)object, cache->object_order); ++ iee_set_bitmap_type((unsigned long)object, cache->object_order, IEE_PGTABLE); ++ #ifdef DEBUG ++ WARN_ONCE(1, "IEE: Failed on %s expansion.", cache->name); ++ atomic_add(1 << cache->object_order, &cache->fail_count); + #endif - rka->cred = get_cred(irka->cred); - rka->pid = irka->pid; - -+ #ifdef CONFIG_KEYP -+ up_read(&KEY_SEM(cred->request_key_auth)); + #else - up_read(&cred->request_key_auth->sem); ++ object = (void *)__get_free_pages(gfp, cache->object_order); ++ set_iee_page((unsigned long)object, cache->object_order); + #endif - } - else { - /* it isn't - use this process as the context */ -diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c -index fee1ab2c734d..5ef67deb1df0 100644 ---- a/security/keys/trusted-keys/trusted_core.c -+++ b/security/keys/trusted-keys/trusted_core.c -@@ -233,7 +233,11 @@ static int trusted_update(struct key *key, struct key_preparsed_payload *prep) - - if (key_is_negative(key)) - return -ENOKEY; -+ #ifdef CONFIG_KEYP -+ p = ((union key_payload *)(key->name_link.next))->data[0]; -+ #else - p = key->payload.data[0]; -+ #endif - if (!p->migratable) - return -EPERM; - if (datalen <= 0 || datalen > 32767 || !prep->data) -@@ -307,7 +311,11 @@ static long trusted_read(const struct key *key, char *buffer, - */ - static void trusted_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ kfree_sensitive(((union key_payload *)(key->name_link.next))->data[0]); -+ #else - kfree_sensitive(key->payload.data[0]); ++ } else { ++ // fast path alloc ++ next_object = __get_freepointer(object); ++ if (unlikely(!__update_freelist(cache, object, next_object, tid))) ++ goto redo; ++ prefetchw(next_object); ++ if (gfp & __GFP_ZERO) ++ __iee_set_freepointer(object, NULL); ++ } ++ return object; ++}; ++ ++void ptp_pg_free(struct pg_cache *cache, void *object) ++{ ++ unsigned long tid; ++ void **freelist; ++ ++ #ifdef CONFIG_X86_64 ++ if (unlikely((unsigned long)object < cache->reserve_start_addr ++ || (unsigned long)object >= cache->reserve_end_addr)) { ++ // slow path free ++ unset_iee_page((unsigned long)object, cache->object_order); ++ free_pages((unsigned long)object, cache->object_order); ++ return; ++ } + #endif - } - - struct key_type key_type_trusted = { -diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c -index 749e2a4dcb13..1985cea69256 100644 ---- a/security/keys/user_defined.c -+++ b/security/keys/user_defined.c -@@ -14,6 +14,10 @@ - #include - #include "internal.h" - -+#ifdef CONFIG_KEYP -+#include -+#endif + - static int logon_vet_description(const char *desc); - - /* -@@ -109,7 +113,11 @@ int user_update(struct key *key, struct key_preparsed_payload *prep) - return ret; ++ // fast path free ++redo: ++ tid = READ_ONCE(cache->tid); ++ barrier(); ++ freelist = READ_ONCE(cache->freelist); ++ __iee_set_freepointer(object, freelist); ++ if (unlikely(!__update_freelist(cache, freelist, object, tid))) ++ goto redo; ++} +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 2bd84157869e..1c6cdd26dafa 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2848,7 +2848,12 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, + old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); - /* attach the new data, displacing the old */ -+ #ifdef CONFIG_KEYP -+ iee_set_key_expiry(key, prep->expiry); + pgtable = pgtable_trans_huge_withdraw(mm, pmd); ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) ++ | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else - key->expiry = prep->expiry; + pmd_populate(mm, &_pmd, pgtable); + #endif - if (key_is_positive(key)) - zap = dereference_key_locked(key); - rcu_assign_keypointer(key, prep->payload.data[0]); -@@ -145,7 +153,11 @@ EXPORT_SYMBOL(user_revoke); - */ - void user_destroy(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ struct user_key_payload *upayload = ((union key_payload *)(key->name_link.next))->data[0]; + + pte = pte_offset_map(&_pmd, haddr); + VM_BUG_ON(!pte); +@@ -3032,7 +3037,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, + * This's critical for some architectures (Power). + */ + pgtable = pgtable_trans_huge_withdraw(mm, pmd); ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ WRITE_ONCE(*(&_pmd), __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) ++ | (PMD_TYPE_TABLE | PMD_TABLE_PXN))); + #else - struct user_key_payload *upayload = key->payload.data[0]; + pmd_populate(mm, &_pmd, pgtable); + #endif - kfree_sensitive(upayload); - } -diff --git a/security/security.c b/security/security.c -index 839e12addac7..12320f9d4989 100644 ---- a/security/security.c -+++ b/security/security.c -@@ -30,6 +30,9 @@ - #include - #include - #include -+#ifdef CONFIG_CREDP -+#include + pte = pte_offset_map(&_pmd, haddr); + VM_BUG_ON(!pte); +diff --git a/mm/memory.c b/mm/memory.c +index d5932b924e08..911ce656a4c2 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -88,6 +88,9 @@ + #include + #include + #include ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include +#endif - /* How many LSMs were built into the kernel? */ - #define LSM_COUNT (__end_lsm_info - __start_lsm_info) -@@ -570,11 +573,19 @@ EXPORT_SYMBOL(unregister_blocking_lsm_notifier); - static int lsm_cred_alloc(struct cred *cred, gfp_t gfp) - { - if (blob_sizes.lbs_cred == 0) { -+ #ifdef CONFIG_CREDP -+ iee_set_cred_security(cred,NULL); + #include + #include "gmem-internal.h" +@@ -6133,6 +6136,9 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + /* If the fault handler drops the mmap_lock, vma may be freed */ + struct mm_struct *mm = vma->vm_mm; + vm_fault_t ret; ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; ++#endif + + __set_current_state(TASK_RUNNING); + +@@ -6159,7 +6165,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + if (unlikely(is_vm_hugetlb_page(vma))) + ret = hugetlb_fault(vma->vm_mm, vma, address, flags); + else ++ #if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ { ++ ptp_disable_iee(®); ++ ret = __handle_mm_fault(vma, address, flags); ++ ptp_enable_iee(reg); ++ } + #else - cred->security = NULL; + ret = __handle_mm_fault(vma, address, flags); + #endif - return 0; - } -+ #ifdef CONFIG_CREDP -+ iee_set_cred_security(cred,kzalloc(blob_sizes.lbs_cred, gfp)); -+ #else - cred->security = kzalloc(blob_sizes.lbs_cred, gfp); -+ #endif - if (cred->security == NULL) - return -ENOMEM; - return 0; -@@ -2965,7 +2976,11 @@ void security_cred_free(struct cred *cred) - call_void_hook(cred_free, cred); + lru_gen_exit_fault(); - kfree(cred->security); -+ #ifdef CONFIG_CREDP -+ iee_set_cred_security(cred,NULL); -+ #else - cred->security = NULL; -+ #endif +diff --git a/mm/mmap.c b/mm/mmap.c +index 6d9e6ab2bb27..64fad6588064 100644 +--- a/mm/mmap.c ++++ b/mm/mmap.c +@@ -55,6 +55,9 @@ + #include + #include + #include ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include ++#endif + + #define CREATE_TRACE_POINTS + #include +@@ -2468,10 +2471,17 @@ static void unmap_region(struct mm_struct *mm, struct ma_state *mas, + update_hiwater_rss(mm); + unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); + mas_set(mas, mt_start); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ tlb_flush_mmu(&tlb); ++#endif + free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, + next ? next->vm_start : USER_PGTABLES_CEILING, + mm_wr_locked); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_tlb_finish_mmu(&tlb); ++#else + tlb_finish_mmu(&tlb); ++#endif } - /** -diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c -index d4a99d98ec77..a08b94e13fd8 100644 ---- a/security/selinux/hooks.c -+++ b/security/selinux/hooks.c -@@ -104,9 +104,22 @@ - #include "audit.h" - #include "avc_ss.h" + /* +@@ -2606,6 +2616,9 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, + MA_STATE(mas_detach, &mt_detach, 0, 0); + mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); + mt_on_stack(mt_detach); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; ++#endif -+#ifdef CONFIG_IEE_SELINUX_P -+#include -+#include + /* + * If we need to split any vma, do it now to save pain later. +@@ -2717,8 +2730,14 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, + * were isolated before we downgraded mmap_lock. + */ + mas_set(&mas_detach, 1); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_disable_iee(®); ++#endif + unmap_region(mm, &mas_detach, vma, prev, next, start, end, count, + !unlock); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_enable_iee(reg); ++#endif + /* Statistics and freeing VMAs */ + mas_set(&mas_detach, 0); + remove_mt(mm, &mas_detach); +@@ -3426,6 +3445,9 @@ void exit_mmap(struct mm_struct *mm) + unsigned long nr_accounted = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); + int count = 0; ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; ++#endif + + /* mm's last user has gone, and its about to be pulled down */ + mmu_notifier_release(mm); +@@ -3445,7 +3467,13 @@ void exit_mmap(struct mm_struct *mm) + tlb_gather_mmu_fullmm(&tlb, mm); + /* update_hiwater_rss(mm) here? but nobody should be looking */ + /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_disable_iee(®); +#endif -+ -+#ifdef CONFIG_KEYP -+#include + unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_enable_iee(reg); +#endif -+ - #define SELINUX_INODE_INIT_XATTRS 1 + mmap_read_unlock(mm); -+#ifdef CONFIG_IEE_SELINUX_P -+struct selinux_state selinux_state __section(".iee.selinux"); + /* +@@ -3456,9 +3484,16 @@ void exit_mmap(struct mm_struct *mm) + mmap_write_lock(mm); + mt_clear_in_rcu(&mm->mm_mt); + mas_set(&mas, vma->vm_end); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ tlb_flush_mmu(&tlb); ++#endif + free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, + USER_PGTABLES_CEILING, true); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_tlb_finish_mmu(&tlb); +#else - struct selinux_state selinux_state; + tlb_finish_mmu(&tlb); +#endif - /* SECMARK reference count */ - static atomic_t selinux_secmark_refcount = ATOMIC_INIT(0); -@@ -6585,7 +6598,11 @@ static int selinux_key_alloc(struct key *k, const struct cred *cred, - else - ksec->sid = tsec->sid; - -+ #ifdef CONFIG_KEYP -+ iee_set_key_security(k, ksec); -+ #else - k->security = ksec; -+ #endif - return 0; - } - -@@ -6593,7 +6610,11 @@ static void selinux_key_free(struct key *k) - { - struct key_security_struct *ksec = k->security; - -+ #ifdef CONFIG_KEYP -+ iee_set_key_security(k, NULL); -+ #else - k->security = NULL; -+ #endif - kfree(ksec); + /* + * Walk the list again, actually closing and freeing it, with preemption +diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c +index 99b3e9408aa0..715e9f8fc5dc 100644 +--- a/mm/mmu_gather.c ++++ b/mm/mmu_gather.c +@@ -141,6 +141,55 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) + } } -@@ -7284,15 +7305,54 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { - #endif - }; - -+#ifdef CONFIG_IEE_SELINUX_P -+struct kmem_cache *policy_jar; -+ -+static void policy_cache_init(void) ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static void __ptp_tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) +{ -+ struct selinux_policy* unused; -+ policy_jar = kmem_cache_create("policy_jar", sizeof(struct selinux_policy), 0, -+ SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); -+ /* Test this cache */ -+ unused = kmem_cache_alloc(policy_jar, GFP_KERNEL); -+ kmem_cache_free(policy_jar, unused); ++ struct encoded_page **pages = batch->encoded_pages; ++ unsigned int nr, nr_pages; ++ ++ while (batch->nr) { ++ if (!page_poisoning_enabled_static() && !want_init_on_free()) { ++ nr = min(MAX_NR_FOLIOS_PER_FREE, batch->nr); ++ ++ /* ++ * Make sure we cover page + nr_pages, and don't leave ++ * nr_pages behind when capping the number of entries. ++ */ ++ if (unlikely(encoded_page_flags(pages[nr - 1]) & ++ ENCODED_PAGE_BIT_NR_PAGES_NEXT)) ++ nr++; ++ } else { ++ /* ++ * With page poisoning and init_on_free, the time it ++ * takes to free memory grows proportionally with the ++ * actual memory size. Therefore, limit based on the ++ * actual memory size and not the number of involved ++ * folios. ++ */ ++ for (nr = 0, nr_pages = 0; ++ nr < batch->nr && nr_pages < MAX_NR_FOLIOS_PER_FREE; ++ nr++) { ++ if (unlikely(encoded_page_flags(pages[nr]) & ++ ENCODED_PAGE_BIT_NR_PAGES_NEXT)) ++ nr_pages += encoded_nr_pages(pages[++nr]); ++ else ++ nr_pages++; ++ } ++ } ++ ++ for (int i = 0; i < nr; i++) { ++ struct page *page = encoded_page_ptr(pages[i]); ++ ++ __ptp_tlb_remove_table((void *)page); ++ } ++ pages += nr; ++ batch->nr -= nr; + -+ printk("IEE SELINUXP: policy cache created."); ++ cond_resched(); ++ } +} +#endif + - static __init int selinux_init(void) + static void tlb_batch_pages_flush(struct mmu_gather *tlb) { - pr_info("SELinux: Initializing.\n"); + struct mmu_gather_batch *batch; +@@ -150,6 +199,17 @@ static void tlb_batch_pages_flush(struct mmu_gather *tlb) + tlb->active = &tlb->local; + } - memset(&selinux_state, 0, sizeof(selinux_state)); -+#ifdef CONFIG_IEE_SELINUX_P -+ WRITE_ONCE(selinux_state.enforcing, selinux_enforcing_boot); -+#else - enforcing_set(selinux_enforcing_boot); -+#endif - selinux_avc_init(); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static void ptp_tlb_batch_pages_flush(struct mmu_gather *tlb) ++{ ++ struct mmu_gather_batch *batch; + -+#ifdef CONFIG_IEE_SELINUX_P -+ /* Put selinux_status inside IEE. */ -+ /* Prepare mutex lock and write the ptr to mutex->owner. */ -+ struct mutex* status_lock = kzalloc(GFP_KERNEL, sizeof(struct mutex)); -+ struct mutex* policy_mutex = kzalloc(GFP_KERNEL, sizeof(struct mutex)); -+ mutex_init(status_lock); -+ mutex_init(policy_mutex); -+ selinux_state.status_lock.owner.counter = (s64)status_lock; -+ selinux_state.policy_mutex.owner.counter = (s64)policy_mutex; -+ -+ /* Setting lm addr to be RO, IEE addr valid. */ -+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa_symbol(&selinux_state))); -+ iee_set_logical_mem_ro((unsigned long)&selinux_state); -+ iee_set_logical_mem_ro((unsigned long)__va(__pa_symbol(&selinux_state))); -+ printk("IEE SELINUXP: Succeeded on preparing selinux_state."); -+ policy_cache_init(); -+#else - mutex_init(&selinux_state.status_lock); - mutex_init(&selinux_state.policy_mutex); -+#endif - - /* Set the security state for the initial task. */ - cred_init_security(); -diff --git a/security/selinux/ima.c b/security/selinux/ima.c -index aa34da9b0aeb..022b3ce8d740 100644 ---- a/security/selinux/ima.c -+++ b/security/selinux/ima.c -@@ -12,6 +12,10 @@ - #include "security.h" - #include "ima.h" - -+#ifdef CONFIG_IEE_SELINUX_P -+#include ++ for (batch = &tlb->local; batch && batch->nr; batch = batch->next) ++ __ptp_tlb_batch_free_encoded_pages(batch); ++ tlb->active = &tlb->local; ++} +#endif + - /* - * selinux_ima_collect_state - Read selinux configuration settings - * -@@ -74,7 +78,11 @@ void selinux_ima_measure_state_locked(void) - size_t policy_len; - int rc = 0; - -+#ifdef CONFIG_IEE_SELINUX_P -+ lockdep_assert_held(iee_get_selinux_policy_lock()); -+#else - lockdep_assert_held(&selinux_state.policy_mutex); -+#endif - - state_str = selinux_ima_collect_state(); - if (!state_str) { -@@ -112,9 +120,21 @@ void selinux_ima_measure_state_locked(void) - */ - void selinux_ima_measure_state(void) + static void tlb_batch_list_free(struct mmu_gather *tlb) { -+#ifdef CONFIG_IEE_SELINUX_P -+ lockdep_assert_not_held(iee_get_selinux_policy_lock()); -+#else - lockdep_assert_not_held(&selinux_state.policy_mutex); -+#endif - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); -+#endif - selinux_ima_measure_state_locked(); -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); -+#endif - } -diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h -index a9de89af8fdc..2eee89071a56 100644 ---- a/security/selinux/include/security.h -+++ b/security/selinux/include/security.h -@@ -113,11 +113,15 @@ static inline bool selinux_initialized(void) - return smp_load_acquire(&selinux_state.initialized); + struct mmu_gather_batch *batch, *next; +@@ -347,6 +407,90 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) + tlb_table_flush(tlb); } -+#ifdef CONFIG_IEE_SELINUX_P -+extern void selinux_mark_initialized(void); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static void __ptp_tlb_remove_table_free(struct mmu_table_batch *batch) ++{ ++ int i; ++ ++ for (i = 0; i < batch->nr; i++) ++ __ptp_tlb_remove_table(batch->tables[i]); ++ ++ free_page((unsigned long)batch); ++} ++ ++#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE ++struct ptp_remove_table_work { ++ struct work_struct work; ++ struct mmu_table_batch *batch; ++}; ++ ++static void ptp_remove_table(struct work_struct *work) ++{ ++ struct ptp_remove_table_work *ptp_remove_table_work = ++ container_of(work, struct ptp_remove_table_work, work); ++ ++ __ptp_tlb_remove_table_free(ptp_remove_table_work->batch); ++ kfree(ptp_remove_table_work); ++} ++ ++static void ptp_tlb_remove_table_rcu(struct rcu_head *head) ++{ ++ struct ptp_remove_table_work *ptp_remove_table_work = ++ kmalloc(sizeof(struct ptp_remove_table_work), GFP_ATOMIC); ++ ++ ptp_remove_table_work->batch = container_of(head, struct mmu_table_batch, rcu); ++ INIT_WORK(&ptp_remove_table_work->work, ptp_remove_table); ++ schedule_work(&ptp_remove_table_work->work); ++} ++ ++static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) ++{ ++ call_rcu(&batch->rcu, ptp_tlb_remove_table_rcu); ++} +#else - static inline void selinux_mark_initialized(void) - { - /* do a synchronized write to avoid race conditions */ - smp_store_release(&selinux_state.initialized, true); - } ++static void ptp_tlb_remove_table_free(struct mmu_table_batch *batch) ++{ ++ __ptp_tlb_remove_table_free(batch); ++} +#endif - - #ifdef CONFIG_SECURITY_SELINUX_DEVELOP - static inline bool enforcing_enabled(void) -@@ -125,10 +129,14 @@ static inline bool enforcing_enabled(void) - return READ_ONCE(selinux_state.enforcing); - } - -+#ifdef CONFIG_IEE_SELINUX_P -+extern void enforcing_set(bool value); -+#else - static inline void enforcing_set(bool value) - { - WRITE_ONCE(selinux_state.enforcing, value); - } ++ ++static void ptp_tlb_remove_table_one(void *table) ++{ ++ tlb_remove_table_sync_one(); ++ __ptp_tlb_remove_table(table); ++} ++ ++static void ptp_tlb_table_flush(struct mmu_gather *tlb) ++{ ++ struct mmu_table_batch **batch = &tlb->batch; ++ ++ if (*batch) { ++ tlb_table_invalidate(tlb); ++ ptp_tlb_remove_table_free(*batch); ++ *batch = NULL; ++ } ++} ++ ++void ptp_tlb_remove_table(struct mmu_gather *tlb, void *table) ++{ ++ struct mmu_table_batch **batch = &tlb->batch; ++ ++ if (*batch == NULL) { ++ *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); ++ if (*batch == NULL) { ++ tlb_table_invalidate(tlb); ++ ptp_tlb_remove_table_one(table); ++ return; ++ } ++ (*batch)->nr = 0; ++ } ++ ++ (*batch)->tables[(*batch)->nr++] = table; ++ if ((*batch)->nr == MAX_TABLE_BATCH) ++ ptp_tlb_table_flush(tlb); ++} +#endif - #else - static inline bool enforcing_enabled(void) ++ + static inline void tlb_table_init(struct mmu_gather *tlb) { -diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c -index 54bc18e8164b..137cdc6ea27c 100644 ---- a/security/selinux/selinuxfs.c -+++ b/security/selinux/selinuxfs.c -@@ -43,6 +43,10 @@ - #include "conditional.h" - #include "ima.h" - -+#ifdef CONFIG_IEE_SELINUX_P -+#include -+#endif + tlb->batch = NULL; +@@ -469,3 +613,55 @@ void tlb_finish_mmu(struct mmu_gather *tlb) + #endif + dec_tlb_flush_pending(tlb->mm); + } + - enum sel_inos { - SEL_ROOT_INO = 2, - SEL_LOAD, /* load policy */ -@@ -371,7 +375,11 @@ static int sel_open_policy(struct inode *inode, struct file *filp) - - BUG_ON(filp->private_data); - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++static void ptp_tlb_flush_mmu_free(struct mmu_gather *tlb) ++{ ++ ptp_tlb_table_flush(tlb); ++#ifndef CONFIG_MMU_GATHER_NO_GATHER ++ ptp_tlb_batch_pages_flush(tlb); +#endif - - rc = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL); -@@ -401,11 +409,19 @@ static int sel_open_policy(struct inode *inode, struct file *filp) - - filp->private_data = plm; - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); ++} ++ ++void ptp_tlb_flush_mmu(struct mmu_gather *tlb) ++{ ++ tlb_flush_mmu_tlbonly(tlb); ++ ptp_tlb_flush_mmu_free(tlb); ++} ++ ++void ptp_tlb_finish_mmu(struct mmu_gather *tlb) ++{ ++ /* ++ * If there are parallel threads are doing PTE changes on same range ++ * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB ++ * flush by batching, one thread may end up seeing inconsistent PTEs ++ * and result in having stale TLB entries. So flush TLB forcefully ++ * if we detect parallel PTE batching threads. ++ * ++ * However, some syscalls, e.g. munmap(), may free page tables, this ++ * needs force flush everything in the given range. Otherwise this ++ * may result in having stale TLB entries for some architectures, ++ * e.g. aarch64, that could specify flush what level TLB. ++ */ ++ if (mm_tlb_flush_nested(tlb->mm)) { ++ /* ++ * The aarch64 yields better performance with fullmm by ++ * avoiding multiple CPUs spamming TLBI messages at the ++ * same time. ++ * ++ * On x86 non-fullmm doesn't yield significant difference ++ * against fullmm. ++ */ ++ tlb->fullmm = 1; ++ __tlb_reset_range(tlb); ++ tlb->freed_tables = 1; ++ } ++ ++ ptp_tlb_flush_mmu(tlb); ++ ++#ifndef CONFIG_MMU_GATHER_NO_GATHER ++ tlb_batch_list_free(tlb); +#endif - - return 0; - err: -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); ++ dec_tlb_flush_pending(tlb->mm); ++} +#endif - - if (plm) - vfree(plm->data); -@@ -594,7 +610,11 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, - if (!count) - return -EINVAL; - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); +diff --git a/mm/mprotect.c b/mm/mprotect.c +index 86f98c453fe4..e3a51f8b2d1a 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -36,6 +36,9 @@ + #include + #include + #include ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++#include +#endif - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__LOAD_POLICY, NULL); -@@ -632,13 +652,31 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, - audit_get_sessionid(current)); - - out: -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); + #include "internal.h" + #include "gmem-internal.h" +@@ -586,6 +589,9 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, + unsigned long charged = 0; + pgoff_t pgoff; + int error; ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ unsigned long reg; +#endif - vfree(data); - return length; - } -+// #ifdef CONFIG_IEE_SELINUX_P -+// /* Do sel_write_load twice to test free process of policy. */ -+// static ssize_t sel_write_load_test(struct file *file, const char __user *buf, -+// size_t count, loff_t *ppos) -+// { -+// sel_write_load(file, buf, count, ppos); -+// return sel_write_load(file, buf, count, ppos); -+// } -+// #endif -+ - static const struct file_operations sel_load_ops = { -+// #ifdef CONFIG_IEE_SELINUX_P -+// .write = sel_write_load_test, -+// #else - .write = sel_write_load, -+// #endif - .llseek = generic_file_llseek, - }; - -@@ -1216,7 +1254,11 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, - unsigned index = file_inode(filep)->i_ino & SEL_INO_MASK; - const char *name = filep->f_path.dentry->d_name.name; + if (newflags == oldflags) { + *pprev = vma; +@@ -666,7 +672,13 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, + mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; + vma_set_page_prot(vma); -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_disable_iee(®); +#endif - - ret = -EINVAL; - if (index >= fsi->bool_num || strcmp(name, -@@ -1235,14 +1277,22 @@ static ssize_t sel_read_bool(struct file *filep, char __user *buf, - } - length = scnprintf(page, PAGE_SIZE, "%d %d", cur_enforcing, - fsi->bool_pending_values[index]); -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); + change_protection(tlb, vma, start, end, mm_cp_flags); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ ptp_enable_iee(reg); +#endif - ret = simple_read_from_buffer(buf, count, ppos, page, length); - out_free: - free_page((unsigned long)page); - return ret; - out_unlock: -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); -+#endif - goto out_free; + /* + * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major +diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c +index 4fcd959dcc4d..ca7095034129 100644 +--- a/mm/pgtable-generic.c ++++ b/mm/pgtable-generic.c +@@ -232,6 +232,22 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, } + #endif -@@ -1267,7 +1317,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, - if (IS_ERR(page)) - return PTR_ERR(page); - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); ++#if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++struct ptp_pte_free_now_work { ++ struct work_struct work; ++ struct page *page; ++}; ++ ++static void ptp_pte_free_now(struct work_struct *work) ++{ ++ struct ptp_pte_free_now_work *ptp_work = ++ container_of(work, struct ptp_pte_free_now_work, work); ++ ++ pte_free(NULL, ptp_work->page); ++ kfree(ptp_work); ++} +#endif ++ + /* arch define pte_free_defer in asm/pgalloc.h for its own implementation */ + #ifndef pte_free_defer + static void pte_free_now(struct rcu_head *head) +@@ -239,7 +255,16 @@ static void pte_free_now(struct rcu_head *head) + struct page *page; - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__SETBOOL, -@@ -1291,7 +1345,11 @@ static ssize_t sel_write_bool(struct file *filep, const char __user *buf, - length = count; - - out: -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); -+#endif - kfree(page); - return length; + page = container_of(head, struct page, rcu_head); ++ #if defined(CONFIG_PTP) && defined(CONFIG_X86_64) ++ struct ptp_pte_free_now_work *ptp_work = ++ kmalloc(sizeof(struct ptp_pte_free_now_work), GFP_ATOMIC); ++ ++ ptp_work->page = page; ++ INIT_WORK(&ptp_work->work, ptp_pte_free_now); ++ schedule_work(&ptp_work->work); ++ #else + pte_free(NULL /* mm not passed and not used */, (pgtable_t)page); ++ #endif } -@@ -1322,7 +1380,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, - if (IS_ERR(page)) - return PTR_ERR(page); - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_policy_lock()); -+#else - mutex_lock(&selinux_state.policy_mutex); -+#endif - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__SETBOOL, -@@ -1343,7 +1405,11 @@ static ssize_t sel_commit_bools_write(struct file *filep, - length = count; + void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) +diff --git a/mm/slub.c b/mm/slub.c +index 25516a5cbf04..5e88d574c20b 100644 +--- a/mm/slub.c ++++ b/mm/slub.c +@@ -44,6 +44,12 @@ - out: -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_policy_lock()); -+#else - mutex_unlock(&selinux_state.policy_mutex); + #include + #include ++#ifdef CONFIG_IEE ++#include +#endif - kfree(page); - return length; - } -diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c -index 379ac7b5c709..6d5cb766116a 100644 ---- a/security/selinux/ss/services.c -+++ b/security/selinux/ss/services.c -@@ -68,6 +68,10 @@ - #include "policycap_names.h" - #include "ima.h" - -+#ifdef CONFIG_IEE_SELINUX_P -+#include ++#ifdef CONFIG_IEE_PTRP ++#include +#endif -+ - struct selinux_policy_convert_data { - struct convert_context_args args; - struct sidtab_convert_params sidtab_params; -@@ -2108,9 +2112,14 @@ static void security_load_policycaps(struct selinux_policy *policy) - p = &policy->policydb; + #include "internal.h" -+#ifdef CONFIG_IEE_SELINUX_P -+ for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) -+ iee_set_sel_policy_cap(i, ebitmap_get_bit(&p->policycaps, i)); -+#else - for (i = 0; i < ARRAY_SIZE(selinux_state.policycap); i++) - WRITE_ONCE(selinux_state.policycap[i], - ebitmap_get_bit(&p->policycaps, i)); -+#endif +@@ -181,6 +187,15 @@ + * the fast path and disables lockless freelists. + */ - for (i = 0; i < ARRAY_SIZE(selinux_policycap_names); i++) - pr_info("SELinux: policy capability %s=%d\n", -@@ -2173,6 +2182,9 @@ void selinux_policy_commit(struct selinux_load_state *load_state) - { - struct selinux_state *state = &selinux_state; - struct selinux_policy *oldpolicy, *newpolicy = load_state->policy; -+#ifdef CONFIG_IEE_SELINUX_P -+ struct selinux_policy *temppolicy; ++#ifdef CONFIG_IEE ++void __weak iee_allocate_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order) {} ++bool __weak iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order) ++{ ++ return false; ++} ++unsigned int __weak iee_calculate_order(struct kmem_cache *s, unsigned int order) { return order; } ++void __weak iee_set_min_partial(struct kmem_cache *s) { } +#endif - unsigned long flags; - u32 seqno; + /* + * We could simply use migrate_disable()/enable() but as long as it's a + * function call even on !PREEMPT_RT, use inline preempt_disable() there. +@@ -2041,6 +2056,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; -@@ -2197,10 +2209,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) - /* Install the new policy. */ - if (oldpolicy) { - sidtab_freeze_begin(oldpolicy->sidtab, &flags); -+#ifdef CONFIG_IEE_SELINUX_P -+ iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); -+ kfree(newpolicy); -+#else - rcu_assign_pointer(state->policy, newpolicy); + slab = alloc_slab_page(alloc_gfp, node, oo); ++#ifdef CONFIG_IEE_PTRP ++ slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); +#endif - sidtab_freeze_end(oldpolicy->sidtab, &flags); - } else { -+#ifdef CONFIG_IEE_SELINUX_P -+ iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); -+ kfree(newpolicy); -+#else - rcu_assign_pointer(state->policy, newpolicy); + if (unlikely(!slab)) { + oo = s->min; + alloc_gfp = flags; +@@ -2049,6 +2067,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + * Try a lower order alloc if possible + */ + slab = alloc_slab_page(alloc_gfp, node, oo); ++#ifdef CONFIG_IEE_PTRP ++ slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); +#endif - } + if (unlikely(!slab)) + return NULL; + stat(s, ORDER_FALLBACK); +@@ -2058,6 +2079,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + slab->inuse = 0; + slab->frozen = 0; - /* Load the policycaps from the new policy */ -@@ -2218,7 +2240,20 @@ void selinux_policy_commit(struct selinux_load_state *load_state) ++#ifdef CONFIG_IEE ++ iee_allocate_slab_data(s, slab, oo_order(oo)); ++#endif + account_slab(slab, oo_order(oo), s, flags); - /* Free the old policy */ - synchronize_rcu(); -+#ifdef CONFIG_IEE_SELINUX_P -+ /* -+ * Normal free process includes setting freed objects pointers to be NULL, however it -+ * would be hard as old policy is already inside IEE. So Make a kernel copy of the old -+ * policy to free objects it points to. -+ */ -+ if (oldpolicy){ -+ temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); -+ selinux_policy_free(temppolicy); -+ kfree(oldpolicy); -+ } -+#else - selinux_policy_free(oldpolicy); + slab->slab_cache = s; +@@ -2110,6 +2134,10 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) + __folio_clear_slab(folio); + mm_account_reclaimed_pages(pages); + unaccount_slab(slab, order, s); ++#ifdef CONFIG_IEE ++ if (iee_free_slab_data(s, slab, order)) ++ return; +#endif - kfree(load_state->convert_data); + __free_pages(&folio->page, order); + } - /* Notify others of the policy change */ -@@ -3016,6 +3051,9 @@ int security_set_bools(u32 len, int *values) - { - struct selinux_state *state = &selinux_state; - struct selinux_policy *newpolicy, *oldpolicy; -+#ifdef CONFIG_IEE_SELINUX_P -+ struct selinux_policy *temppolicy; +@@ -4491,6 +4519,9 @@ static int calculate_sizes(struct kmem_cache *s) + s->size = size; + s->reciprocal_size = reciprocal_value(size); + order = calculate_order(size); ++#ifdef CONFIG_IEE ++ order = iee_calculate_order(s, order); +#endif - int rc; - u32 i, seqno = 0; -@@ -3068,8 +3106,13 @@ int security_set_bools(u32 len, int *values) - newpolicy->latest_granting = oldpolicy->latest_granting + 1; - seqno = newpolicy->latest_granting; + if ((int)order < 0) + return 0; +@@ -4553,6 +4584,9 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) + s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); + s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); -+#ifdef CONFIG_IEE_SELINUX_P -+ iee_sel_rcu_assign_policy(newpolicy, kmem_cache_alloc(policy_jar, GFP_KERNEL)); -+ kfree(newpolicy); -+#else - /* Install the new policy */ - rcu_assign_pointer(state->policy, newpolicy); -+#endif ++ #ifdef CONFIG_IEE ++ iee_set_min_partial(s); ++ #endif + set_cpu_partial(s); - /* - * Free the conditional portions of the old policydb -@@ -3077,7 +3120,20 @@ int security_set_bools(u32 len, int *values) - * structure itself but not what it references. - */ - synchronize_rcu(); -+#ifdef CONFIG_IEE_SELINUX_P -+ /* -+ * Normal free process includes setting freed objects pointers to be NULL, however it -+ * would be hard as old policy is already inside IEE. So Make a kernel copy of the old -+ * policy to free objects it points to. -+ */ -+ temppolicy = kmemdup(oldpolicy, sizeof(*temppolicy), GFP_KERNEL); -+ if (!temppolicy) -+ return -ENOMEM; -+ selinux_policy_cond_free(temppolicy); -+ kfree(oldpolicy); -+#else - selinux_policy_cond_free(oldpolicy); -+#endif + #ifdef CONFIG_NUMA +diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c +index c33dc69a26df..fa0e7b853729 100644 +--- a/mm/sparse-vmemmap.c ++++ b/mm/sparse-vmemmap.c +@@ -30,9 +30,13 @@ + #include - /* Notify others of the policy change */ - selinux_notify_policy_change(seqno); -diff --git a/security/selinux/status.c b/security/selinux/status.c -index dffca22ce6f7..642965684325 100644 ---- a/security/selinux/status.c -+++ b/security/selinux/status.c -@@ -13,6 +13,10 @@ - #include "avc.h" - #include "security.h" + #include +- ++#include + #include "internal.h" -+#ifdef CONFIG_IEE_SELINUX_P -+#include ++#ifdef CONFIG_PTP ++extern void *__ptp_vmemmap_alloc_block(unsigned long size, int node); +#endif + /* - * The selinux_status_page shall be exposed to userspace applications - * using mmap interface on /selinux/status. -@@ -44,9 +48,17 @@ struct page *selinux_kernel_status_page(void) - struct selinux_kernel_status *status; - struct page *result = NULL; + * Allocate a block of memory to be used to back the virtual memory map + * or to back the page tables that are used to create the mapping. +@@ -151,6 +155,12 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + pte_t entry; + void *p; -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_status_lock()); -+#else - mutex_lock(&selinux_state.status_lock); -+#endif - if (!selinux_state.status_page) { -+ #ifdef CONFIG_IEE_SELINUX_P -+ iee_set_selinux_status_pg(alloc_page(GFP_KERNEL|__GFP_ZERO)); ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ WARN_ONCE(reuse, "PTP: reuse in vmemmap."); ++ p = __ptp_vmemmap_alloc_block(PAGE_SIZE, node); ++ if (!p) ++ return NULL; + #else - selinux_state.status_page = alloc_page(GFP_KERNEL|__GFP_ZERO); -+ #endif - - if (selinux_state.status_page) { - status = page_address(selinux_state.status_page); -@@ -66,7 +78,11 @@ struct page *selinux_kernel_status_page(void) + if (!reuse) { + p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); + if (!p) +@@ -168,6 +178,7 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + get_page(reuse); + p = page_to_virt(reuse); } ++ #endif + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); } - result = selinux_state.status_page; -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_status_lock()); -+#else - mutex_unlock(&selinux_state.status_lock); -+#endif - - return result; - } -@@ -80,7 +96,11 @@ void selinux_status_update_setenforce(bool enforcing) - { - struct selinux_kernel_status *status; - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_status_lock()); -+#else - mutex_lock(&selinux_state.status_lock); -+#endif - if (selinux_state.status_page) { - status = page_address(selinux_state.status_page); - -@@ -92,7 +112,11 @@ void selinux_status_update_setenforce(bool enforcing) - smp_wmb(); - status->sequence++; - } -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_status_lock()); -+#else - mutex_unlock(&selinux_state.status_lock); -+#endif - } +@@ -176,11 +187,18 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, - /* -@@ -105,7 +129,11 @@ void selinux_status_update_policyload(u32 seqno) + static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) { - struct selinux_kernel_status *status; - -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_lock(iee_get_selinux_status_lock()); -+#else - mutex_lock(&selinux_state.status_lock); -+#endif - if (selinux_state.status_page) { - status = page_address(selinux_state.status_page); - -@@ -118,5 +146,9 @@ void selinux_status_update_policyload(u32 seqno) - smp_wmb(); - status->sequence++; - } -+#ifdef CONFIG_IEE_SELINUX_P -+ mutex_unlock(iee_get_selinux_status_lock()); -+#else - mutex_unlock(&selinux_state.status_lock); -+#endif - } -diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c -index 4625674f0e95..cbd843db9d41 100644 ---- a/security/smack/smack_lsm.c -+++ b/security/smack/smack_lsm.c -@@ -45,6 +45,10 @@ - #include - #include "smack.h" - -+#ifdef CONFIG_KEYP -+#include -+#endif ++ #if defined(CONFIG_PTP) && defined(CONFIG_ARM64) ++ void *p = __ptp_vmemmap_alloc_block(size, node); + - #define TRANS_TRUE "TRUE" - #define TRANS_TRUE_SIZE 4 - -@@ -4411,7 +4415,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, - { - struct smack_known *skp = smk_of_task(smack_cred(cred)); - -+ #ifdef CONFIG_KEYP -+ iee_set_key_security(key, skp); ++ if (!p) ++ return NULL; + #else - key->security = skp; -+ #endif - return 0; - } + void *p = vmemmap_alloc_block(size, node); -@@ -4423,7 +4431,11 @@ static int smack_key_alloc(struct key *key, const struct cred *cred, - */ - static void smack_key_free(struct key *key) - { -+ #ifdef CONFIG_KEYP -+ iee_set_key_security(key, NULL); -+ #else - key->security = NULL; + if (!p) + return NULL; + memset(p, 0, size); + #endif - } - /** + return p; + } -- -2.34.1 +2.49.0 diff --git a/haoc-kernel.spec b/haoc-kernel.spec index 5bb13e43891fd7e1945e41716d279881abd2cafb..d00ddf87bee1d767bf5ce3a64bc8629d4a79e4ed 100644 --- a/haoc-kernel.spec +++ b/haoc-kernel.spec @@ -40,9 +40,9 @@ rm -f test_openEuler_sign.ko test_openEuler_sign.ko.sig %global upstream_version 6.6 %global upstream_sublevel 0 -%global devel_release 69 +%global devel_release 126 %global maintenance_release .0.0 -%global pkg_release .76 +%global pkg_release .104 %global openeuler_lts 1 %global openeuler_major 2403 @@ -78,7 +78,7 @@ rm -f test_openEuler_sign.ko test_openEuler_sign.ko.sig %ifarch aarch64 %define with_64kb %{?_with_64kb: 1} %{?!_with_64kb: 0} %if %{with_64kb} -%global package64kb -64kb +%define with_kabichk 0 %endif %else %define with_64kb 0 @@ -87,7 +87,7 @@ rm -f test_openEuler_sign.ko test_openEuler_sign.ko.sig #default is enabled. You can disable it with --without option %define with_perf %{?_without_perf: 0} %{?!_without_perf: 1} -Name: haoc-kernel +Name: kernel Version: %{upstream_version}.%{upstream_sublevel} Release: %{devel_release}%{?maintenance_release}%{?pkg_release} Summary: Linux Kernel @@ -114,6 +114,8 @@ Source25: Module.kabi_ext2_x86_64 %endif Source200: mkgrub-menu-aarch64.sh +Source300: extra-modules_x86_64.list +Source301: extra-modules_aarch64.list Source2000: cpupower.service Source2001: cpupower.config @@ -238,6 +240,15 @@ Obsoletes: %{name}-tools-libs-devel This package contains the development files for the tools/ directory from the kernel source. +%ifarch x86_64 aarch64 +%package extra-modules +Summary: Extra kernel modules to match the kernel +AutoReqProv: no +Provides: kernel-extra-modules = %{version}-%{release} +%description extra-modules +This package contains optional modules that may be dynamically loaded but not needed for base system operation. +%endif + %if %{with_perf} %package -n perf Summary: Performance monitoring for the Linux kernel @@ -360,6 +371,7 @@ Applypatches() Applypatches series.conf %{_builddir}/kernel-%{version}/linux-%{KernelVer} %endif +# apply HAOC %patch0005 -p1 # riscv-kernel patch @@ -391,10 +403,6 @@ cp -a tools/perf tools/python3-perf %build cd linux-%{KernelVer} -# Workaround for flask.h and av_permissions.h -gcc scripts/selinux/genheaders/genheaders.c -o scripts/selinux/genheaders/genheaders -Iinclude/uapi -Iinclude -Isecurity/selinux/include -scripts/selinux/genheaders/genheaders security/selinux/flask.h security/selinux/av_permissions.h - perl -p -i -e "s/^EXTRAVERSION.*/EXTRAVERSION = -%{release}.%{_target_cpu}/" Makefile perl -p -i -e "s/^OPENEULER_LTS.*/OPENEULER_LTS = %{openeuler_lts}/" Makefile.oever perl -p -i -e "s/^OPENEULER_MAJOR.*/OPENEULER_MAJOR = %{openeuler_major}/" Makefile.oever @@ -406,10 +414,6 @@ make mrproper %{_smp_mflags} %if %{with_64kb} sed -i arch/arm64/configs/openeuler_defconfig -e 's/^CONFIG_ARM64_4K_PAGES.*/CONFIG_ARM64_64K_PAGES=y/' -sed -i arch/arm64/configs/openeuler_defconfig -e 's/^CONFIG_ARM64_PA_BITS=.*/CONFIG_ARM64_PA_BITS=52/' -sed -i arch/arm64/configs/openeuler_defconfig -e 's/^CONFIG_ARM64_PA_BITS_.*/CONFIG_ARM64_PA_BITS_52=y/' -sed -i arch/arm64/configs/openeuler_defconfig -e 's/^CONFIG_ARM64_VA_BITS=.*/CONFIG_ARM64_VA_BITS=52/' -sed -i arch/arm64/configs/openeuler_defconfig -e 's/^CONFIG_ARM64_VA_BITS_.*/CONFIG_ARM64_VA_BITS_52=y/' %endif %if "%toolchain" == "clang" @@ -472,6 +476,8 @@ TargetImage=$(basename $(make -s image_name)) chmod 0755 %{SOURCE18} if [ -e $RPM_SOURCE_DIR/Module.kabi_%{_target_cpu} ]; then %{SOURCE18} -k $RPM_SOURCE_DIR/Module.kabi_%{_target_cpu} -s Module.symvers || exit 1 + %{SOURCE18} -k $RPM_SOURCE_DIR/Module.kabi_ext1_%{_target_cpu} -s Module.symvers || exit 1 + %{SOURCE18} -k $RPM_SOURCE_DIR/Module.kabi_ext2_%{_target_cpu} -s Module.symvers || exit 1 else echo "**** NOTE: Cannot find reference Module.kabi file. ****" fi @@ -634,8 +640,31 @@ install -m 755 %{SOURCE200} $RPM_BUILD_ROOT%{_sbindir}/mkgrub-menu-%{version}-%{ # deal with module, if not kdump %{make} ARCH=%{Arch} INSTALL_MOD_PATH=$RPM_BUILD_ROOT modules_install KERNELRELEASE=%{KernelVer} mod-fw= ######## to collect ko to module.filelist about netwoking. block. drm. modesetting ############### +# 1. Generates extra-module.list by: +# - Processing predefined extra-modules_%{_target_cpu}.list +# 2. Creates core-module.list by comparing installed modules with extra modules +# Final output: Generates kernel-modules-filelist (core) and kernel-extra-modules-filelist +# for RPM package specification, while ensuring proper path mapping to /lib/modules/%{KernelVer}/ + pushd $RPM_BUILD_ROOT/lib/modules/%{KernelVer} +%ifnarch x86_64 aarch64 find -type f -name "*.ko" >modnames +%else +sed 's!^!kernel/!; s!\.ko$!!' %{_sourcedir}/extra-modules_%{_target_cpu}.list > modules-extra.list + +find -type f -name "*.ko" | sort >modnames + +grep -vFf modules-extra.list modnames > modules-core.list || true + +sed -e 's!^\.\/kernel/!/lib/modules/%{KernelVer}/kernel/!; s!\.ko$!.ko.xz!' \ + modules-core.list > %{_builddir}/%{name}-%{version}/kernel-modules-filelist + +if [ -s modules-extra.list ]; then + sed 's!^kernel/!!; s!$!.ko.xz!; s!^!/lib/modules/%{KernelVer}/kernel/!' modules-extra.list > %{_builddir}/%{name}-%{version}/kernel-extra-modules-filelist +else + echo "%ghost /nonexistent/dummy/file" > %{_builddir}/%{name}-%{version}/kernel-extra-modules-filelist +fi +%endif # mark modules executable so that strip-to-file can strip them xargs --no-run-if-empty chmod u+x < modnames @@ -676,6 +705,9 @@ grep -E -v \ modinfo && exit 1 rm -f modinfo modnames drivers.undef +%ifarch x86_64 aarch64 +rm -f modules-extra.list modules-core.list modules.list +%endif for i in alias alias.bin builtin.bin ccwmap dep dep.bin ieee1394map inputmap isapnpmap ofmap pcimap seriomap symbols symbols.bin usbmap do @@ -979,7 +1011,11 @@ fi /sbin/ldconfig %systemd_postun cpupower.service +%ifnarch x86_64 aarch64 %files +%else +%files -f kernel-modules-filelist +%endif %defattr (-, root, root) %doc /boot/config-* @@ -992,11 +1028,21 @@ fi %ghost /boot/initramfs-%{KernelVer}.img /boot/.vmlinuz-*.hmac /etc/ld.so.conf.d/* +%ifnarch x86_64 aarch64 /lib/modules/%{KernelVer}/ +%else +/lib/modules/%{KernelVer}/vdso/ +/lib/modules/%{KernelVer}/modules.* +%endif %exclude /lib/modules/%{KernelVer}/source %exclude /lib/modules/%{KernelVer}/build %{_sbindir}/mkgrub-menu*.sh +%ifarch x86_64 aarch64 +%files extra-modules -f kernel-extra-modules-filelist +%defattr(-,root,root) +%endif + %files devel %defattr (-, root, root) %doc @@ -1096,8 +1142,9848 @@ fi %endif %changelog -* Fri Dec 20 2024 Liu Zhehui - 6.6.0-69.0.0.76 -- update HAOC to 6.6.0-69.0.0 +* Mon Dec 08 2025 Liu Zhehui - 6.6.0-126.0.0.104 +- update HAOC to 6.6.0-126.0.0 + +* Thu Dec 04 2025 Mingzheng Xing - 6.6.0-126.0.0.103 +- RISC-V kernel upgrade to 6.6.0-126.0.0 +- Add support for LRW, DP1000 +- Add IOMMU related support +- Backport mainline RISC-V features. + +* Thu Dec 04 2025 Li Nan - 6.6.0-126.0.0.102 +- !19468 urma: Bugfix for ubcore_get_route_list implement +- urma: Bugfix for ubcore_get_route_list implement +- !19329 hisi_acc_vfio_pci: adapt to new migration configuration +- !19435 crypto: hisilicon/qm - fix incorrect judgment in qm_get_complete_eqe_num() +- !19383 [OLK-6.6] backport important bugfix for Broadcom NIC driver from upstream linux-6.6.y +- !19460 xsched: XCU Partition +- !19316 Fix cves CVE-2025-40040 & CVE-2025-40058 +- !19447 net: hns3: remove VF multi-TC compat bit +- xsched: add throttling statistics for xsched groups +- xsched: fix task stall when quota is disabled +- xsched: suppress excessive error logs during cgroup file retry +- xsched: fix shares misaccounting when multiple tasks run in one cgroup +- xsched: switch xcu cgroup subsystem from domain mode to thread mode +- !19439 obmm: remove redundant code and fix log issues +- !18481 [OLK-6.6] Add platform-specific quirks for ZX-200 device +- !19085 scsi: mpi3mr: Add support for status reply descriptor +- !19078 scsi: mpi3mr: Split off bus_reset function from host_reset +- !19416 kernfs: Fix UAF in polling when open file is released +- !19415 smb: client: fix race with concurrent opens in rename(2) +- !19446 net/mlx5: Fix ECVF vports unload on shutdown flow +- obmm: remove redundant code and fix log issues +- !19441 rv: Use strings in da monitors tracepoints +- !19411 CVE-2025-39697 +- !19442 CVE-2025-22022 +- net: hns3: remove VF multi-TC compat bit +- net/mlx5: Fix ECVF vports unload on shutdown flow +- usb: xhci: Apply the link chain quirk on NEC isoc endpoints +- usb: xhci: move link chain bit quirk checks into one helper function. +- !19429 net/sched: Abort __tc_modify_qdisc if parent class does not exist +- rv: Use strings in da monitors tracepoints +- !19438 drm/scheduler: signal scheduled fence when kill job +- !19410 Fix CVE-2025-39994 +- !17888 [OLK-6.6] Fix CVE-2025-37777 +- drm/scheduler: signal scheduled fence when kill job +- !19371 RDMA/rxe: Fix race in do_task() when draining +- !19393 usb: typec: ucsi: displayport: Fix NULL pointer access +- obmm: set up default mem allocator granule +- !19377 [OLK-6.6] Fix CVE-2025-23133 +- !18704 [OLK-6.6] Fix CVE-2024-58095 +- !19401 smb: client: fix crypto buffers in non-linear memory +- !19323 nfsd: don't ignore the return code of svc_proc_register() +- crypto: hisilicon/qm - fix incorrect judgment in qm_get_complete_eqe_num() +- crypto: hisilicon/qm - add missing default in switch in qm_vft_data_cfg +- crypto: hisilicon/sgl - remove unnecessary checks for curr_hw_sgl error +- crypto: hisilicon/qm - add concurrency protection for variable err_threshold +- crypto: hisilicon/qm - add the save operation of eqe and aeqe +- crypto: hisilicon/qm - restore original qos values +- crypto: hisilicon/qm - Fix device reference leak in qm_get_qos_value +- !19386 x86/mm: Check return value from memblock_phys_alloc_range() +- !19433 urma: bugfix ubagg drivers refent error +- urma: bugfix ubagg drivers refent error +- !19391 urma: fixing ubcore hash table initialization failure. +- !19263 [OLK-6.6]Fix CVE-2025-37861 +- !19253 [OLK-6.6] Fix CVE-2025-37973 +- !19252 [OLK-6.6] Fix CVE-2025-37972 +- !19251 [OLK-6.6] Fix CVE-2025-37956 +- !19374 [OLK-6.6] Fix CVE-2025-37899 +- net/sched: Abort __tc_modify_qdisc if parent class does not exist +- kernfs: Fix UAF in polling when open file is released +- !19402 sched: Fix incorrect cluster mask +- smb: client: fix race with concurrent opens in rename(2) +- !19414 xsched: fix compile error when CONFIG_XCU_SCHEDULER is disabled +- !19398 v2 bugfix for xcall2.0 +- !19412 sunrpc: fix handling of server side tls alerts +- !19409 smc: Use __sk_dst_get() and dst_dev_rcu() in in smc_clc_prfx_set(). +- !19405 [OLK-6.6]:update patches for sw64 architecture +- xsched: fix compile error when CONFIG_XCU_SCHEDULER is disabled +- !19266 __legitimize_mnt(): check for MNT_SYNC_UMOUNT should be under mount_lock +- !19358 Fix CVE-2025-40016 +- sunrpc: fix handling of server side tls alerts +- NFS: Fix a race when updating an existing write +- nfs: fold nfs_page_group_lock_subrequests into nfs_lock_and_join_requests +- media: tuner: xc5000: Fix use-after-free in xc5000_release +- media: tunner: xc5000: Refactor firmware load +- urma: fixing ubcore hash table initialization failure. +- smc: Use __sk_dst_get() and dst_dev_rcu() in in smc_clc_prfx_set(). +- !19365 net/mlx5e: Harden uplink netdev access against device unbind +- sw64: config: add initial openeuler_defconfig for sw64 +- sw64: fix compile error of pte_pgprot() +- sw64: add missing definition of MAP_PEER_SHARED +- sched: Fix incorrect cluster mask +- smb: client: fix crypto buffers in non-linear memory +- !19392 Fixes and cleanup of page cache limit +- !18760 CVE-2025-38501 +- !19390 v3 xsched: add xsched_{set,get}attr syscall +- xcall2.0: fix panic while detaching xcall found in DT case +- xcall2.0: prefetch: fix access NULL ptr in __do_sys_epoll_pwait() +- xcall2.0: prefetch: get prefetch_mm_data once in get_pfi() +- !19311 nbd: fix uaf in nbd_genl_connect() error path +- !19310 fs: writeback: fix use-after-free in __mark_inode_dirty() +- !19256 efivarfs: Fix slab-out-of-bounds in efivarfs_d_compare +- !19237 nfsd: provide locking for v4_end_grace +- !19247 dmaengine: ti: k3-udma: Add missing locking +- !19220 NFSv4: Fix deadlock during the running of state manager +- !19248 nfsd: Initialize ssc before laundromat_work to prevent NULL dereference +- usb: typec: ucsi: displayport: Fix NULL pointer access +- !19378 net: hibmcge: add support for pagepool on rx +- mm: Add page cache limit check before queueing shrink worker +- mm: Move vm_cache_limit_mbytes check to page_cache_over_limit() +- mm: Replace deferrable timer with delay timer for shrink worker +- !19388 [sync] PR-19363: Revert "scsi: libsas: Fix exp-attached device scan after probe failure scanned in again after probe failed" +- !19368 net/sched: Fix backlog accounting in qdisc_dequeue_internal +- !19367 net/sched: sch_qfq: Fix CVE-2025-38477 +- !19366 i40e: fix IRQ freeing in i40e_vsi_request_irq_msix error path +- !19364 ixgbevf: Fix CVE-2025-40104 +- xsched: add xsched_{set,get}attr syscall +- Revert "scsi: libsas: Fix exp-attached device scan after probe failure scanned in again after probe failed" +- x86/mm: Check return value from memblock_phys_alloc_range() +- bnxt_en: Fix a possible memory leak in bnxt_ptp_init +- net: hibmcge: add support for pagepool on rx +- net: hibmcge: reduce packet drop under stress testing +- net: hibmcge: add support for tracepoint to dump some fields of rx_desc +- Revert "net: hibmcge: reduce packet drop under stress testing" +- Revert "net: hibmcge: support pagepool for rx" +- wifi: ath11k: update channel list in reg notifier instead reg worker +- !18451 RDMA/bnxt_re: Fix budget handling of notification queue +- ksmbd: fix use-after-free in session logoff +- !18837 CVE-2025-37997 +- !19361 media: b2c2: Fix use-after-free causing by irq_check_work in flexcop_pci_remove +- RDMA/rxe: Fix race in do_task() when draining +- !19353 ubcore: fix query route list by eid pair. +- ubcore: fix query route list by eid pair. +- net/sched: Fix backlog accounting in qdisc_dequeue_internal +- net/sched: sch_qfq: Avoid triggering might_sleep in atomic context in qfq_delete_class +- net/sched: sch_qfq: Fix race condition on qfq_aggregate +- i40e: fix IRQ freeing in i40e_vsi_request_irq_msix error path +- net/mlx5e: Harden uplink netdev access against device unbind +- ixgbevf: fix mailbox API compatibility by negotiating supported features +- ixgbevf: fix getting link speed data for E610 devices +- ixgbevf: Add support for Intel(R) E610 device +- PCI: Add PCI_VDEVICE_SUB helper macro +- media: b2c2: Fix use-after-free causing by irq_check_work in flexcop_pci_remove +- media: uvcvideo: Use heuristic to find stream entity +- media: uvcvideo: Mark invalid entities with id UVC_INVALID_ENTITY_ID +- hisi_acc_vfio_pci: adapt to new migration configuration +- crypto: hisilicon - qm updates BAR configuration +- hisi_acc_vfio_pci: Fix reference leak in hisi_acc_vfio_debug_init +- Revert "migration: update BAR space size" +- Revert "migration: qm updates BAR configuration" +- Revert "migration: adapt to new migration configuration" +- nfsd: don't ignore the return code of svc_proc_register() +- iommu/vt-d: Disallow dirty tracking if incoherent page walk +- mm/ksm: fix flag-dropping behavior in ksm_madvise +- nbd: fix uaf in nbd_genl_connect() error path +- fs: writeback: fix use-after-free in __mark_inode_dirty() +- RDMA/bnxt_re: Remove unusable nq variable +- RDMA/bnxt_re: Fix budget handling of notification queue +- __legitimize_mnt(): check for MNT_SYNC_UMOUNT should be under mount_lock +- scsi: mpi3mr: Synchronous access b/w reset and tm thread for reply queue +- efivarfs: Fix slab-out-of-bounds in efivarfs_d_compare +- wifi: cfg80211: fix out-of-bounds access during multi-link element defragmentation +- Input: mtk-pmic-keys - fix possible null pointer dereference +- ksmbd: prevent rename with empty string +- ksmbd: move leading slash check to smb2_get_name() +- nfsd: Initialize ssc before laundromat_work to prevent NULL dereference +- dmaengine: ti: k3-udma: Add missing locking +- nfsd: provide locking for v4_end_grace +- NFSv4: Fix deadlock during the running of state manager +- scsi: mpi3mr: Add support for status reply descriptor +- scsi: mpi3mr: Split off bus_reset function from host_reset +- netfilter: ipset: Remove unused htable_bits in macro ahash_region +- netfilter: ipset: fix region locking in hash types +- Add platform-specific quirks for ZX-200 device +- ksmbd: add max ip connections parameter +- ksmbd: extend the connection limiting mechanism to support IPv6 +- ksmbd: limit repeated connections from clients with the same IP +- jfs: add check read-only before txBeginAnon() call +- ksmbd: add free_transport ops in ksmbd connection +- ksmbd: fix use-after-free in __smb2_lease_break_noti() + +* Sat Nov 29 2025 Li Nan - 6.6.0-124.0.0.101 +- !19152 Fix CVE-2025-39810 fix CVE-2025-39810 +- !19359 ub: hisi-ubus: Fix ub memory decoder create +- !19346 sched: Fix race between yield_to() and try_to_wake_up() +- ub: hisi-ubus: Fix ub memory decoder create +- !19341 v7 bugfix for xcall2.0 +- sched: Fix race between yield_to() and try_to_wake_up() +- !17661 kvm: Only enable shadow mode for ioeventfd and Align the lifecycle of … +- !19334 bpf: Sync pending IRQ work before freeing ring buffer +- xcall2.0: prefetch: introduce struct prefetch_mm_data +- xcall2.0: Fix mem leak in proc_xcall_command +- xcall2.0: add xcall_subdir_create help to create subdir below /proc/xcall +- xcall2.0: prefetch: keep prefetch module name same with file name +- xcall2.0: prefetch: fix value name typos in __do_sys_epoll_pwait +- !19339 zcopy: Fix context switch within RCU read-side +- !19300 Add Support for Disabling Hugetlb in soft_offline_page +- zcopy: Fix context switch within RCU read-side error +- bpf: Sync pending IRQ work before freeing ring buffer +- kvm: Only enable shadow mode for ioeventfd and Align the lifecycle of KVM shadow with KVM +- mm/memory-failure: support disabling soft offline for HugeTLB pages +- mm/memory-failure: userspace controls soft-offlining pages +- bnxt_en: Fix memory corruption when FW resources change during ifdown +- bnxt_en: Fix possible crash after creating sw mqprio TCs + +* Thu Nov 27 2025 Li Nan - 6.6.0-123.0.0.100 +- !19327 ub: ub_fwctl: add ub_fwctl driver-api documentation description +- !19338 iommu/ummu: Fix UMMU documentation error +- !19287 net: unic: Add support for uboe and ethernet related tools +- !19337 ub: udma: Support retrieving the migr field. +- ub: ub_fwctl: add ub_fwctl driver-api documentation description +- iommu/ummu: Fix UMMU documentation error +- !19332 ub: udma: add requirement about document and tp attr. +- !19330 iommu/ummu: Add UMMU documentation description +- ub: udma: Support retrieving the migr field. +- !18794 【OLK-6.6】fix CVE-2025-39801 +- ub: udma: Support eid and guid updates. +- ub: udma: bugfix for set and get tp attr. +- !18453 [OLK-6.6] ALSA: HDA: Add Zhaoxin HDMI Controller and Codec support +- !18483 [OLK-6.6] x86/hpet: Set dynamic IRQ feature for HPET for KH-50000 +- iommu/ummu: Add UMMU-PMU devicetree file +- iommu/ummu: Add UMMU devicetree file +- iommu/ummu: Add UMMU documentation description +- ub: udma: mask jetty context addr info. +- ub: udma: add udma driver module doc. +- ub: ubase: Remove non-cluster mode code +- net: unic: Add support for uboe reset +- net: unic: Add support for MAC statistics +- net: unic: Add pause frame support. +- net: unic: Add Priority Flow Control (PFC) support +- net: unic: Add debugfs support for dumping MAC tables. +- net: unic: Supports MAC filter, and allows users to add and delete MAC +- net: unic: Add debugfs support for dumping vlan table +- net: unic: Integrate valn filtering features with network device operations. +- net: unic: Add valn filtering support. +- net: unic: Add support for querying link diagnosis status +- net: unic: Extend loopback test support to include external loopback modes +- net: unic: Extend loopback test support to include serdes loopback modes. +- net: unic: Add loopback test support. +- net: unic: Add debugfs support for dumping hardware MAC tables. +- net: unic: Add support for setting link speed, duplex, and autonegotiation via ethtool. +- !19328 ub:hisi-ubus: Adding compatibility Interfaces for ub memory +- !19264 ub: cdma: add CDMA userspace-api documentation description +- ub:hisi-ubus: Adding compatibility Interfaces for ub memory +- ub: cdma: add CDMA userspace-api documentation description +- ub: cdma: add CDMA kernel driver design document specification +- ub: cdma: add CDMA driver-api documentation description +- !19305 ub:ubus: Add ubus and ubfi opensource document +- !19320 urma: added functions for getting and setting TP attribute fields. +- ub:ubus: Add ubus devicetree file +- urma: added functions for getting and setting TP attribute fields. +- ub:ubus: Add ubus and ubfi opensource document +- usb: dwc3: Remove WARN_ON for device endpoint command timeouts +- ALSA: HDA: Add Zhaoxin HDMI Controller and Codec support +- x86/hpet: Set dynamic IRQ feature for HPET for KH-50000 +- ALSA: HDA: Roll back the old version for consistency with the mainline + +* Thu Nov 27 2025 Li Nan - 6.6.0-122.0.0.99 +- !19317 ub: ubase: Remove useless character '+' +- ub: ubase: Remove useless character '+' +- !18870 add sentry kmod +- drivers/ub/sentry: add sentry_remote_reporter module +- drivers/ub/sentry: add sentry_uvb_comm module +- drivers/ub/sentry: add sentry_urma_comm module +- drivers/ub/sentry: add sentry_reporter and sentry_msg_help +- !19306 net: Netdev use RCU protection +- !19290 urma: bugfix the issue of urma stream generation failure +- !19292 blk-ioinf: accumulate total latency properly across sampling periods +- !19230 drm/vmwgfx: Fix Use-after-free in validation +- !19291 usbnet: Fix using smp_processor_id() in preemptible code warnings +- net: use dst_dev_rcu() in sk_setup_caps() +- ipv4: use RCU protection in ip_dst_mtu_maybe_forward() +- ipv6: use RCU in ip6_output() +- tls: Use __sk_dst_get() and dst_dev_rcu() in get_netdev_for_sock(). +- net: dst: Fix kabi-breakage for struct dst_entry +- ipv6: use RCU in ip6_xmit() +- ipv4: start using dst_dev_rcu() +- tcp_metrics: use dst_dev_net_rcu() +- net: dst: introduce dst->dev_rcu +- net: Add locking to protect skb->dev access in ip_output +- net: dst: add four helpers to annotate data-races around dst->dev +- !19191 Fix CVE-2025-40200 +- urma: bugfix the issue of urma stream generation failure +- acpi: provides acpi power notifier chain +- !19303 pid: Add a judgment for ns null in pid_nr_ns +- !19282 fix cves +- !19211 ACPI: video: Fix use-after-free in acpi_video_switch_brightness() +- !19175 [OLK-6.6] Fix CVE-2025-38666 +- !18496 [OLK-6.6] Fix CVE-2025-22107 +- !18029 [OLK-6.6] Fix CVE-2025-37892 +- !19151 [OLK-6.6] Fix CVE-2025-40055 +- !19131 [OLK-6.6] Fix CVE-2025-40012 +- !19125 [OLK-6.6] Fix CVE-2025-39692 +- !19124 [OLK-6.6] Fix CVE-2025-39694 +- !19133 [OLK-6.6] Fix CVE-2025-40082 +- !19118 [OLK-6.6] Fix CVE-2025-39703 +- !19113 [OLK-6.6] Fix CVE-2025-39938 +- !19111 [OLK-6.6] Fix CVE-2025-40043 +- !19100 obmm: Ownership Based Memory Management framework +- pid: Add a judgment for ns null in pid_nr_ns +- !19285 xsched: XCU Partition +- !19293 sched/rt: Fix race in push_rt_task +- !19294 dm-stripe: fix a possible integer overflow +- !19278 ub: ubase: Improve performance and fix some bug +- dm-stripe: fix a possible integer overflow +- sched/rt: Fix race in push_rt_task +- blk-ioinf: accumulate total latency properly across sampling periods +- usbnet: Fix using smp_processor_id() in preemptible code warnings +- xsched: move stream_lock into xsched_vsm_add_tail() to avoid sleeping in atomic context +- xsched: rename vstream->id to vstream->sq_id +- xsched: add null check for sched in xsched_xse_set_class +- xsched: modify the logic for inc and dec the count value +- xsched: replace hard-coded numeric values +- xsched: unify root detection logic for cgroups +- xsched: remove parent->lock and rely on cgroup_mutex for shares update +- xsched: avoid sleeping while holding spinlock in xcu_move_task() +- xsched: add missing spin_unlock() in xcu_move_task() error path +- xsched: enforce valid xsched scheduler config dependencies +- xsched: unify log prefix format and remove duplicated prefix macros +- mm: zswap: fix crypto_free_acomp() deadlock in zswap_cpu_comp_dead() +- mm: zswap: move allocations during CPU init outside the lock +- mm: zswap: properly synchronize freeing resources during CPU hotunplug +- mm/zswap: change per-cpu mutex and buffer to per-acomp_ctx +- mm/hugetlb: unshare page tables during VMA split, not before +- !19194 ASoC: Intel: bytcr_rt5651: Fix invalid quirk input mapping +- !19195 iommu/vt-d: debugfs: Fix legacy mode page table dump logic +- !19273 net/ip6_tunnel: Prevent perpetual tunnel growth +- !19272 tls: wait for pending async decryptions if tls_strp_msg_hold fails +- !19274 netfilter: nft_objref: validate objref and objrefmap expressions +- !19052 arm64/mpam: Add build dependencies for CONFIG_RESCTRL_IOMMU +- !19182 ALSA: ump: Fix buffer overflow at UMP SysEx message conversion +- !19179 x86/iopl: Cure TIF_IO_BITMAP inconsistencies +- !19178 fbdev: Fix fb_set_var to prevent null-ptr-deref in fb_videomode_to_var +- !19177 ACPICA: fix acpi parse and parseext cache leaks +- !19176 crypto: ccp - Fix crash when rebind ccp device for ccp.ko +- ub: ubase: change cmdq/mbx timeout val & fix spell error +- ub: ubase: adapt to response message structure change for ctrlq sl query +- net: unic: Fix UE deactivate message cleaning +- net: unic: Use copy instead of ummu map in tx +- netfilter: nft_objref: validate objref and objrefmap expressions +- net/ip6_tunnel: Prevent perpetual tunnel growth +- tls: wait for pending async decryptions if tls_strp_msg_hold fails +- !19257 init/Kconfig: Move ARCH_SUPPORTS_SCHED_SOFT_QUOTA to drop CGROUPS dependency +- obmm: Add resource management support for imported memory +- obmm: Add sysfs interface for configuration and monitoring +- obmm: Add low memory management support +- obmm: Add ownership management and transfer mechanisms +- obmm: Add mmap support for shared memory regions +- obmm: Add shared memory device interface +- obmm: Add pre-import support for performance optimization +- obmm: Add memory region import functionality +- obmm: Add address validation and checking +- obmm: Add user address export support +- obmm: Add memory region export functionality +- obmm: Implement memory pool allocator +- obmm: Add continuous memory allocator for OBMM framework +- obmm: Add cache maintenance helpers for inter-node consistency +- obmm: Add region definitions and core data structures +- obmm: Add Ownership Based Memory Management framework +- ub:hisi-ubus: Support for ub memory decoder ras handle +- ub:hisi-ubus: Support for UB Memory Decoder Initialization +- ub:ubus: Support for UB Memory Decoder general layer implementation +- !19227 bpf: Avoid RCU context warning when unpinning htab with internal structs +- !18830 mm: swap: check for stable address space before operating on the VMA +- init/Kconfig: Move ARCH_SUPPORTS_SCHED_SOFT_QUOTA to drop CGROUPS dependency +- !19165 Documentation: Add opensource document for ubase&unic&ubl +- !19243 Revert "net: hns3: add support for queue bonding mode of flow director" +- !19115 nvmet-fc: move lsop put work to nvmet_fc_ls_req_op +- !19222 sctp: Fix MAC comparison to be constant-time +- !19223 net/sctp: fix a null dereference in sctp_disposition sctp_sf_do_5_1D_ce() +- !19224 bpf: Fix metadata_dst leak __bpf_redirect_neigh_v{4,6} +- !19226 sunrpc: fix null pointer dereference on zero-length checksum +- Revert "net: hns3: add support for queue bonding mode of flow director" +- Revert "net: hns3: add queue bonding mode support for VF" +- Revert "net: hns3: refactor the debugfs for dumping FD tcam" +- Revert "net: hns3: correct the logic of hclge_sync_vf_qb_mode()" +- Revert "net: hns3: allocate fd counter for queue bonding" +- !19166 [OLK-6.6][linux-6.6.y sync] Backport 6.6.89-6.6.90 LTS Patches +- !19116 Fix SPI registration failure logging level +- !19216 net: usb: asix: hold PM usage ref to avoid PM/MDIO + RTNL deadlock +- drm/vmwgfx: Fix Use-after-free in validation +- Fix SPI registration failure logging level +- !19072 [OLK-6.6] Fix CVE-2025-39827 +- !18647 [OLK-6.6] Fix CVE-2025-38729 +- !19117 [OLK-6.6] Fix CVE-2025-40088 +- !19114 [OLK-6.6] Fix CVE-2025-40084 +- !19048 [OLK-6.6] Fix CVE-2025-39839 +- !18615 [OLK-6.6] Fix CVE-2025-39737 +- !18791 【OLK-6.6】fix CVE-2025-39736 +- !18644 [OLK-6.6] Fix CVE-2025-38717 +- !18653 [OLK-6.6] Fix CVE-2025-38705 +- !18780 【OLK-6.6】fix CVE-2025-38712 +- !18613 [OLK-6.6] Fix CVE-2025-38713 +- !18632 [OLK-6.6] Fix CVE-2025-38714 +- !18861 [OLK-6.6] Fix CVE-2025-38697 +- !18345 [OLK-6.6] Fix CVE-2025-38663 +- !18948 [OLK-6.6] Fix CVE-2025-38105 +- !18661 [OLK-6.6] Fix CVE-2025-39790 +- !18867 [OLK-6.6] Fix CVE-2025-39766 +- !18636 [OLK-6.6]Fix CVE-2025-39751 +- !19110 CVE-2025-40198 +- !19212 CVE-2025-40179 +- !19057 blk-mq: check kobject state_in_sysfs before deleting in blk_mq_unregister_hctx +- !19213 Fix CVE-2025-39964 +- !19189 KVM: arm64: ipiv: change declaration argument +- bpf: Avoid RCU context warning when unpinning htab with internal structs +- sunrpc: fix null pointer dereference on zero-length checksum +- bpf: Fix metadata_dst leak __bpf_redirect_neigh_v{4,6} +- net/sctp: fix a null dereference in sctp_disposition sctp_sf_do_5_1D_ce() +- sctp: Fix MAC comparison to be constant-time +- !19214 ub: udma: recent bugfix for udma. +- !19209 Input: uinput - zero-initialize uinput_ff_upload_compat to avoid info leak +- !19193 crypto: af_alg - Set merge to zero early in af_alg_sendmsg +- !19206 net: dlink: handle copy_thresh allocation failure +- !19207 net: usb: Remove disruptive netif_wake_queue in rtl8150_set_multicast +- net: usb: asix: hold PM usage ref to avoid PM/MDIO + RTNL deadlock +- ub: ubase: Introduces the functions and data structures exposed by the ubase driver +- ub: ubase: Add ubase opensource document +- net: unic: Add a doc for unic driver +- ub: udma: Resolve issues related to stream logout and interrupt lock. +- crypto: af_alg - Fix incorrect boolean values in af_alg_ctx +- crypto: af_alg - Disallow concurrent writes in af_alg_sendmsg +- ext4: align max orphan file size with e2fsprogs limit +- ext4: free orphan info with kvfree +- ext4: verify orphan file size is not too big +- ext4: use kmalloc_array() for array space allocation +- ub: udma: reset and segment permission issues are resolved. +- !19203 Fix CVE-2025-40202 +- ACPI: video: Fix use-after-free in acpi_video_switch_brightness() +- Input: uinput - zero-initialize uinput_ff_upload_compat to avoid info leak +- net: usb: Remove disruptive netif_wake_queue in rtl8150_set_multicast +- net: dlink: handle copy_thresh allocation failure +- spi: tegra114: Use value to check for invalid delays +- platform/x86/amd: pmc: Clear metrics table at start of cycle +- dm: fix copying after src array boundaries +- iommu/arm-smmu-v3: Fix iommu_device_probe bug due to duplicated stream ids +- drivers: base: handle module_kobject creation +- kernel: globalize lookup_or_create_module_kobject() +- kernel: param: rename locate_module_kobject +- xhci: split free interrupter into separate remove and free parts +- xhci: Clean up stale comment on ERST_SIZE macro +- xhci: Use more than one Event Ring segment +- xhci: Set DESI bits in ERDP register correctly +- Revert "drm/meson: vclk: fix calculation of 59.94 fractional rates" +- arm64: dts: st: Use 128kB size for aliased GIC400 register access on stm32mp25 SoCs +- arm64: dts: st: Adjust interrupt-controller for stm32mp25 SoCs +- ARM: dts: opos6ul: add ksz8081 phy properties +- firmware: arm_ffa: Skip Rx buffer ownership release if not acquired +- firmware: arm_scmi: Balance device refcount when destroying devices +- sch_htb: make htb_qlen_notify() idempotent +- ASoC: soc-core: Stop using of_property_read_bool() for non-boolean properties +- ASoC: Use of_property_read_bool() +- net: vertexcom: mse102x: Fix RX error handling +- net: vertexcom: mse102x: Add range check for CMD_RTS +- net: vertexcom: mse102x: Fix LEN_MASK +- net: vertexcom: mse102x: Fix possible stuck of SPI interrupt +- net: fec: ERR007885 Workaround for conventional TX +- net: ipv6: fix UDPv6 GSO segmentation with NAT +- net: dsa: felix: fix broken taprio gate states after clock jump +- net: mdio: mux-meson-gxl: set reversed bit when using internal phy +- net: dlink: Correct endianness handling of led_mode +- nvme-pci: fix queue unquiesce check on slot_reset +- ALSA: ump: Fix buffer overflow at UMP SysEx message conversion +- pds_core: make pdsc_auxbus_dev_del() void +- pds_core: delete VF dev on reset +- pds_core: check health in devcmd wait +- net: ethernet: mtk-star-emac: rearm interrupts in rx_poll only when advised +- net: mscc: ocelot: delete PVID VLAN when readding it as non-PVID +- net: mscc: ocelot: treat 802.1ad tagged traffic as 802.1Q-untagged +- !19190 Revert "ipmi: fix msg stack when IPMI is disconnected" +- !19167 drm/vmwgfx: Fix a null-ptr access in the cursor snooper +- ipmi: Fix handling of messages with provided receive message pointer +- ipmi: Rework user message limit handling +- !19134 Bluetooth: ISO: Fix possible UAF on iso_conn_free +- !19119 dm: fix NULL pointer dereference in __dm_suspend() +- !19137 ext4: guard against EA inode refcount underflow in xattr update +- iommu/vt-d: debugfs: Fix legacy mode page table dump logic +- ASoC: Intel: bytcr_rt5651: Fix invalid quirk input mapping +- crypto: af_alg - Set merge to zero early in af_alg_sendmsg +- Bluetooth: btusb: avoid NULL pointer dereference in skb_dequeue() +- net/mlx5: E-switch, Fix error handling for enabling roce +- net/mlx5: E-Switch, Initialize MAC Address for Default GID +- powerpc/boot: Fix dash warning +- drm/i915/pxp: fix undefined reference to `intel_pxp_gsccs_is_ready_for_sessions' +- powerpc/boot: Check for ld-option support +- ASoC: soc-pcm: Fix hw_params() and DAPM widget sequence +- Squashfs: reject negative file sizes in squashfs_read_inode() +- Squashfs: add additional inode sanity checking +- Revert "ipmi: fix msg stack when IPMI is disconnected" +- KVM: arm64: ipiv: change declaration argument +- !19030 cpufreq: intel_pstate: Fix object lifecycle issue in update_qos_request() +- ALSA: ump: Fix buffer overflow at UMP SysEx message conversion +- !19045 net: fix the compile problem introduced by AF_UB +- x86/iopl: Cure TIF_IO_BITMAP inconsistencies +- fbdev: Fix fb_set_var to prevent null-ptr-deref in fb_videomode_to_var +- ACPICA: fix acpi parse and parseext cache leaks +- crypto: ccp - Fix crash when rebind ccp device for ccp.ko +- net: appletalk: Fix use-after-free in AARP proxy probe +- net: ubl: Add a doc for ubl module +- !19161 uburma: add tp_type in urma ioctl operation. +- drm/vmwgfx: Fix a null-ptr access in the cursor snooper +- PCI: imx6: Skip controller_id generation logic for i.MX7D +- Revert "PCI: imx6: Skip controller_id generation logic for i.MX7D" +- selftests/bpf: extend changes_pkt_data with cases w/o subprograms +- selftests/bpf: validate that tail call invalidates packet pointers +- selftests/bpf: freplace tests for tracking of changes_packet_data +- cpufreq: Fix setting policy limits when frequency tables are used +- cpufreq: Avoid using inconsistent policy->min and policy->max +- smb: client: fix zero length for mkdir POSIX create context +- platform/x86/intel-uncore-freq: Fix missing uncore sysfs during CPU hotplug +- platform/x86/amd: pmc: Require at least 2.5 seconds between HW sleep cycles +- iommu/vt-d: Apply quirk_iommu_igfx for 8086:0044 (QM57/QS57) +- iommu/amd: Fix potential buffer overflow in parse_ivrs_acpihid +- dm: always update the array size in realloc_argv on success +- dm-integrity: fix a warning on invalid table line +- dm-bufio: don't schedule in atomic context +- tracing: Do not take trace_event_sem in print_event_fields() +- spi: tegra114: Don't fail set_cs_timing when delays are zero +- mmc: renesas_sdhi: Fix error handling in renesas_sdhi_probe +- mm/memblock: repeat setting reserved region nid if array is doubled +- mm/memblock: pass size instead of end to memblock_set_node() +- amd-xgbe: Fix to ensure dependent features are toggled with RX checksum offload +- i2c: imx-lpi2c: Fix clock count when probe defers +- EDAC/altera: Set DDR and SDMMC interrupt mask before registration +- EDAC/altera: Test the correct error reg offset +- drm/fdinfo: Protect against driver unbind +- btrfs: fix COW handling in run_delalloc_nocow() +- ALSA: usb-audio: Add second USB ID for Jabra Evolve 65 headset +- ALSA: usb-audio: Add retry on -EPROTO from usb_set_interface() +- !19143 [OLK-6.6][linux-6.6.y sync] Backport 6.6.88-6.6.89 LTS Patches +- uburma: add tp_type in urma ioctl operation. +- !19132 IPIV: fix bug in live migration +- ocfs2: fix double free in user_cluster_connect() +- !17056 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.68-6.6.69 LTS Conflicts Patches +- !18479 [OLK-6.6] xhci: fix memory leak regression when freeing xhci vdev devices depth first +- !19007 uio_hv_generic: Let userspace take care of interrupt mask +- ACPI: PPTT: Fix processor subtable walk +- scsi: pm80xx: Fix array-index-out-of-of-bounds on rmmod +- cifs: Query EA $LXMOD in cifs_query_path_info() for WSL reparse points +- dmaengine: Revert "dmaengine: dmatest: Fix dmatest waiting less when interrupted" +- Revert "usb: xhci: Avoid Stop Endpoint retry loop if the endpoint seems Running" +- ata: libata-scsi: Fix CDL control +- net: fix the compile problem introduced by AF_UB +- objtool: Silence more KCOV warnings, part 2 +- objtool: Ignore end-of-section jumps for KCOV/GCOV +- nvme: fixup scan failure for non-ANA multipath controllers +- MIPS: cm: Fix warning if MIPS_CM is disabled +- net: dsa: mv88e6xxx: enable STU methods for 6320 family +- net: dsa: mv88e6xxx: enable .port_set_policy() for 6320 family +- net: dsa: mv88e6xxx: enable PVT for 6321 switch +- net: dsa: mv88e6xxx: fix atu_move_port_mask for 6341 family +- comedi: jr3_pci: Fix synchronous deletion of timer +- driver core: introduce device_set_driver() helper +- x86/pvh: Call C code via the kernel virtual mapping +- md/raid1: Add check for missing source disk in process_checks() +- x86/cpu: Add CPU model number for Bartlett Lake CPUs with Raptor Cove cores +- ubsan: Fix panic from test_ubsan_out_of_bounds +- spi: tegra210-quad: add rate limiting and simplify timeout error message +- spi: tegra210-quad: use WARN_ON_ONCE instead of WARN_ON for timeouts +- loop: aio inherit the ioprio of original request +- riscv: Provide all alternative macros all the time +- iomap: skip unnecessary ifs_block_is_uptodate check +- x86/i8253: Call clockevent_i8253_disable() with interrupts disabled +- scsi: pm80xx: Set phy_attached to zero when device is gone +- scsi: ufs: exynos: Ensure pre_link() executes before exynos_ufs_phy_init() +- ext4: make block validity check resistent to sb bh corruption +- cifs: Fix querying of WSL CHR and BLK reparse points over SMB1 +- timekeeping: Add a lockdep override in tick_freeze() +- cifs: Fix encoding of SMB1 Session Setup Kerberos Request in non-UNICODE mode +- nvmet-fc: put ref when assoc->del_work is already scheduled +- nvmet-fc: take tgtport reference only once +- x86/bugs: Don't fill RSB on context switch with eIBRS +- x86/bugs: Don't fill RSB on VMEXIT with eIBRS+retpoline +- x86/bugs: Use SBPB in write_ibpb() if applicable +- selftests/mincore: Allow read-ahead pages to reach the end of the file +- gpiolib: of: Move Atmel HSMCI quirk up out of the regulator comment +- objtool: Stop UNRET validation on UD2 +- nvme: multipath: fix return value of nvme_available_path +- nvme: re-read ANA log page after ns scan completes +- ACPI PPTT: Fix coding mistakes in a couple of sizeof() calls +- ACPI: EC: Set ec_no_wakeup for Lenovo Go S +- nvme: requeue namespace scan on missed AENs +- xen: Change xen-acpi-processor dom0 dependency +- selftests: ublk: fix test_stripe_04 +- udmabuf: fix a buf size overflow issue during udmabuf creation +- KVM: s390: Don't use %pK through debug printing +- KVM: s390: Don't use %pK through tracepoints +- sched/isolation: Make CONFIG_CPU_ISOLATION depend on CONFIG_SMP +- io_uring: always do atomic put from iowq +- rtc: pcf85063: do a SW reset if POR failed +- ntb_hw_amd: Add NTB PCI ID for new gen CPU +- ntb: reduce stack usage in idt_scan_mws +- objtool, lkdtm: Obfuscate the do_nothing() pointer +- objtool, regulator: rk808: Remove potential undefined behavior in rk806_set_mode_dcdc() +- objtool, ASoC: codecs: wcd934x: Remove potential undefined behavior in wcd934x_slim_irq_handler() +- objtool, panic: Disable SMAP in __stack_chk_fail() +- objtool: Silence more KCOV warnings +- thunderbolt: Scan retimers after device router has been enumerated +- usb: host: xhci-plat: mvebu: use ->quirks instead of ->init_quirk() func +- usb: gadget: aspeed: Add NULL pointer check in ast_vhub_init_dev() +- usb: xhci: Avoid Stop Endpoint retry loop if the endpoint seems Running +- dmaengine: dmatest: Fix dmatest waiting less when interrupted +- sound/virtio: Fix cancel_sync warnings on uninitialized work_structs +- usb: dwc3: gadget: Avoid using reserved endpoints on Intel Merrifield +- usb: dwc3: gadget: Refactor loop to avoid NULL endpoints +- fs/ntfs3: Fix WARNING in ntfs_extend_initialized_size +- usb: host: max3421-hcd: Add missing spi_device_id table +- bpf: Reject attaching fexit/fmod_ret to __noreturn functions +- bpf: Only fails the busy counter check in bpf_cgrp_storage_get if it creates storage +- bpf: bpftool: Setting error code in do_loader() +- s390/tty: Fix a potential memory leak bug +- s390/sclp: Add check for get_zeroed_page() +- parisc: PDT: Fix missing prototype warning +- clk: check for disabled clock-provider in of_clk_get_hw_from_clkspec() +- crypto: ccp - Add support for PCI device 0x1134 +- MIPS: cm: Detect CM quirks from device tree +- pinctrl: renesas: rza2: Fix potential NULL pointer dereference +- USB: wdm: add annotation +- USB: wdm: wdm_wwan_port_tx_complete mutex in atomic context +- USB: wdm: close race between wdm_open and wdm_wwan_port_stop +- USB: wdm: handle IO errors in wdm_wwan_port_start +- USB: VLI disk crashes if LPM is used +- usb: quirks: Add delay init quirk for SanDisk 3.2Gen1 Flash Drive +- usb: quirks: add DELAY_INIT quirk for Silicon Motion Flash Drive +- usb: dwc3: xilinx: Prevent spike in reset signal +- USB: OHCI: Add quirk for LS7A OHCI controller (rev 0x02) +- usb: chipidea: ci_hdrc_imx: implement usb_phy_init() error handling +- usb: chipidea: ci_hdrc_imx: fix call balance of regulator routines +- usb: cdns3: Fix deadlock when using NCM gadget +- USB: serial: simple: add OWON HDS200 series oscilloscope support +- USB: serial: option: add Sierra Wireless EM9291 +- USB: serial: ftdi_sio: add support for Abacus Electrics Optical Probe +- serial: sifive: lock port in startup()/shutdown() callbacks +- serial: msm: Configure correct working mode before starting earlycon +- misc: microchip: pci1xxxx: Fix incorrect IRQ status handling during ack +- misc: microchip: pci1xxxx: Fix Kernel panic during IRQ handler registration +- char: misc: register chrdev region with all possible minors +- KVM: x86: Explicitly treat routing entry type changes as changes +- mei: me: add panther lake H DID +- scsi: Improve CDL control +- USB: storage: quirk for ADATA Portable HDD CH94 +- ata: libata-scsi: Fix ata_msense_control_ata_feature() +- ata: libata-scsi: Fix ata_mselect_control_ata_feature() return type +- ata: libata-scsi: Improve CDL control +- cxl/core/regs.c: Skip Memory Space Enable check for RCD and RCH Ports +- KVM: SVM: Allocate IR data using atomic allocation +- io_uring: fix 'sync' handling of io_fallback_tw() +- LoongArch: Remove a bogus reference to ZONE_DMA +- drm/amd/display: Force full update in gpu reset +- drm/amd/display: Fix gpu reset in multidisplay config +- net: phy: microchip: force IRQ polling mode for lan88xx +- net: selftests: initialize TCP header and skb payload with zero +- crypto: atmel-sha204a - Set hwrng quality to lowest possible +- virtio_console: fix missing byte order handling for cols and rows +- LoongArch: Make do_xyz() exception handlers more robust +- LoongArch: Make regs_irqs_disabled() more clear +- LoongArch: Select ARCH_USE_MEMTEST +- splice: remove duplicate noinline from pipe_clear_nowait +- iommu/amd: Return an error if vCPU affinity is set for non-vCPU IRTE +- pds_core: Remove unnecessary check in pds_client_adminq_cmd() +- net: dsa: mt7530: sync driver-specific behavior of MT7531 variants +- net: ethernet: mtk_eth_soc: net: revise NETSYSv3 hardware configuration +- scsi: core: Clear flags for scsi_cmnd that did not complete +- btrfs: avoid page_lockend underflow in btrfs_punch_hole_lock_range() +- cpufreq: apple-soc: Fix null-ptr-deref in apple_soc_cpufreq_get_rate() +- dma/contiguous: avoid warning about unused size_bytes +- ceph: Fix incorrect flush end position calculation +- scsi: ufs: qcom: fix dev reference leaked through of_qcom_ice_get +- of: resolver: Fix device node refcount leakage in of_resolve_phandles() +- of: resolver: Simplify of_resolve_phandles() using __free() +- clk: renesas: r9a07g043: Fix HP clock source for RZ/Five +- clk: renesas: r9a07g04[34]: Fix typo for sel_shdi variable +- clk: renesas: r9a07g04[34]: Use SEL_SDHI1_STS status configuration for SD1 mux +- clk: renesas: rzg2l: Refactor SD mux driver +- clk: renesas: rzg2l: Remove CPG_SDHI_DSEL from generic header +- clk: renesas: rzg2l: Add struct clk_hw_data +- clk: renesas: rzg2l: Use u32 for flag and mux_flags +- arm64: tegra: Remove the Orin NX/Nano suspend key +- iio: adc: ad7768-1: Fix conversion result sign +- iio: adc: ad7768-1: Move setting of val a bit later to avoid unnecessary return value check +- ASoC: q6apm-dai: make use of q6apm_get_hw_pointer +- ASoC: qcom: Fix trivial code style issues +- ASoC: qcom: lpass: Make asoc_qcom_lpass_cpu_platform_remove() return void +- ASoC: q6apm-dai: schedule all available frames to avoid dsp under-runs +- ASoC: qcom: q6apm-dai: drop unused 'q6apm_dai_rtd' fields +- net: dsa: mv88e6xxx: fix VTU methods for 6320 family +- net: dsa: mv88e6xxx: fix internal PHYs for 6320 family +- auxdisplay: hd44780: Fix an API misuse in hd44780.c +- auxdisplay: hd44780: Convert to platform remove callback returning void +- mmc: sdhci-msm: fix dev reference leaked through of_qcom_ice_get +- soc: qcom: ice: introduce devm_of_qcom_ice_get +- module: sign with sha512 instead of sha1 by default +- !18797 [OLK-6.6] Fix CVE-2025-38331 +- !18978 [OLK-6.6][linux-6.6.y sync] Backport 6.6.87-6.6.88 LTS Patches +- !19141 crash: fix crashkernel resource shrink +- !19136 ext4: detect invalid INLINE_DATA + EXTENTS flag combination +- crash: fix crashkernel resource shrink +- ext4: guard against EA inode refcount underflow in xattr update +- ext4: detect invalid INLINE_DATA + EXTENTS flag combination +- net: rose: fix a typo in rose_clear_routes() +- Bluetooth: ISO: Fix possible UAF on iso_conn_free +- net: nfc: nci: Add parameter validation for packet data +- KVM: arm64: ipiv: fix bug in live migration +- KVM: arm64: ipiv: Change parameter from vcpu to kvm +- hfsplus: fix slab-out-of-bounds read in hfsplus_uni2asc() +- hfsplus: fix slab-out-of-bounds read in hfsplus_uni2asc() +- net/smc: fix warning in smc_rx_splice() when calling get_page() +- smb: server: split ksmbd_rdma_stop_listening() out of ksmbd_rdma_destroy() +- s390/sclp: Fix SCCB present check +- net: rose: include node references in rose_neigh refcount +- net: rose: convert 'use' field to refcount_t +- net: rose: split remove and free operations in rose_remove_neigh() +- dm: fix NULL pointer dereference in __dm_suspend() +- net, hsr: reject HSR frame if skb can't hold tag +- hfsplus: fix slab-out-of-bounds read in hfsplus_strcasecmp() +- nvmet-fc: move lsop put work to nvmet_fc_ls_req_op +- ksmbd: transport_ipc: validate payload size before reading handle +- ASoC: qcom: q6apm-lpass-dais: Fix NULL pointer dereference if source graph failed +- ext4: fix string copying in parse_apply_sb_mount_options() +- ext4: avoid potential buffer over-read in parse_apply_sb_mount_options() +- blk-mq: check kobject state_in_sysfs before deleting in blk_mq_unregister_hctx +- arm64/mpam: Add build dependencies for CONFIG_RESCTRL_IOMMU +- batman-adv: fix OOB read/write in network-coding decode +- cpufreq: intel_pstate: Fix object lifecycle issue in update_qos_request() +- md/md-bitmap: fix GPF in bitmap_get_stats() +- igc: fix lock order in igc_ptp_reset +- Reapply "wifi: mac80211: Update skb's control block key in ieee80211_tx_dequeue()" +- xen/x86: fix initial memory balloon target +- pwm: mediatek: Ensure to disable clocks in error path +- ALSA: hda/realtek: Fix built-in mic on another ASUS VivoBook model +- drm/tests: Build KMS helpers when DRM_KUNIT_TEST_HELPERS is enabled +- btrfs: fix the length of reserved qgroup to free +- MIPS: ds1287: Match ds1287_set_base_clock() function types +- MIPS: cevt-ds1287: Add missing ds1287.h include +- MIPS: dec: Declare which_prom() as static +- sign-file,extract-cert: use pkcs11 provider for OPENSSL MAJOR >= 3 +- sign-file,extract-cert: avoid using deprecated ERR_get_error_line() +- sign-file,extract-cert: move common SSL helper functions to a header +- nvmet-fc: Remove unused functions +- landlock: Add the errata interface +- wifi: rtw89: pci: disable PCIE wake bit when PCIE deinit +- wifi: rtw89: pci: add pre_deinit to be called after probe complete +- LoongArch: Eliminate superfluous get_numa_distances_cnt() +- x86/split_lock: Fix the delayed detection logic +- x86/xen: fix memblock_reserve() usage on PVH +- x86/xen: move xen_reserve_extra_memory() +- Fix mmu notifiers for range-based invalidates +- misc: pci_endpoint_test: Fix 'irq_type' to convey the correct type +- misc: pci_endpoint_test: Fix displaying 'irq_type' after 'request_irq' error +- kbuild: Add '-fno-builtin-wcslen' +- cpufreq: Reference count policy in cpufreq_update_limits() +- io_uring/net: fix accept multishot handling +- drm/i915/gvt: fix unterminated-string-initialization warning +- drm/sti: remove duplicate object names +- drm/amdgpu/dma_buf: fix page_link check +- drm/amd/pm/powerplay/hwmgr/vega20_thermal: Prevent division by zero +- drm/amd/pm/powerplay/hwmgr/smu7_thermal: Prevent division by zero +- drm/amd/pm/smu11: Prevent division by zero +- drm/amd/pm/powerplay: Prevent division by zero +- drm/amd/pm: Prevent division by zero +- drm/amd: Handle being compiled without SI or CIK support better +- drm/msm/a6xx: Fix stale rpmh votes from GPU +- drm/repaper: fix integer overflows in repeat functions +- perf/x86/intel/uncore: Fix the scale of IIO free running counters on SPR +- perf/x86/intel/uncore: Fix the scale of IIO free running counters on ICX +- perf/x86/intel/uncore: Fix the scale of IIO free running counters on SNR +- perf/x86/intel: Allow to update user space GPRs from PEBS records +- scsi: ufs: exynos: Ensure consistent phy reference counts +- scsi: megaraid_sas: Block zero-length ATA VPD inquiry +- x86/boot/sev: Avoid shared GHCB page for early memory acceptance +- x86/cpu/amd: Fix workaround for erratum 1054 +- x86/microcode/AMD: Extend the SHA check to Zen5, block loading of any unreleased standalone Zen5 microcode patches +- string: Add load_unaligned_zeropad() code path to sized_strscpy() +- smb3 client: fix open hardlink on deferred close file error +- selftests/mm: generate a temporary mountpoint for cgroup filesystem +- riscv: Avoid fortify warning in syscall_get_arguments() +- ksmbd: fix the warning from __kernel_write_iter +- ksmbd: Prevent integer overflow in calculation of deadtime +- ksmbd: fix use-after-free in smb_break_all_levII_oplock() +- ovl: don't allow datadir only +- mm: fix apply_to_existing_page_range() +- mm/gup: fix wrongly calculated returned value in fault_in_safe_writeable() +- loop: LOOP_SET_FD: send uevents for partitions +- loop: properly send KOBJ_CHANGED uevent for disk device +- crypto: caam/qi - Fix drv_ctx refcount bug +- btrfs: correctly escape subvol in btrfs_show_options() +- Bluetooth: vhci: Avoid needless snprintf() calls +- Bluetooth: l2cap: Process valid commands in too long frame +- ftrace: fix incorrect hash size in register_ftrace_direct() +- i2c: atr: Fix wrong include +- nfs: add missing selections of CONFIG_CRC32 +- asus-laptop: Fix an uninitialized variable +- ASoC: codecs:lpass-wsa-macro: Fix logic of enabling vi channels +- ASoC: codecs:lpass-wsa-macro: Fix vi feedback rate +- writeback: fix false warning in inode_to_wb() +- cpufreq/sched: Fix the usage of CPUFREQ_NEED_UPDATE_LIMITS +- riscv: KGDB: Remove ".option norvc/.option rvc" for kgdb_compiled_break +- riscv: KGDB: Do not inline arch_kgdb_breakpoint() +- kunit: qemu_configs: SH: Respect kunit cmdline +- riscv: Properly export reserved regions in /proc/iomem +- net: ethernet: mtk_eth_soc: revise QDMA packet scheduler settings +- net: ethernet: mtk_eth_soc: correct the max weight of the queue limit for 100Mbps +- ptp: ocp: fix start time alignment in ptp_ocp_signal_set +- net: dsa: avoid refcount warnings when ds->ops->tag_8021q_vlan_del() fails +- net: bridge: switchdev: do not notify new brentries as changed +- net: b53: enable BPDU reception for management port +- netlink: specs: rt-link: adjust mctp attribute naming +- netlink: specs: rt-link: add an attr layer around alt-ifname +- ata: libata-sata: Save all fields from sense data descriptor +- net: ethernet: ti: am65-cpsw: fix port_np reference counting +- net: ethernet: ti: am65-cpsw-nuss: rename phy_node -> port_np +- pds_core: fix memory leak in pdsc_debugfs_add_qcq() +- test suite: use %zu to print size_t +- igc: add lock preventing multiple simultaneous PTM transactions +- igc: cleanup PTP module if probe fails +- igc: handle the IGC_PTP_ENABLED flag correctly +- igc: move ktime snapshot into PTM retry loop +- igc: increase wait time before retrying PTM +- Revert "wifi: mac80211: Update skb's control block key in ieee80211_tx_dequeue()" +- Bluetooth: hci_event: Fix sending MGMT_EV_DEVICE_FOUND for invalid address +- ASoC: cs42l43: Reset clamp override on jack removal +- RDMA/usnic: Fix passing zero to PTR_ERR in usnic_ib_pci_probe() +- ovl: remove unused forward declaration +- ASoC: Intel: avs: Fix null-ptr-deref in avs_component_probe() +- ASoC: dwc: always enable/disable i2s irqs +- md/md-bitmap: fix stats collection for external bitmaps +- md/raid10: fix missing discard IO accounting +- scsi: iscsi: Fix missing scsi_host_put() in error path +- wifi: wl1251: fix memory leak in wl1251_tx_work +- wifi: mac80211: Update skb's control block key in ieee80211_tx_dequeue() +- Bluetooth: hci_uart: Fix another race during initialization +- media: mediatek: vcodec: mark vdec_vp9_slice_map_counts_eob_coef noinline +- x86/e820: Fix handling of subpage regions when calculating nosave ranges in e820__register_nosave_regions() +- ACPI: platform-profile: Fix CFI violation when accessing sysfs files +- selftests: mptcp: fix incorrect fd checks in main_loop +- selftests: mptcp: close fd_in before returning in main_loop +- pinctrl: qcom: Clear latched interrupt status when changing IRQ type +- phy: freescale: imx8m-pcie: assert phy reset and perst in power off +- PCI: Fix reference leak in pci_alloc_child_bus() +- PCI: brcmstb: Fix missing of_node_put() in brcm_pcie_probe() +- of/irq: Fix device node refcount leakages in of_irq_init() +- of/irq: Fix device node refcount leakage in API irq_of_parse_and_map() +- of/irq: Fix device node refcount leakages in of_irq_count() +- of/irq: Fix device node refcount leakage in API of_irq_parse_raw() +- of/irq: Fix device node refcount leakage in API of_irq_parse_one() +- ntb: use 64-bit arithmetic for the MSI doorbell mask +- KVM: x86: Explicitly zero-initialize on-stack CPUID unions +- gve: handle overflow when reporting TX consumed descriptors +- gpio: zynq: Fix wakeup source leaks on device unbind +- gpio: tegra186: fix resource handling in ACPI probe path +- dt-bindings: coresight: qcom,coresight-tpdm: Fix too many 'reg' +- dt-bindings: coresight: qcom,coresight-tpda: Fix too many 'reg' +- dm-verity: fix prefetch-vs-suspend race +- dm-integrity: set ti->error on memory allocation failure +- dm-ebs: fix prefetch-vs-suspend race +- crypto: ccp - Fix check for the primary ASP device +- clk: qcom: gdsc: Set retain_ff before moving to HW CTRL +- clk: qcom: gdsc: Capture pm_genpd_add_subdomain result code +- clk: qcom: gdsc: Release pm subdomains in reverse add order +- clk: qcom: clk-branch: Fix invert halt status bit check for votable clocks +- cifs: fix integer overflow in match_server() +- cifs: avoid NULL pointer dereference in dbg call +- thermal/drivers/rockchip: Add missing rk3328 mapping entry +- x86/xen: fix balloon target initialization for PVH dom0 +- mm/hugetlb: move hugetlb_sysctl_init() to the __init section +- mm/userfaultfd: fix release hang over concurrent GUP +- ALSA: usb-audio: fix race condition to UAF in snd_usbmidi_free +- ALSA: usb-audio: Kill timer properly at removal +- hfsplus: fix slab-out-of-bounds read in hfsplus_uni2asc() +- uio_hv_generic: Let userspace take care of interrupt mask +- mm/mremap: correctly handle partial mremap() of VMA starting at 0 +- mm: fix lazy mmu docs and usage +- mm: make page_mapped_in_vma() hugetlb walk aware +- mm/rmap: reject hugetlb folios in folio_make_device_exclusive() +- sparc/mm: avoid calling arch_enter/leave_lazy_mmu() in set_ptes +- sparc/mm: disable preemption in lazy mmu mode +- iommufd: Fix uninitialized rc in iommufd_access_rw() +- btrfs: zoned: fix zone finishing with missing devices +- btrfs: zoned: fix zone activation with missing devices +- btrfs: fix non-empty delayed iputs list on unmount due to compressed write workers +- arm64: dts: mediatek: mt8173: Fix disp-pwm compatible string +- arm64: mm: Correct the update of max_pfn +- mtd: rawnand: Add status chack in r852_ready() +- mtd: inftlcore: Add error check for inftl_read_oob() +- mptcp: only inc MPJoinAckHMacFailure for HMAC failures +- lib: scatterlist: fix sg_split_phys to preserve original scatterlist offsets +- locking/lockdep: Decrease nr_unused_locks if lock unused in zap_class() +- mailbox: tegra-hsp: Define dimensioning masks in SoC data +- leds: rgb: leds-qcom-lpg: Fix calculation of best period Hi-Res PWMs +- leds: rgb: leds-qcom-lpg: Fix pwm resolution max for Hi-Res PWMs +- jbd2: remove wrong sb->s_sequence check +- i3c: Add NULL pointer check in i3c_master_queue_ibi() +- i3c: master: svc: Use readsb helper for reading MDB +- smb311 client: fix missing tcon check when mounting with linux/posix extensions +- udf: Fix inode_getblk() return value +- vdpa/mlx5: Fix oversized null mkey longer than 32bit +- f2fs: fix to avoid atomicity corruption of atomic file +- ext4: fix off-by-one error in do_split +- ASoC: qdsp6: q6asm-dai: fix q6asm_dai_compr_set_params error path +- ASoC: qdsp6: q6apm-dai: fix capture pipeline overruns. +- ASoC: qdsp6: q6apm-dai: set 10 ms period and buffer alignment. +- ASoC: q6apm: add q6apm_get_hw_pointer helper +- io_uring/kbuf: reject zero sized provided buffers +- wifi: mac80211: fix integer overflow in hwmp_route_info_get() +- wifi: mt76: Add check for devm_kstrdup() +- clocksource/drivers/stm32-lptimer: Use wakeup capable instead of init wakeup +- mtd: Replace kcalloc() with devm_kcalloc() +- net: dsa: mv88e6xxx: workaround RGMII transmit delay erratum for 6320 family +- mtd: Add check for devm_kcalloc() +- mptcp: sockopt: fix getting IPV6_V6ONLY +- media: i2c: imx219: Rectify runtime PM handling in probe and remove +- media: venus: hfi_parser: add check to avoid out of bound access +- media: i2c: ov7251: Introduce 1 ms delay between regulators and en GPIO +- media: i2c: ov7251: Set enable GPIO low in probe +- media: i2c: ccs: Set the device's runtime PM status correctly in probe +- media: i2c: ccs: Set the device's runtime PM status correctly in remove +- media: v4l2-dv-timings: prevent possible overflow in v4l2_detect_gtf() +- media: platform: stm32: Add check for clk_enable() +- media: visl: Fix ERANGE error when setting enum controls +- media: streamzap: prevent processing IR data on URB failure +- tpm, tpm_tis: Fix timeout handling when waiting for TPM status +- spi: cadence-qspi: Fix probe on AM62A LP SK +- xenfs/xensyms: respect hypervisor's "next" indication +- media: siano: Fix error handling in smsdvb_module_init() +- media: vim2m: print device name after registering device +- media: venus: hfi: add check to handle incorrect queue size +- media: venus: hfi: add a check to handle OOB in sfr region +- media: i2c: adv748x: Fix test pattern selection mask +- media: mtk-vcodec: venc: avoid -Wenum-compare-conditional warning +- media: mediatek: vcodec: Fix a resource leak related to the scp device in FW initialization +- dt-bindings: media: st,stmipid02: correct lane-polarities maxItems +- ext4: don't treat fhandle lookup of ea_inode as FS corruption +- bpf: support SKF_NET_OFF and SKF_LL_OFF on skb frags +- pwm: fsl-ftm: Handle clk_get_rate() returning 0 +- pwm: rcar: Improve register calculation +- pwm: mediatek: Prevent divide-by-zero in pwm_mediatek_config() +- tpm, tpm_tis: Workaround failed command reception on Infineon devices +- ktest: Fix Test Failures Due to Missing LOG_FILE Directories +- drm/amdgpu: grab an additional reference on the gang fence v2 +- drm/mediatek: mtk_dpi: Explicitly manage TVD clock in power on/off +- drm/mediatek: mtk_dpi: Move the input_2p_en bit to platform data +- drm/amdkfd: debugfs hang_hws skip GPU with MES +- drm/amdkfd: Fix pqm_destroy_queue race with GPU reset +- drm/amdkfd: clamp queue size to minimum +- drivers: base: devres: Allow to release group on device release +- drm/bridge: panel: forbid initializing a panel with unknown connector type +- drm: panel-orientation-quirks: Add quirk for OneXPlayer Mini (Intel) +- drm: panel-orientation-quirks: Add new quirk for GPD Win 2 +- drm: panel-orientation-quirks: Add quirk for AYA NEO Slide +- drm: panel-orientation-quirks: Add quirks for AYA NEO Flip DS and KB +- drm: panel-orientation-quirks: Add support for AYANEO 2S +- drm/amd/display: add workaround flag to link to force FFE preset +- drm/amd/display: Update Cursor request mode to the beginning prefetch always +- drm: allow encoder mode_set even when connectors change for crtc +- Bluetooth: qca: simplify WCN399x NVM loading +- Bluetooth: hci_uart: fix race during initialization +- tracing: fix return value in __ftrace_event_enable_disable for TRACE_REG_UNREGISTER +- net: vlan: don't propagate flags on open +- wifi: mt76: mt76x2u: add TP-Link TL-WDN6200 ID to device table +- cdc_ether|r8152: ThinkPad Hybrid USB-C/A Dock quirk +- Revert "f2fs: rebuild nat_bits during umount" +- ext4: protect ext4_release_dquot against freezing +- ahci: add PCI ID for Marvell 88SE9215 SATA Controller +- net: sfp: add quirk for 2.5G OEM BX SFP +- ata: libata-eh: Do not use ATAPI DMA for a device limited to PIO mode +- jfs: add sanity check for agwidth in dbMount +- fs/jfs: cast inactags to s64 to prevent potential overflow +- net: usb: asix_devices: add FiberGecko DeviceID +- scsi: target: spc: Fix RSOC parameter data header size +- f2fs: don't retry IO for corrupted data scenario +- wifi: ath12k: Fix invalid entry fetch in ath12k_dp_mon_srng_process +- ASoC: amd: yc: update quirk data for new Lenovo model +- ASoC: amd: Add DMI quirk for ACP6X mic support +- ALSA: usb-audio: Fix CME quirk for UF series keyboards +- mmc: dw_mmc: add a quirk for accessing 64-bit FIFOs in two halves +- media: uvcvideo: Add quirk for Actions UVC05 +- ASoC: fsl_audmix: register card device depends on 'dais' property +- ALSA: hda: intel: Add Lenovo IdeaPad Z570 to probe denylist +- ALSA: hda: intel: Fix Optimus when GPU has no sound +- HID: pidff: Add FIX_WHEEL_DIRECTION quirk +- HID: pidff: Add hid_pidff_init_with_quirks and export as GPL symbol +- HID: pidff: Add PERMISSIVE_CONTROL quirk +- HID: pidff: Add MISSING_PBO quirk and its detection +- HID: pidff: Add MISSING_DELAY quirk and its detection +- HID: pidff: Do not send effect envelope if it's empty +- HID: pidff: Convert infinite length from Linux API to PID standard +- ASoC: SOF: topology: Use krealloc_array() to replace krealloc() +- zstd: Increase DYNAMIC_BMI2 GCC version cutoff from 4.8 to 11.0 to work around compiler segfault +- xen/mcelog: Add __nonstring annotations for unterminated strings +- arm64: cputype: Add QCOM_CPU_PART_KRYO_3XX_GOLD +- x86/cpu: Don't clear X86_FEATURE_LAHF_LM flag in init_amd_k8() on AMD when running in a virtual machine +- x86/ia32: Leave NULL selector values 0~3 unchanged +- x86/mm: Clear _PAGE_DIRTY for kernel mappings when we clear _PAGE_RW +- pm: cpupower: bench: Prevent NULL dereference on malloc failure +- umount: Allow superblock owners to force umount +- iommu/mediatek: Fix NULL pointer deference in mtk_iommu_device_group +- nft_set_pipapo: fix incorrect avx2 match of 5th field octet +- nvmet-fcloop: swap list_add_tail arguments +- ata: sata_sx4: Add error handling in pdc20621_i2c_read() +- drm/tests: probe-helper: Fix drm_display_mode memory leak +- drm/tests: modes: Fix drm_display_mode memory leak +- drm/tests: cmdline: Fix drm_display_mode memory leak +- drm/tests: helpers: Create kunit helper to destroy a drm_display_mode +- drm/tests: helpers: Fix compiler warning +- drm/tests: helpers: Add helper for drm_display_mode_from_cea_vic() +- drm/tests: Add helper to create mock crtc +- drm/tests: Add helper to create mock plane +- drm/tests: helpers: Add atomic helpers +- drm/tests: modeset: Fix drm_display_mode memory leak +- net: ethtool: Don't call .cleanup_data when prepare_data fails +- tc: Ensure we have enough buffer space when sending filter netlink notifications +- net/sched: cls_api: conditional notification of events +- rtnl: add helper to check if a notification is needed +- rtnl: add helper to check if rtnl group has listeners +- octeontx2-pf: qos: fix VF root node parent queue index +- net: tls: explicitly disallow disconnect +- objtool: Fix INSN_CONTEXT_SWITCH handling in validate_unret() +- ata: pata_pxa: Fix potential NULL pointer dereference in pxa_ata_probe() +- drm/i915: Disable RPG during live selftest +- drm/i915/dg2: wait for HuC load completion before running selftests +- drm/i915/xelpg: Extend driver code of Xe_LPG to Xe_LPG+ +- drm/i915/mocs: use to_gt() instead of direct &i915->gt +- selftests/futex: futex_waitv wouldblock test should fail +- net/sched: Make cake_enqueue return NET_XMIT_CN when past buffer_limit +- jfs: upper bound check of tree index in dbAllocAG +- mm: swap: check for stable address space before operating on the VMA +- net: ethernet: cortina: Use TOE/TSO on all TCP +- mm/kmemleak: avoid deadlock by moving pr_warn() outside kmemleak_lock +- hfsplus: don't use BUG_ON() in hfsplus_create_attributes_file() +- bus: mhi: host: Detect events pointing to unexpected TREs +- drm/amd/pm: fix null pointer access +- ALSA: usb-audio: Validate UAC3 power domain descriptors, too +- net: kcm: Fix race condition in kcm_unattach() +- ALSA: hda/ca0132: Fix buffer overflow in add_tuning_control +- hfsplus: fix slab-out-of-bounds in hfsplus_bnode_read() +- mm/kmemleak: avoid soft lockup in __kmemleak_do_cleanup() +- hfsplus: fix slab-out-of-bounds read in hfsplus_uni2asc() +- xhci: fix memory leak regression when freeing xhci vdev devices depth first +- usb: xhci: Fix slot_id resource race conflict +- net: dsa: sja1105: fix kasan out-of-bounds warning in sja1105_table_delete_entry() +- nilfs2: reject invalid file types when reading inodes +- mtd: inftlcore: Add error check for inftl_read_oob() +- freezer, sched: Report frozen tasks as 'D' instead of 'R' +- io_uring/sqpoll: fix sqpoll error handling races +- i2c: microchip-core: actually use repeated sends +- PCI/MSI: Handle lack of irqdomain gracefully +- i2c: imx: add imx7d compatible string for applying erratum ERR007805 +- i2c: microchip-core: fix "ghost" detections + +* Fri Nov 21 2025 Liu Wang <1823363429@qq.com> - 6.6.0-120.0.0.98 +- Split kernel modules into kernel-extra-modules subpackage +- Prioritizes core kmod (networking/drm/block/modesetting) in main kernel package +- All extra modules and their dependencies are isolated to kernel-extra-modules +- Added module-filelist generation scripts for both packages +- Updated post-install scripts to handle extra modules dependency + +* Thu Nov 20 2025 Li Nan - 6.6.0-120.0.0.97 +- !18976 ubagg: implement ubagg module with ioctl interface and topology management +- !19139 ub: ubase: Fix CTRLQ init process +- !18998 uburma: implement jetty linking features. +- ub: ubase: Fix CTRLQ init process +- !19123 ub: udma: Support udma basic transaction operations and advanced features. +- !19122 crypto: rng - Ensure set_ent is always present +- !19121 ub: ubase: Do some bugfix about cmdq/ctrlq message and debugfs +- urma: add ubagg ioctl interface and topology management +- ub: udma: jetty and rct support the hugepage buffer. +- ub: udma: Support 2M hugepage function. +- ub: udma: Support query ub memory info. +- ub: udma: Support query resource ratio. +- ub: udma: Support query entity information. +- ub: udma: Support create and destroy stars jfc. +- ub: udma: Support create and destroy stars jfs. +- ub: udma: Support tp context inquiry. +- ub: udma: Support entity index inquiry. +- ub: udma: Support device status inquiry. +- ub: udma: Support flush jfs and jetty. +- ub: udma: Support destroy jfs and jetty batch. +- ub: udma: Support bind and unbind jetty. +- ub: udma: Support dump ce aux info. +- ub: udma: Support dump ae aux info. +- ub: udma: Support query NPU info. +- ub: udma: Support deactivate tp. +- ub: udma: Support active tp. +- ub: udma: Support get tp list. +- ub: udma: Support rearm jfc and clean jfc. +- ub: udma: Support poll jfc. +- ub: udma: Support post jetty work request. +- ub: udma: Support post jfr work request. +- ub: udma: Support post jfs work request. +- !19103 Fix CVE-2025-40123 +- !19082 ub:ubus: call ub_host_probe inside register_ub_manage_subsystem_ops +- uburma: implement comprehensive jetty linking and transport features +- uburma: implement jetty resource control and command structures +- uburma: add uburma command and user object framework headers +- uburma: introduce uburma basic types and helper functions +- !19120 KVM: arm64: Fix access_actlr() build warning +- !18952 ubcore: implement memory and netlink features. +- ub: ubase: Fix the lose of HIMAC reset. +- ub: ubase: flush the work queue. +- net: unic: Fix lose vport_ctx and vport_buf information query in debugfs +- !19091 ub: ub_fwctl: Release rpc_out when kernel state return error +- ub: ubase: Fix link status timestamp information +- net: unic: Fix configure coal parameters without deactivate +- ub: ubase: Fix CTRLQ white list +- ub: ubase: Optimization of ubase_ctrlq_send_msg interface parameters +- ub: ubase: Fix verification to ctrlq message seq +- net: unic: Fix the ethtool stats and basic capability information query interface +- ub: ubase: add CMDQ&CTRLQ compatibility code +- ub: ubase: Fix priqos infomaton interface functions to query and check sl_bitmap +- !19058 ub: udma: udma driver TA layer resource control operation. +- crypto: rng - Ensure set_ent is always present +- KVM: arm64: Fix access_actlr() build warning +- !19081 ub:ubus: Fix X86 build error +- ub: ub_fwctl: Release rpc_out when kernel state return error +- fwctl:Change the CONFIG_FWCTL in the config options to m +- selftests/bpf: Add test case for different expected_attach_type +- bpf: Enforce expected_attach_type for tailcall compatibility +- ubagg: implement ubagg basic infrastructure components +- !19093 some minor bugfix about zcopy +- !19087 v2 Fix bugs +- !19064 iommu/ummu: Bugfix of ummu +- zcopy: Remove the dump_pagetable interface +- zcopy: Fix the incorrect mapping size passed in attach_pte_range +- ub: udma: Add and remove jetty to jetty group. +- ub: udma: Support import and unimport jfr and jetty. +- iommu/ummu: Fix builds under different kconfigs +- xcall2.0: Fix stack-out-of-bounds bug in xcall_write() +- xcall2.0: Fix the semantic changes in xcall_enable +- xcall2.0: Fix module unload issue by fixing refcount +- !19071 net:nebula-matrix:fix compile err +- !19080 KVM: arm64: ipiv: Fix compilation warning +- !19070 xsched: XCU Partition +- !18995 CVE-2025-40169 +- !19028 mm/slab: make __free(kfree) accept error pointers +- ub:ubus: call ub_host_probe inside register_ub_manage_subsystem_ops +- ub:ubus: Fix X86 build error +- KVM: arm64: ipiv: Fix compilation warning +- !19069 mm: hugetlb: avoid soft lockup when mprotect to large memory area +- !19066 arch_numa: make node_reclaim_distance_setup static +- !19053 blk-ioinf: fix default weight being reset to 0 for non-root cgroups +- ubcore: add ubcore_umem and ubcore_tp implementations +- net:nebula-matrix:fix compile err +- xsched: Add support for CFS quota for cgroups +- xsched: Add XCU control group implementation and its backend in xsched CFS +- xsched: Add xsched CFS class +- xsched: Add xsched RT class +- xsched: Add basic scheduler core support +- xsched: Introduce vstream management +- xcu: Add base NPU driver support +- xsched: Add base vstream support +- mm: hugetlb: avoid soft lockup when mprotect to large memory area +- ubcore: add ubcore_umem and ubcore_tp implementations +- ubcore: implement ubcore connection management and messaging feature. +- ubcore: add segment management support to ubcore module. +- ubcore: add genl, netlink and vtp support to ubcore module. +- arch_numa: make node_reclaim_distance_setup static +- ub: udma: Support modify jfc. +- ub: udma: Support modify jfr. +- ub: udma: Support modify jetty. +- ub: udma: Support modify jfs. +- ub: udma: Support destroy jetty group. +- ub: udma: Support create jetty group. +- ub: udma: Support destroy jetty. +- ub: udma: Support create jetty. +- ub: udma: Support destroy jfc. +- ub: udma: Support create jfc. +- ub: udma: Support destroy jfr. +- ub: udma: Support create jfr. +- ub: udma: Support destroy jfs. +- ub: udma: Support create jfs. +- ub: udma: Support query table item from hw. +- ub: udma: Support query jfr context from hw. +- ub: udma: Support query jfs and jetty context from hw. +- !19043 EDAC/i10nm: Skip DIMM enumeration on a disabled memory controller +- iommu/ummu: Fix 2P virtualization error +- iommu/ummu: Fix ubmem unmap return value error +- !18348 Intel: Backport PMU support for Intel Clearwater Forest platform +- blk-ioinf: fix default weight being reset to 0 for non-root cgroups +- !18703 ub: udma: Support loading and unloading driver. +- !19031 v6 fuse:io_uring: support fuse over io_uring +- !19033 ub:ubus: Add ummu_map attribute in sysfs +- !18980 misc: uacce - fix a null pointer access issue when poweroff +- !18519 media: ivsc: Fix crash at shutdown due to missing mei_cldev_disable() calls +- !18518 iio: imu: bno055: fix OOB access of hw_xlate array +- !18515 s390/ism: fix concurrency management in ism_cmd() +- !18493 drm: bridge: anx7625: Fix NULL pointer dereference with early IRQ +- !18506 Fix CVE-2025-39781 +- !18505 Fix CVE-2025-38696 +- !18504 Fix CVE-2025-38640 +- !18503 Fix CVE-2025-39716 +- !18502 Fix CVE-2025-39767 +- !18501 Fix CVE-2025-39693 +- !18494 Fix CVE-2025-39909 +- !18497 Fix CVE-2025-39952 +- !18492 Fix CVE-2025-39916 +- !18957 [OLK-6.6] Fix CVE-2025-39846 +- !18956 [OLK-6.6] Fix CVE-2025-39842 +- !18955 [OLK-6.6] Fix CVE-2025-39861 +- !18954 [OLK-6.6] Fix CVE-2025-39794 +- !18953 [OLK-6.6] Fix CVE-2025-39863 +- !18633 [OLK-6.6] Fix CVE-2025-38715 +- !18631 [OLK-6.6]Fix CVE-2025-38698 +- !18215 [OLK-6.6] Fix CVE-2025-38576 +- !18398 btrfs: avoid NULL pointer dereference if no valid csum tree +- !18876 Fix CVE-2025-38011 +- !18388 fs/ntfs3: handle hdr_first_de() return value +- !18369 usb: renesas_usbhs: Reorder clock handling and power management in probe +- !18367 clk: bcm: rpi: Add NULL check in raspberrypi_clk_register() +- !18321 wifi: mt76: mt7915: Fix null-ptr-deref in mt7915_mmio_wed_init() +- !18310 net: usb: aqc111: debug info before sanitation +- !18309 coresight: fix indentation error in cscfg_remove_owned_csdev_configs() +- !18949 [OLK-6.6] Fix CVE-2025-38107 +- !18947 [OLK-6.6] Fix CVE-2025-38126 +- !18946 [OLK-6.6] Fix CVE-2025-38218 +- !18945 [OLK-6.6] Fix CVE-2025-38217 +- !18944 [OLK-6.6] Fix CVE-2025-38230 +- !18814 [OLK-6.6] Fix CVE-2025-38225 +- !18813 [OLK-6.6] Fix CVE-2025-38185 +- EDAC/i10nm: Skip DIMM enumeration on a disabled memory controller +- !18812 [OLK-6.6] Fix CVE-2025-38206 +- !18675 wifi: ath12k: Fix memory leak due to multiple rx_stats allocation +- !18432 [OLK-6.6] Fix CVE-2025-38259 +- !18244 eventpoll: don't decrement ep refcount while still holding the ep mutex +- !18245 wifi: p54: prevent buffer-overflow in p54_rx_eeprom_readback() +- !18243 f2fs: fix to do sanity check on ino and xnid +- !18242 jffs2: check jffs2_prealloc_raw_node_refs() result in few other places +- !18246 aoe: clean device rq_list in aoedev_downdev() +- ub: udma: Support init and uninit dfx function. +- ub: udma: Support register and unregister segment function. +- ub: udma: Support set eid function. +- ub: udma: Support register and unregister ctrlq event. +- ub: udma: Support register and unregister ce and crq event. +- ub: udma: Support register and unregister ae event. +- ub: udma: Register and unregister debugfs. +- ub: udma: Create and destroy u-context. +- ub:ubus: Add ummu_map attribute in sysfs +- openeuler_defconfig: enable CONFIG_FUSE_IO_URING by default +- fuse: remove unneeded atomic set in uring creation +- fuse: removed unused function fuse_uring_create() from header +- fuse: {io-uring} Fix a possible req cancellation race +- fuse: fix possible deadlock if rings are never initialized +- fuse: fix uring race condition for null dereference of fc +- fuse: prevent disabling io-uring on active connections +- fuse: enable fuse-over-io-uring +- fuse: block request allocation until io-uring init is complete +- fuse: {io-uring} Prevent mount point hang on fuse-server termination +- fuse: Allow to queue bg requests through io-uring +- fuse: Allow to queue fg requests through io-uring +- fuse: {io-uring} Make fuse_dev_queue_{interrupt,forget} non-static +- fuse: {io-uring} Handle teardown of ring entries +- fuse: Add io-uring sqe commit and fetch support +- fuse: {io-uring} Make hash-list req unique finding functions non-static +- fuse: Add fuse-io-uring handling into fuse_copy +- fuse: Make fuse_copy non static +- fuse: {io-uring} Handle SQEs - register commands +- fuse: make args->in_args[0] to be always the header +- fuse: Add fuse-io-uring design documentation +- fuse: Move request bits +- fuse: Move fuse_get_dev to header file +- fuse: rename to fuse_dev_end_requests and make non-static +- io_uring: fix kabi broken for header file changed +- io_uring: fix kabi broken in struct io_submit_state +- io_uring: fix kabi broken in struct io_ring_ctx +- io_uring: fix kabi broken in struct io_tw_state +- io_uring: kill dead code in io_req_complete_post +- io_uring: remove dead struct io_submit_state member +- io_uring/cmd: let cmds to know about dying task +- io_uring: clean up io_lockdep_assert_cq_locked +- io_uring: refactor io_req_complete_post() +- io_uring: remove current check from complete_post +- io_uring: get rid of intermediate aux cqe caches +- io_uring: refactor io_fill_cqe_req_aux +- io_uring: remove struct io_tw_state::locked +- io_uring: force tw ctx locking +- io_uring/cmd: document some uring_cmd related helpers +- io_uring/cmd: fix tw <-> issue_flags conversion +- io_uring/cmd: kill one issue_flags to tw conversion +- io_uring/cmd: move io_uring_try_cancel_uring_cmd() +- io_uring/cmd: give inline space in request to cmds +- io_uring/cmd: expose iowq to cmds +- io_uring/cmd: inline io_uring_cmd_do_in_task_lazy +- io_uring: split out cmd api into a separate header +- io_uring: cancelable uring_cmd +- io_uring: retain top 8bits of uring_cmd flags for kernel internal use +- !19006 ub:ubus: Bugfix of ubus and ubfi +- mm/slab: make __free(kfree) accept error pointers +- Revert "mm/slab: make __free(kfree) accept error pointers" +- !18969 arm64/mpam: Add quirk to shrink MATA PMG range +- ub:ubus: Bugfix of ubus and ubfi +- ub: udma: Support loading and unloading driver. +- !18859 KVM: arm64: Deactivate the spurious vtimer interrupt +- !19009 v2 mm: Fix shmem build and numa fault counting errors +- !19019 media: v4l2-subdev: Fix alloc failure check in v4l2_subdev_call_state_try() +- KVM: arm64: Deactivate the spurious vtimer interrupt +- !19026 accel/qaic: Treat remaining == 0 as error in find_and_map_user_pages() +- !19024 sparc: fix accurate exception reporting in copy_{from_to}_user for UltraSPARC +- !19027 v11 Add xcall 2.0 support +- xcall2.0: Introduce xcall epollwait prefetch feature +- openeuler_defconfig: Enable CONFIG_DYNAMIC_XCALL by default +- xcall2.0: Add a basic testcase +- xcall: Add debug mode to force fast syscall for all threads +- xcall: Rework the early exception vector of XCALL and SYNC +- xcall2.0: Support xcall1.0 for hardware xcall +- xcall2.0: Hajack syscall with dynamic instruciton replace +- xcall2.0: Add xcall_area +- xcall2.0: Add xcall module register interface +- xcall2.0: Add userspace proc interface +- accel/qaic: Treat remaining == 0 as error in find_and_map_user_pages() +- sparc: fix accurate exception reporting in copy_{from_to}_user for UltraSPARC +- !18988 zcopy: A zero-copy data transfer mechanism +- media: v4l2-subdev: Fix alloc failure check in v4l2_subdev_call_state_try() +- mm: fix nr_pages counting for folio in task_numa_fault +- mm: fix shmem build errors with CONFIG_SWAP disable +- !18970 net: unic: Support advanced functions of unic driver +- !18964 ub: add cdma driver code +- !18874 mm: introduce memalloc_flags_{save,restore} +- !18428 Add ccp-mdev feature into hygon ccp driver +- selftests/bpf: Add tests for rejection of ALU ops with negative offsets +- bpf: Reject negative offsets for ALU ops +- !18440 [OLK-6.6] pinctrl: Zhaoxin: Add KH-50000 support +- !18992 blk-ioinf: fix division by zero caused by inflight overflow in ioinf_qos_write() +- !18482 [OLK-6.6] ACPI: processor: idle: Set timer broadcast state on C3 for KH-50000 +- !18478 [OLK-6.6] hwmon: zhaoxin-cputemp: Update for KH-50000 +- !18477 [OLK-6.6] perf/zhaoxin/core: Tailor FMS for KH-50000 +- !18467 [OLK-6.6] Enhance SM3 SM4 drivers with optimizations and compatibility fixes +- !18450 [OLK-6.6] i2c: zhaoxin: Enhancements and Modifications to SMBus Driver +- !18449 [OLK-6.6] Optimize Zhaoxin CPU Temp Monitoring During Suspend/Resume +- !18447 RDMA/bnxt_re: Fix error recovery sequence +- !18448 bnxt_en: Fix ethtool -d byte order for 32-bit values +- !18452 bnxt_en: Fix DCB ETS validation +- !18961 mm: add huge pfnmap support for remap_pfn_range() +- blk-ioinf: fix division by zero caused by inflight overflow in ioinf_qos_write() +- !18950 urma: implement netlink and memory management features. +- !18930 ubcore: implement device registration and jetty management +- zcopy: enable ZCOPY module +- zcopy: Add ZCOPY documentation +- zcopy: Add debug inerface dump_pagetable +- zcopy: Add tracepoint for PageAttach +- zcopy: Extend PMD trans hugepage mapping ability +- zcopy: Introduce the pageattach interface +- zcopy: Initialize zcopy module +- ubcore: add genl, netlink and vtp support to ubcore module. +- !18960 KVM: SVM: Skip fastpath emulation on VM-Exit if next RIP isn't valid +- misc: uacce - fix a null pointer access issue when poweroff +- !18966 wakeup kswapd during node_reclaim +- !18857 Backport PLR (Power Limit Reasons) TPMI driver for 6.6 +- !17988 [OLK-6.6] Fix CVE-2025-38734 +- !18005 [OLK-6.6] Fix CVE-2025-38718 +- !18977 mm: migrate: Fix the compilation error caused by redundant semicolons +- mm: migrate: Fix the compilation error caused by redundant semicolons +- !18943 smb: client: validate change notify buffer before copy +- !18942 rtc: rx8025: fix incorrect register reference +- !18051 [OLK-6.6] [Bugfix] Bug fix for min_free_kbytes warning storms +- !18040 [OLK-6.6] Fix CVE-2025-39808 +- ub: cdma: support for cdma kernelspace north-south compatibility requirements +- ub: cdma: support reset function +- ub: cdma: support RX stop flow function +- ub: cdma: support debugfs interface +- ub: cdma: support dma faa semantic +- ub: cdma: support dma cas semantic +- ub: cdma: support dma read semantic +- ub: cdma: support dma write with notify semantic +- ub: cdma: support dma write semantic delivery +- ub: cdma: support dma write semantic configuration +- ub: cdma: support kernel resource reclamation +- !18892 CCA: Fix soft lockup bug while destroy realm +- ubcore: implement support for UVS commands +- ubcore: implement device registration and jetty management +- !18968 kvm: supports hugepages POD when migrate vm +- ubcore: add device management for character device and class operations +- kvm: supports hugepages POD when migrate vm +- net: unic: Debugfs supports querying IP specifications and IP entries. +- net: unic: support subscribes to the RX stream stop and recovery interface. +- net: unic: support config/query traffic class parameters +- net: unic: support config/query ets parameters +- net: unic: Add debugfs support for QoS configuration and query. +- net: unic: support config/query the mapping between dscp and tc +- net: unic: Support RAS +- net: unic: Drive supports ub entity reset. +- net: unic: Support to query and clear historical NIC link status information +- net: unic: support querying and configuring coalesce parameters. +- net: unic: Add debugfs for JFS/JFR/JFC context. +- net: unic: Support driver dump register. +- arm64/mpam: Merge redundant local static variables 'cpus' +- arm64/mpam: Add quirk to shrink MATA PMG range +- !18963 kvm: supports hugepages reclaime notify +- mm: node_reclaim: add wakeup kswapd mode +- mm: vmscan: wakeup kswapd during node_reclaim +- arch_numa: Make node reclaim distance adjustment always available +- kvm: supports hugepages reclaime notify +- !18940 [OLK-6.6][linux-6.6.y sync] Backport 6.6.86-6.6.87 LTS Patches +- arm64/x86: configs: Enable huge pfnmap support in openeuler defconfig +- mm: introduce remap_pfn_range_try_pmd() for PMD-level hugepage mapping +- pgtable: add pte_clrhuge() implementation for arm64 +- arm64: mm: Drop dead code for pud special bit handling +- mm/huge_memory: check pmd_special() only after pmd_present() +- mm/arm64: support large pfn mappings +- mm/x86: support large pfn mappings +- mm: always define pxx_pgprot() +- mm/fork: accept huge pfnmap entries +- mm/pagewalk: check pfnmap for folio_walk_start() +- mm/gup: detect huge pfnmap entries in gup-fast +- mm: allow THP orders for PFNMAPs +- mm: mark special bits for huge pfn mappings when inject +- mm: drop is_huge_zero_pud() +- mm: introduce ARCH_SUPPORTS_HUGE_PFNMAP and special bits to pmd/pud +- !18841 mm: add auxillary functions for folios and hugetlb +- !18896 kvm: add struct kvm if only CONFIG_KVM enabled +- KVM: SVM: Skip fastpath emulation on VM-Exit if next RIP isn't valid +- mm: hugetlb: add putback_hugetlb_folio() +- mm: hugetlb: add get_hugetlb_folio_nodemask() mm: hugetlb: add filter function to dequeue hugetlb functions +- mm: migrate: add isolate_and_migrate_folios() +- kvm: add struct kvm if only CONFIG_KVM enabled +- !18908 [Backport] mm/hugetlb: declare hugetlbfs_pagecache_present() non-static +- !18376 net: wangxun: ngbe: add support for wangxun 1G +- pcmcia: Fix a NULL pointer dereference in __iodyn_find_io_region() +- ocfs2: prevent release journal inode after journal shutdown +- Bluetooth: vhci: Prevent use-after-free by removing debugfs files early +- ARM: tegra: Use I/O memcpy to write to IRAM +- wifi: brcmfmac: fix use-after-free when rescheduling brcmf_btcoex_info work +- !18887 urma: implement ubcore sysfs interface and device management +- net_sched: ets: fix a race in ets_qdisc_change() +- [Backport] mm/hugetlb: declare hugetlbfs_pagecache_present() non-static +- net: stmmac: make sure that ptp_rate is not 0 before configuring timestamping +- f2fs: fix to do sanity check on sit_bitmap_size +- hwmon: (ftsteutates) Fix TOCTOU race in fts_read() +- jfs: validate AG parameters in dbMount() to prevent crashes +- fs/jfs: consolidate sanity checking in dbMount +- jfs: add sanity check for agwidth in dbMount +- perf: Fix dangling cgroup pointer in cpuctx +- nfsd: decrease sc_count directly if fail to queue dl_recall +- tracing: Do not add length to print format in synthetic events +- ksmbd: fix memory leak in parse_lease_state() +- ipv6: Align behavior across nexthops during path selection +- gre: Fix IPv6 multicast route creation. +- gre: Fix again IPv6 link-local address generation. +- Revert "drm/amd: Keep display off while going into S4" +- Revert "rndis_host: Flag RNDIS modems as WWAN devices" +- Revert "PCI: Avoid reset when disabled via sysfs" +- PM: sleep: Fix power.is_suspended cleanup for direct-complete devices +- tracing: Do not use PERF enums when perf is not defined +- NFSD: Skip sending CB_RECALL_ANY when the backchannel isn't up +- nfsd: put dl_stid if fail to queue dl_recall +- exec: fix the racy usage of fs_struct->in_exec +- jfs: add index corruption check to DT_GETPAGE() +- ext4: fix OOB read when checking dotdot dir +- ext4: don't over-report free space or inodes in statvfs +- arm64: Don't call NULL in do_compat_alignment_fixup() +- tracing: Fix synth event printk format for str fields +- tracing: Ensure module defining synth event cannot be unloaded while tracing +- ksmbd: add bounds check for create lease context +- ksmbd: add bounds check for durable handle context +- mmc: sdhci-omap: Disable MMC_CAP_AGGRESSIVE_PM for eMMC/SD +- mmc: sdhci-pxav3: set NEED_RSP_BUSY capability +- mmc: omap: Fix memory leak in mmc_omap_new_slot +- ACPI: resource: Skip IRQ override on ASUS Vivobook 14 X1404VAP +- x86/tsc: Always save/restore TSC sched_clock() on suspend/resume +- kunit/overflow: Fix UB in overflow_allocation_test +- ntb_perf: Delete duplicate dmaengine_unmap_put() call in perf_copy_chunk() +- platform/x86: ISST: Correct command storage data length +- ACPI: x86: Extend Lenovo Yoga Tab 3 quirk with skip GPIO event-handlers +- x86/hyperv: Fix check of return value from snp_set_vmsa() +- LoongArch: BPF: Use move_addr() for BPF_PSEUDO_FUNC +- media: imx-jpeg: Cleanup after an allocation error +- media: imx-jpeg: Move mxc_jpeg_free_slot_data() ahead +- smb: client: validate change notify buffer before copy +- rtc: rx8025: fix incorrect register reference +- perf/core: Fix child_total_time_enabled accounting bug at task exit +- drm/amdgpu/gfx11: fix num_mec +- ipv6: Do not consider link down nexthops in path selection +- ipv6: Start path selection from the first nexthop +- net: dsa: mv88e6xxx: propperly shutdown PPU re-enable timer on destroy +- ipv6: fix omitted netlink attributes when using RTEXT_FILTER_SKIP_STATS +- tunnels: Accept PACKET_HOST in skb_tunnel_check_pmtu(). +- vsock: avoid timeout during connect() if the socket is closing +- netfilter: nft_set_hash: GC reaps elements with conncount for dynamic sets only +- e1000e: change k1 configuration on MTP and later platforms +- riscv: Fix hugetlb retrieval of number of ptes in case of !present pte +- ASoC: codecs: rt5665: Fix some error handling paths in rt5665_probe() +- x86/uaccess: Improve performance by aligning writes to 8 bytes in copy_user_generic(), on non-FSRM/ERMS CPUs +- RISC-V: errata: Use medany for relocatable builds +- ALSA: hda/realtek: Fix built-in mic breakage on ASUS VivoBook X515JA +- ntb: intel: Fix using link status DB's +- riscv: ftrace: Add parentheses in macro definitions of make_call_t0 and make_call_ra +- spufs: fix a leak in spufs_create_context() +- hwmon: (nct6775-core) Fix out of bounds access for NCT679{8,9} +- memory: omap-gpmc: drop no compatible check +- can: statistics: use atomic access in hot path +- ALSA: hda/realtek: Add mute LED quirk for HP Pavilion x360 14-dy1xxx +- drm/amd: Keep display off while going into S4 +- nvme-pci: fix stuck reset on concurrent DPC and HP +- x86/sgx: Warn explicitly if X86_FEATURE_SGX_LC is not enabled +- x86/hyperv: Fix output argument to hypercall that changes page visibility +- locking/semaphore: Use wake_q to wake up processes outside lock critical section +- wifi: mac80211: flush the station before moving it to UN-AUTHORIZED state +- x86/hyperv/vtl: Stop kernel from probing VTL0 low memory +- sched/deadline: Use online cpus for validating runtime +- ALSA: hda/realtek: Add support for ASUS Zenbook UM3406KA Laptops using CS35L41 HDA +- HID: i2c-hid: improve i2c_hid_get_report error message +- platform/x86/intel/vsec: Add Diamond Rapids support +- platform/x86: intel-hid: fix volume buttons on Microsoft Surface Go 4 tablet +- cifs: fix incorrect validation for num_aces field of smb_acl +- perf/core: Fix perf_pmu_register() vs. perf_init_event() +- ALSA: hda: Fix speakers on ASUS EXPERTBOOK P5405CSA 1.0 +- ALSA: hda/realtek: Fix Asus Z13 2025 audio +- affs: don't write overlarge OFS data block size fields +- affs: generate OFS sequence numbers starting at 1 +- wifi: brcmfmac: keep power during suspend if board requires it +- nvme-pci: skip CMB blocks incompatible with PCI P2P DMA +- nvme-pci: clean up CMBMSC when registering CMB fails +- wifi: iwlwifi: mvm: use the right version of the rate API +- wifi: iwlwifi: fw: allocate chained SG tables for dump +- rcu-tasks: Always inline rcu_irq_work_resched() +- context_tracking: Always inline ct_{nmi,irq}_{enter,exit}() +- sched/smt: Always inline sched_smt_active() +- objtool: Fix verbose disassembly if CROSS_COMPILE isn't set +- octeontx2-af: Free NIX_AF_INT_VEC_GEN irq +- octeontx2-af: Fix mbox INTR handler when num VFs > 64 +- ACPI: processor: idle: Return an error if both P_LVL{2,3} idle states are invalid +- LoongArch: Rework the arch_kgdb_breakpoint() implementation +- LoongArch: Fix help text of CMDLINE_EXTEND in Kconfig +- objtool: Fix segfault in ignore_unreachable_insn() +- ring-buffer: Fix bytes_dropped calculation issue +- net/mlx5e: SHAMPO, Make reserved size independent of page size +- ksmbd: fix multichannel connection failure +- ksmbd: use aead_request_free to match aead_request_alloc +- rndis_host: Flag RNDIS modems as WWAN devices +- exfat: fix the infinite loop in exfat_find_last_cluster() +- NFS: Shut down the nfs_client only after all the superblocks +- perf tools: annotate asm_pure_loop.S +- fs/procfs: fix the comment above proc_pid_wchan() +- tty: n_tty: use uint for space returned by tty_write_room() +- staging: rtl8723bs: select CONFIG_CRYPTO_LIB_AES +- perf python: Check if there is space to copy all the event +- perf python: Don't keep a raw_data pointer to consumed ring buffer space +- perf python: Decrement the refcount of just created event on failure +- perf python: Fixup description of sample.id event member +- i3c: master: svc: Fix missing the IBI rules +- um: hostfs: avoid issues on inode number reuse by host +- um: remove copy_from_kernel_nofault_allowed +- selftests/mm/cow: fix the incorrect error handling +- fuse: fix dax truncate/punch_hole fault path +- NFSv4: Don't trigger uneccessary scans for return-on-close delegations +- arch/powerpc: drop GENERIC_PTDUMP from mpc885_ads_defconfig +- kexec: initialize ELF lowest address to ULONG_MAX +- kernel/events/uprobes: handle device-exclusive entries correctly in __replace_page() +- perf units: Fix insufficient array space +- perf evlist: Add success path to evlist__create_syswide_maps +- iio: adc: ad7124: Fix comparison of channel configs +- iio: adc: ad4130: Fix comparison of channel setups +- dmaengine: fsl-edma: cleanup chan after dma_async_device_unregister +- fs/ntfs3: Prevent integer overflow in hdr_first_de() +- usb: xhci: correct debug message page size calculation +- perf bench: Fix perf bench syscall loop count +- perf arm-spe: Fix load-store operation checking +- iio: accel: msa311: Fix failure to release runtime pm if direct mode claim fails. +- iio: accel: mma8452: Ensure error return on failure to matching oversampling ratio +- ucsi_ccg: Don't show failed to get FW build information error +- perf pmu: Don't double count common sysfs and json events +- coresight-etm4x: add isb() before reading the TRCSTATR +- vhost-scsi: Fix handling of multiple calls to vhost_scsi_set_endpoint +- coresight: catu: Fix number of pages while using 64k pages +- soundwire: slave: fix an OF node reference leak in soundwire slave device +- isofs: fix KMSAN uninit-value bug in do_isofs_readdir() +- perf stat: Fix find_stat for mixed legacy/non-legacy events +- clk: qcom: mmcc-sdm660: fix stuck video_subcore0 clock +- pinctrl: intel: Fix wrong bypass assignment in intel_pinctrl_probe_pwm() +- leds: Fix LED_OFF brightness race +- mfd: sm501: Switch to BIT() to mitigate integer overflows +- pinctrl: renesas: rzv2m: Fix missing of_node_put() call +- crypto: nx - Fix uninitialised hv_nxc on error +- power: supply: max77693: Fix wrong conversion of charge input threshold value +- x86/entry: Fix ORC unwinder for PUSH_REGS with save_ret=1 +- clk: amlogic: g12a: fix mmc A peripheral clock +- clk: clk-imx8mp-audiomix: fix dsp/ocram_a clock parents +- selftests/bpf: Select NUMA_NO_NODE to create map +- clk: amlogic: gxbb: drop non existing 32k clock parent +- clk: amlogic: g12b: fix cluster A parent data +- pinctrl: tegra: Set SFIO mode to Mux Register +- IB/mad: Check available slots before posting receive WRs +- remoteproc: qcom_q6v5_mss: Handle platforms with one power domain +- RDMA/erdma: Prevent use-after-free in erdma_accept_newconn() +- RDMA/mlx5: Fix calculation of total invalidated pages +- clk: rockchip: rk3328: fix wrong clk_ref_usb3otg parent +- pinctrl: renesas: rzg2l: Fix missing of_node_put() call +- pinctrl: renesas: rza2: Fix missing of_node_put() call +- lib: 842: Improve error handling in sw842_compress() +- remoteproc: qcom_q6v5_pas: Use resource with CX PD for MSM8226 +- RDMA/mana_ib: Ensure variable err is initialized +- clk: qcom: gcc-msm8953: fix stuck venus0_core0 clock +- clk: samsung: Fix UBSAN panic in samsung_clk_init() +- selftests/bpf: Fix string read in strncmp benchmark +- libbpf: Fix hypothetical STT_SECTION extern NULL deref case +- remoteproc: qcom_q6v5_pas: Make single-PD handling more robust +- remoteproc: core: Clear table_sz when rproc_shutdown +- clk: amlogic: gxbb: drop incorrect flag on 32k clock +- fbdev: sm501fb: Add some geometry checks. +- mdacon: rework dependency list +- fbdev: au1100fb: Move a variable assignment behind a null pointer check +- PCI: pciehp: Don't enable HPIE when resuming in poll mode +- PCI: histb: Fix an error handling path in histb_pcie_probe() +- drm/amd/display: avoid NPD when ASIC does not support DMUB +- drm/mediatek: dsi: fix error codes in mtk_dsi_host_transfer() +- PCI: xilinx-cpm: Fix IRQ domain leak in error path of probe +- PCI: Remove stray put_device() in pci_register_host_bridge() +- drm/amd/display: fix type mismatch in CalculateDynamicMetadataParameters() +- PCI: Avoid reset when disabled via sysfs +- PCI/portdrv: Only disable pciehp interrupts early when needed +- PCI: brcmstb: Fix potential premature regulator disabling +- PCI: brcmstb: Use internal register to change link capability +- PCI: cadence-ep: Fix the driver to send MSG TLP for INTx without data payload +- drm/amdkfd: Fix Circular Locking Dependency in 'svm_range_cpu_invalidate_pagetables' +- drm/msm/dsi: Set PHY usescase (and mode) before registering DSI host +- drm/msm/dsi: Use existing per-interface slice count in DSC timing +- drm/msm/dpu: don't use active in atomic_check() +- drm/mediatek: mtk_hdmi: Fix typo for aud_sampe_size member +- drm/mediatek: mtk_hdmi: Unregister audio platform device on failure +- PCI: Use downstream bridges for distributing resources +- drm: xlnx: zynqmp: Fix max dma segment size +- drm/bridge: it6505: fix HDCP V match check is not performed correctly +- drm/dp_mst: Fix drm RAD print +- drm/bridge: ti-sn65dsi86: Fix multiple instances +- ASoC: ti: j721e-evm: Fix clock configuration for ti,j7200-cpb-audio compatible +- ALSA: hda/realtek: Always honor no_shutup_pins +- dt-bindings: vendor-prefixes: add GOcontroll +- HID: remove superfluous (and wrong) Makefile entry for CONFIG_INTEL_ISH_FIRMWARE_DOWNLOADER +- ASoC: cs35l41: check the return value from spi_setup() +- platform/x86: dell-ddv: Fix temperature calculation +- media: platform: allgro-dvt: unregister v4l2_device on the error path +- media: verisilicon: HEVC: Initialize start_bit field +- x86/fpu/xstate: Fix inconsistencies in guest FPU xfeatures +- x86/traps: Make exc_double_fault() consistently noreturn +- perf/ring_buffer: Allow the EPOLLRDNORM flag for poll +- lockdep: Don't disable interrupts on RT in disable_irq_nosync_lockdep.*() +- PM: sleep: Fix handling devices with direct_complete set on errors +- EDAC/ie31200: Fix the error path order of ie31200_init() +- EDAC/ie31200: Fix the DIMM size mask for several SoCs +- EDAC/ie31200: Fix the size of EDAC_MC_LAYER_CHIP_SELECT layer +- selinux: Chain up tool resolving errors in install_policy.sh +- RISC-V: KVM: Disable the kernel perf counter during configure +- PM: sleep: Adjust check before setting power.must_resume +- lockdep/mm: Fix might_fault() lockdep check of current->mm->mmap_lock +- x86/sev: Add missing RIP_REL_REF() invocations during sme_enable() +- x86/platform: Only allow CONFIG_EISA for 32-bit +- x86/fpu: Avoid copying dynamic FP state from init_task in arch_dup_task_struct() +- x86/fpu: Fix guest FPU state buffer allocation size +- smack: dont compile ipv6 code unless ipv6 is configured +- cpufreq: scpi: compare kHz instead of Hz +- x86/mm/pat: cpa-test: fix length for CPA_ARRAY test +- watch_queue: fix pipe accounting mismatch +- urma: implement ubcore sysfs interface and device management +- mm: migrate: add isolate_folio_to_list() +- urma: implement ubcore basic connection and bonding support +- urma: add network framework facilities for ubcore. +- urma: add tp table support and related cmd header for ubcore. +- CCA: unmap state2 range can block to fix soft lockup while destroy realm +- config: enable config for PLR +- platform/x86/intel: power-domains: Use topology_logical_package_id() for package ID +- platform/x86/intel: power-domains: Add Clearwater Forest support +- platform/x86/intel/tpmi/plr: Fix output in plr_print_bits() +- platform/x86/intel/tpmi/plr: Add support for the plr mailbox +- platform/x86/intel/tpmi: Add new auxiliary driver for performance limits +- platform/x86/intel: TPMI domain id and CPU mapping +- platform/x86/intel/tpmi: Add API to get debugfs root +- platform/x86/intel/tpmi: Add support for performance limit reasons +- batman-adv: Switch to linux/array_size.h +- drm/amdgpu: csa unmap use uninterruptible lock +- mm/damon/reclaim: avoid divide-by-zero in damon_reclaim_apply_parameters() +- mm/damon/lru_sort: avoid divide-by-zero in damon_lru_sort_apply_parameters() +- ASoC: codecs: wcd9335: Fix missing free of regulator supplies +- ASoC: codec: wcd9335: Convert to GPIO descriptors +- ASoC: codecs: wcd9335: Handle nicer probe deferral and simplify with dev_err_probe() +- parisc: Drop WARN_ON_ONCE() from flush_cache_vmap +- mm: introduce memalloc_flags_{save,restore} +- MIPS: Don't crash in stack_top() for tasks without ABI or vDSO +- bpf: Disable migration in nf_hook_run_bpf(). +- parisc: Revise __get_user() to probe user read access +- LoongArch: Optimize module load time by optimizing PLT/GOT counting +- drm/amd/display: Avoid a NULL pointer dereference +- wifi: wilc1000: avoid buffer overflow in WID string configuration +- eventpoll: don't decrement ep refcount while still holding the ep mutex +- wifi: p54: prevent buffer-overflow in p54_rx_eeprom_readback() +- f2fs: fix to do sanity check on ino and xnid +- jffs2: check jffs2_prealloc_raw_node_refs() result in few other places +- aoe: clean device rq_list in aoedev_downdev() +- kernel.h: Move ARRAY_SIZE() to a separate header +- atm: atmtcp: Free invalid length skb in atmtcp_c_send(). +- exfat: fix double free in delayed_free +- configs: Add Zhaoxin GPIO pinctrl for kh-50000 in openeuler_config +- pinctrl: Zhaoxin: Add KH-50000 support +- wifi: ath12k: Fix memory leak due to multiple rx_stats allocation +- media: ivsc: Fix crash at shutdown due to missing mei_cldev_disable() calls +- iio: imu: bno055: fix OOB access of hw_xlate array +- s390/ism: fix concurrency management in ism_cmd() +- drm: bridge: anx7625: Fix NULL pointer dereference with early IRQ +- btrfs: avoid NULL pointer dereference if no valid csum tree +- fs/ntfs3: handle hdr_first_de() return value +- clk: bcm: rpi: Add NULL check in raspberrypi_clk_register() +- wifi: mt76: mt7915: Fix null-ptr-deref in mt7915_mmio_wed_init() +- net: usb: aqc111: debug info before sanitation +- net: usb: aqc111: fix error handling of usbnet read calls +- coresight: fix indentation error in cscfg_remove_owned_csdev_configs() +- coresight: holding cscfg_csdev_lock while removing cscfg from csdev +- coresight: change coresight_device lock type to raw_spinlock_t +- hfs: fix slab-out-of-bounds in hfs_bnode_read() +- jfs: Regular file corruption check +- net: wangxun: ngbe: add support for wangxun 1G +- RDMA/bnxt_re: Fix error recovery sequence +- bnxt_en: Fix ethtool -d byte order for 32-bit values +- bnxt_en: Fix DCB ETS validation +- Fix kabi for CWF PMU support +- crypto: sm4 - Refactor mode handling and optimize performance +- i2c: zhaoxin: Enhancements and Modifications to SMBus Driver +- ACPI: processor: idle: Set timer broadcast state on C3 for KH-50000 +- hwmon: zhaoxin-cputemp: Update for KH-50000 +- perf/zhaoxin/core: Tailor FMS for KH-50000 +- crypto: sm3 - Refactor code and harden security +- perf mem: Fix printing PERF_MEM_LVLNUM_{L2_MHB|MSC} +- perf/x86: Print PMU counters bitmap in x86_pmu_show_pmu_cap() +- perf/x86/intel: Add ICL_FIXED_0_ADAPTIVE bit into INTEL_FIXED_BITS_MASK +- KVM: x86/pmu: Manipulate FIXED_CTR_CTRL MSR with macros +- KVM: x86/pmu: Change ambiguous _mask suffix to _rsvd in kvm_pmu +- KVM: x86/pmu: Add common define to capture fixed counters offset +- perf/x86/intel: Fix IA32_PMC_x_CFG_B MSRs access error +- tools arch x86: Sync the msr-index.h copy with the kernel sources +- tools/include: Sync x86 headers with the kernel sources +- perf/x86/intel: Fix incorrect MSR index calculations in intel_pmu_config_acr() +- perf/x86/intel: Fix bitmask of OCR and FRONTEND events for LNC +- perf/x86/intel: Fix event constraints for LNC +- perf/x86/intel/ds: Fix counter backwards of non-precise events counters-snapshotting +- perf/x86/intel: Fix segfault with PEBS-via-PT with sample_freq +- perf vendor events: Add Clearwaterforest events +- perf/x86/intel/uncore: Support customized MMIO map size +- perf/x86/intel/uncore: Support MSR portal for discovery tables +- perf/x86/intel/uncore: Add Clearwater Forest support +- perf/uapi: Clean up a bit +- tools headers: Update the uapi/linux/perf_event.h copy with the kernel sources +- tools/include: Sync uapi/linux/perf.h with the kernel sources +- perf/x86: Optimize the is_x86_event +- perf/x86/intel: Check the X86 leader for ACR group +- perf/x86/intel: Check the X86 leader for pebs_counter_event_group +- perf/x86/intel: Parse CPUID archPerfmonExt leaves for non-hybrid CPUs +- perf/x86/intel: Add PMU support for Clearwater Forest +- perf/x86/intel: Don't clear perf metrics overflow bit unconditionally +- perf/x86/intel: Support auto counter reload +- perf/x86/intel: Add CPUID enumeration for the auto counter reload +- perf: Extend the bit width of the arch-specific flag +- perf/x86/intel: Track the num of events needs late setup +- perf/x86: Add dynamic constraint +- perf: Extend per event callchain limit to branch stack +- perf/x86/intel: Fix ARCH_PERFMON_NUM_COUNTER_LEAF +- perf/x86/intel: Support PEBS counters snapshotting +- Optimize Zhaoxin CPU Temp Monitoring During Suspend/Resume +- Optimize VIA CPU Temp Monitoring During Suspend/Resume +- perf/x86/intel: Support RDPMC metrics clear mode +- perf/x86/intel/ds: Add PEBS format 6 +- perf/x86/intel/ds: Simplify the PEBS records processing for adaptive PEBS +- perf/x86/intel/ds: Factor out functions for PEBS records processing +- perf/x86/intel/ds: Clarify adaptive PEBS processing +- perf/x86/intel: Support Perfmon MSRs aliasing +- perf/x86/intel: Support PERFEVTSEL extension +- perf/x86: Add config_mask to represent EVENTSEL bitmask +- perf/x86/intel: Support new data source for Lunar Lake +- perf/x86/intel: Rename model-specific pebs_latency_data functions +- perf/x86: Add Lunar Lake and Arrow Lake support +- perf/x86: Support counter mask +- crypto: ccp: add ccp-mdev functionality to the ccp module. +- usb: renesas_usbhs: Reorder clock handling and power management in probe +- perf/x86/intel: Support the PEBS event mask +- tools headers UAPI: Sync include/uapi/linux/perf_event.h header with the kernel +- perf/x86/intel/uncore: Support more units on Granite Rapids +- perf/x86/intel/uncore: Clean up func_id +- powerpc/eeh: Make EEH driver device hotplug safe +- mm: Use pr_warn_once() for min_free_kbytes warning +- HID: hid-ntrig: fix unable to handle page fault in ntrig_report_version() +- sctp: linearize cloned gso packets in sctp_rcv +- net/smc: fix UAF on smcsk after smc_listen_out() + +* Thu Nov 13 2025 Li Nan - 6.6.0-118.0.0.96 +- !18685 ub: add cdma driver code +- !18684 ub: ub_fwctl:Add the ub_fwctl driver +- !18928 gmem: fix some warnings +- !18938 dma-mapping: benchmark: add support for UB devices +- ub: cdma: support register segment +- ub: cdma: support unregister segment +- ub: cdma: support reporting completed events +- ub: cdma: support reporting asynchronous events +- ub: cdma: support the creation of jfs +- ub: cdma: support the deletion of jfs +- ub: cdma: support the creation of ctp +- ub: cdma: support the deletion of ctp +- ub: cdma: support the creation of jfc +- ub: cdma: support the deletion of jfc +- ub: cdma: support for releasing queue +- ub: cdma: support for allocating queue +- ub: cdma: support for getting device by eid +- ub: cdma: support for getting device list +- ub: cdma: support querying sl information and updating eu +- ub: cdma: support the probe and remove processes +- !18906 v2 Add MFS Support +- !18920 perf: Fix tt_core mismatch over PG CPU +- !18934 mm/thp: Add a new sysfs knob for zero COW +- !18845 fs/resctrl: Fix the L3 CMAX conversion mechanism +- !18858 ub: ubase: Support debugs functions +- !18853 net: unic: add support of unic driver for ub network +- !18901 ub:hisi-ubus: Support HISI-UBUS driver framework +- !18927 mm: add nr_free_highatomic in show_free_areas +- !18929 v2:mm/swap: get the swap device offset directly +- ub: ub_fwctl: support ummu data processing +- ub: ub_fwctl: query the MSG queue information and entry details within UB. +- ub: ub_fwctl: support querying SCC and io_die related information. +- ub: ub_fwctl: support querying UB link trace information. +- ub: ub_fwctl: Support Dump register. +- ub: ub_fwctl: supports querying and configuring some scattered registers. +- ub: ub_fwctl: supports querying TP, BA related register information +- dma-mapping: benchmark: add support for UB devices +- ub: ub_fwctl: supports querying NL, TA, DL related register information +- ub: ub_fwctl: Add some simple common framework functions +- ub: ub_fwctl: Add the ub_fwctl driver and its basic features. +- mm/thp: Add a new sysfs knob for zero COW +- !18803 arm64/mpam: Add quirk to retrigger L3 CSU monitor +- net: unic: support querying and configuring queue parameters. +- net: unic: support querying and configuring the number of queues. +- net: unic: support querying and configuring simple RSS. +- net: unic: support querying and configuring promisc. +- net: unic: support querying and configuring MTU. +- net: unic: support querying and configuring FEC parameters. +- net: unic: add fec statistic support +- net: unic: add rack ip for unic +- net: unic: add io basic Rx/Tx functionality for unic +- net: unic: add support of unic driver for ub network +- !18914 misc: pci_endpoint_test: Avoid issue of interrupts remaining after request_irq error +- !18915 scsi: ufs: mcq: Add NULL check in ufshcd_mcq_abort() +- !18916 software node: Correct a OOB check in software_node_get_reference_args() +- !18917 KVM: x86: use array_index_nospec with indices that come from guest +- !18877 iommu/ummu: Implement IOMMU ops and advanced capabilities. +- mm/swap: get the swap device offset directly +- mm: add nr_free_highatomic in show_free_areas +- ub:ubus: reserve space for ub subsystem related structure +- ub:hisi-ubus: Support UBUS configure EID-UPI table +- ub:hisi-ubus: Support UBUS vdm entity enable message +- ub:hisi-ubus: Support for MSGQ's debugging functionality +- !17926 spufs: fix a leak on spufs_new_file() failure +- !18393 platform/x86: dell-wmi-sysman: Avoid buffer overflow in current_password_store() +- !18389 regulator: max20086: Fix refcount leak in max20086_parse_regulators_dt() +- !18381 f2fs: zone: fix to avoid inconsistence in between SIT and SSA +- !18370 dmaengine: ti: Add NULL check in udma_probe() +- !18674 jfs: fix array-index-out-of-bounds read in add_missing_indices +- !18438 jfs: Fix null-ptr-deref in jfs_ioc_trim +- !18435 ksmbd: fix null pointer dereference in destroy_previous_session +- !18434 tty: serial: uartlite: register uart driver in init +- !18404 mm/damon/sysfs-schemes: free old damon_sysfs_scheme_filter->memcg_path on write +- !18799 [OLK-6.6] Fix CVE-2025-38336 +- !18798 [OLK-6.6] Fix CVE-2025-38335 +- !18793 [OLK-6.6] Fix CVE-2025-38275 +- !18789 [OLK-6.6] Fix CVE-2025-38274 +- !18786 [OLK-6.6] Fix CVE-2025-38300 +- !18781 [OLK-6.6] Fix CVE-2025-38295 +- !18778 [OLK-6.6] Fix CVE-2025-38293 +- !18777 [OLK-6.6] Fix CVE-2025-38292 +- !18849 urma: introduce ubcore main module. +- !18842 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.85-6.6.86 LTS Patches +- !18484 btf: Avoid weak external references +- !18921 blk-ioinf: introduce inflight-based IO QoS controller +- blk-rq-qos: fix kabi broken in struct rq_qos +- blk-mq: fix kabi broken in struct request +- openeuler_defconfig: Enable CONFIG_BLK_CGROUP_IOINFLIGHT +- blk-io-hierarchy: support new bio based stage ioinf +- blk-ioinf: support percentile latency QoS for oneline workloads +- blk-ioinf: dynamically adjust inflight limit to balance workloads +- blk-ioinf: add rqos/inflight/stat debufs interface +- blk-ioinf: introduce inflight-based IO QoS controller +- perf: Fix tt_core mismatch over PG CPU +- iommu/ummu: Add UMMU trace log +- iommu/ummu: Support UMMU identity mapping +- iommu/ummu: Add UB Memory support +- iommu/ummu: Support UMMU iopf capability +- iommu/ummu: Support UMMU nested mode +- iommu/ummu: Add SVA API support to UMMU driver +- iommu/ummu: Implement iommu_ops and iommu_domain_ops for UMMU driver +- iommu/ummu: Implement domain and core ops in logic UMMU framework +- iommu/ummu: Implement MPAM for QoS +- iommu/ummu: Support UMMU attribute show and store operations +- iommu/ummu: Implement IOMMU ops forwarding in logic UMMU framework +- iommu/ummu: Support UMMU page table management +- iommu/ummu: Introduce logic UMMU framework +- iommu/ummu: Add UMMU permission queue +- iommu/ummu: Add grant and ungrant operations for permission table +- iommu/ummu: Add UMMU permission table memory management +- iommu/ummu: Add UMMU private implementation +- iommu/ummu: Add manager for UMMU permission table +- iommu/ummu: Add tct/tect ops for configuration table +- iommu/ummu: Support UMMU configuration table management +- !18863 drivers/perf: hisi: Add support for HiSilicon MN PMU driver +- ub: ubase: Support querying dev caps. +- ub: ubase: Supports the query of UBCL config. +- ub: ubase: add function that query aeq/ceq/tp/tpg context +- KVM: x86: use array_index_nospec with indices that come from guest +- software node: Correct a OOB check in software_node_get_reference_args() +- scsi: ufs: mcq: Add NULL check in ufshcd_mcq_abort() +- misc: pci_endpoint_test: Avoid issue of interrupts remaining after request_irq error +- !18885 ub: ubase: Fix some compile warnings&fix pages free problem +- !18910 um: virtio_uml: Fix use-after-free after put_device in probe +- !18902 i2c: cros-ec-tunnel: defer probe if parent EC is not present +- urma: introduce ubcore main module. +- um: virtio_uml: Fix use-after-free after put_device in probe +- mfs: tool: Add mfs user space tool demo +- mfs: Optimize mfs syncer process and metadata ops +- mfs: enable MFS +- MAINTAINERS: Add maintainer for mfs +- mfs: Add MFS documentation +- mfs: Add tracepoint for MFS +- mfs: Add user command for handling events +- mfs: Add communication devie and event acquisition operations +- mfs: Add basic events framework +- mfs: Add basic cache framework: check method and control structure +- mfs: Add basic data operation for MFS +- mfs: Add basic metadata operation for MFS +- mfs: implement the mount procedure +- mfs: Initialize mfs module +- ub: ubase: Fix some compile warnings +- !18894 some bugfixes for NUMA.remote +- i2c: cros-ec-tunnel: defer probe if parent EC is not present +- ub:hisi-ubus: Support MSGQ queue exception handling +- ub:hisi-ubus: Support for processing CQ request messages +- ub:hisi-ubus: Provide callback functions for message devices +- ub:hisi-ubus: Support for MSGQ channel processing in SQ and CQ +- ub:hisi-ubus: Support for hisi MSGQ framework +- ub:hisi-ubus: Support UBUS local RAS interfaces +- ub:hisi-ubus: Support UBUS vdm pue/ue register and release +- ub:hisi-ubus: Support UBUS vendor defined message framework +- ub:hisi-ubus: Add HiSilicon ubus driver framework +- !18723 GMEM: General Memory Management +- !18852 vfio:ubus: Support Virtual Function I/O of UBUS +- !18788 ub: Support new features for UBUS +- !18701 ubios_uvb: support uvb driver +- mm/numa_remote: check if memory block is preonlined when undoing fake-online +- mm/numa_remote: fix restoring to preonline when failing to offline from preonline +- mm/numa_remote: fix memory leak of struct memory_block +- !18840 v2 Backport some memory policy feature and bugfix from mainline +- ub: ubase: Fix the problem that free pages when ubase driver destroy TA context +- ubios_uvb: support CIS framework receive function +- ubios_uvb: support CIS framework send function +- ubios_uvb: support parse ubios information by dtb +- ubios_uvb: support UBIOS object description specification parse +- vfio:ubus: support obtaining and configuring function entity information in user mode +- vfio:ubus: Support configuration for function entity interrupt +- vfio:ubus: Support access to function entity resource space +- vfio:ubus: Init vfio ub config space 1 perm and port perm bits during module init +- vfio:ubus: Init vfio ub config space 0 perm bits during module init +- vfio:ubus: Init vfio ub configuration space +- vfio:ubus: Support ub vfio config read and write +- vfio:ubus: Support for vfio ubus driver framework +- ub:ubus: Support UBUS driver host probe and remove +- ub:ubus: Add configuration and resource sysfs interfaces +- ub:ubus: Add basic System Filesystem attributes +- ub:ubus: Support processing protocol link messages +- ub:ubus: Support hotplug processing main workflow +- ub:ubus: Support for device enumeration during the hotplug phase +- ub:ubus: Support for slot register configuration +- ub:ubus: Support hotplug service driver +- ub:ubus: Add generic ub component driver +- ub:ubus: Add ub service driver framework +- ub:ubus: Add new UBUS pool device interfaces +- ub:ubus: Support UBUS pool devices register +- ub:ubfi: register ubc usi domain +- ub:ubus: Support for ub device resource space registration +- ub:ubus: Add UBUS receive message framework +- ub:ubus: Support device binding and unbinding bus instance +- ub:ubus: Support for static cluster bus instance +- ub:ubus: Support for dynamic bus instance +- ub:ubus: Support for bus instance framework +- ub:ubus: Support handle device link up and link down +- ub:ubus: Support enumerate port and route set/clear for port +- ub:ubus: Support route modify/delete function +- !18750 iommu/ummu: Add ummu driver basic capabilities. +- iommu/ummu: Add UMMU cache flush interfaces +- iommu/ummu: Support UMMU event handling +- iommu/ummu: Support UMMU global error handling +- iommu/ummu: Support UMMU command queue +- iommu/ummu: Support UMMU device +- !18748 ub: ubfi: Support ummu node in ubfi +- !18804 drivers/perf: hisi: Support UMMU pmu +- !18855 mm: do not install PMD mappings when handling a COW fault +- drivers/perf: hisi: Add support for HiSilicon MN PMU driver +- drivers/perf: hisi: Add support for HiSilicon NoC PMU +- Revert "drivers/perf: hisi: Add support for HiSilicon NoC PMU" +- Revert "drivers/perf: hisi: Add support for HiSilicon MN PMU driver" +- ub: ubfi: Parsing ummu node in the ubrt table +- ub: ubase: Added debug information query function for FST/FVT/RQMT entries +- ub: ubase: support debugfs for active dev stats. +- mm: do not install PMD mappings when handling a COW fault +- urma: implement workqueue management for ubcore module. +- urma: implement topology information management for ubcore module. +- urma: implement hash table utilities for ubcore module. +- urma: implement log module for ubcore subsystem. +- gmem: fix some warnings +- gmem: open CONFIG_GMEM for arm64 and x86 +- mm: implement hmemcpy API +- mm: implement hmadvise API +- mm: support mmap/unmap peer-shared memory area +- gmem: add gmem stat +- gmem: support pin/unpin device memory +- gmem: support swapping heterogeneous memory +- gmem: implement page faulting for peer-shared vma +- gmem: implement allocating/deallocating peer-shared vma. +- gmem: introduce hnode for free peer-shared free page management +- gmem: introduce vm_object for managing hetergoneous memory mapping +- gmem: prepare config, boot parameter, vma flag and basic GMEM related structs +- arm64/mpam: Add quirk for L3 CPBM validity check +- fs/resctrl: Fix the L3 CMAX conversion mechanism +- wifi: ath12k: fix invalid access to memory +- serial: 8250_dma: terminate correct DMA in tx_dma_flush() +- memstick: rtsx_usb_ms: Fix slab-use-after-free in rtsx_usb_ms_drv_remove +- net: usb: usbnet: restore usb%d name exception for local mac addresses +- net: usb: qmi_wwan: add Telit Cinterion FE990B composition +- net: usb: qmi_wwan: add Telit Cinterion FN990B composition +- tty: serial: fsl_lpuart: disable transmitter before changing RS485 related registers +- tty: serial: 8250: Add Brainboxes XC devices +- tty: serial: 8250: Add some more device IDs +- counter: microchip-tcb-capture: Fix undefined counter channel state on probe +- counter: stm32-lptimer-cnt: fix error handling when enabling +- ALSA: hda/realtek: Support mute LED on HP Laptop 15s-du3xxx +- mm/page_alloc: fix memory accept before watermarks gets initialized +- drm/amd/display: Don't write DP_MSTM_CTRL after LT +- drm/dp_mst: Add a helper to queue a topology probe +- drm/dp_mst: Factor out function to queue a topology probe work +- ARM: Remove address checking for MMUless devices +- ARM: 9351/1: fault: Add "cut here" line for prefetch aborts +- ARM: 9350/1: fault: Implement copy_from_kernel_nofault_allowed() +- HID: hid-plantronics: Add mic mute mapping and generalize quirks +- ALSA: usb-audio: Add quirk for Plantronics headsets to fix control names +- mm/mempolicy: count MPOL_WEIGHTED_INTERLEAVE to "interleave_hit" +- mm/mempolicy: fix memory leaks in weighted interleave sysfs +- mempolicy: clean up minor dead code in queue_pages_test_walk() +- mm/mempolicy: protect task interleave functions with tsk->mems_allowed_seq +- mm/mempolicy: introduce MPOL_WEIGHTED_INTERLEAVE for weighted interleaving +- mm/mempolicy: refactor a read-once mechanism into a function for re-use +- mm/mempolicy: implement the sysfs-based weighted_interleave interface +- mempolicy: migration attempt to match interleave nodes +- mempolicy: mmap_lock is not needed while migrating folios +- mempolicy: alloc_pages_mpol() for NUMA policy without vma +- kernfs: drop shared NUMA mempolicy hooks +- mempolicy: remove confusing MPOL_MF_LAZY dead code +- mempolicy: mpol_shared_policy_init() without pseudo-vma +- mempolicy trivia: use pgoff_t in shared mempolicy tree +- mempolicy trivia: slightly more consistent naming +- mempolicy trivia: delete those ancient pr_debug()s +- mempolicy: fix migrate_pages(2) syscall return nr_failed +- Revert "mm/mempolicy.c: fix the out-of-bounds access issue in mpol_parse_str" +- arm64/mpam: Add quirk to retrigger L3 CSU monitor +- drivers/perf: hisi: Add the sysfs interface to configure pmu mpam +- drivers/perf: hisi: Implement ummu-pmu driver. +- ata: pata_via: Force PIO for ATAPI devices on VT6415/VT6330 +- Input: gpio-keys - fix a sleep while atomic with PREEMPT_RT +- phy: qcom-qmp-usb: Fix an NULL vs IS_ERR() bug +- fpga: fix potential null pointer deref in fpga_mgr_test_img_load_sgt() +- crypto: sun8i-ce-cipher - fix error handling in sun8i_ce_cipher_prepare() +- perf/amlogic: Replace smp_processor_id() with raw_smp_processor_id() in meson_ddr_pmu_create() +- wifi: ath11k: fix node corruption in ar->arvifs list +- jfs: fix array-index-out-of-bounds read in add_missing_indices +- platform/x86: dell-wmi-sysman: Avoid buffer overflow in current_password_store() +- regulator: max20086: Fix refcount leak in max20086_parse_regulators_dt() +- regulator: max20086: fix invalid memory access +- dmaengine: ti: Add NULL check in udma_probe() +- btf: Avoid weak external references +- vmlinux: Avoid weak reference to notes section +- kallsyms: Avoid weak references for kallsyms symbols +- jfs: Fix null-ptr-deref in jfs_ioc_trim +- ksmbd: fix null pointer dereference in destroy_previous_session +- tty: serial: uartlite: register uart driver in init +- mm/damon/sysfs-schemes: free old damon_sysfs_scheme_filter->memcg_path on write +- f2fs: zone: fix to avoid inconsistence in between SIT and SSA +- spufs: fix a leak on spufs_new_file() failure + +* Fri Nov 07 2025 Li Nan - 6.6.0-116.0.0.95 +- !18848 net:nebula-matrix:fix compile err +- net:nebula-matrix:fix compile err +- !18810 ub: ubase: support for advanced functions +- !18758 nbd: restrict sockets to TCP and UDP +- !18407 net:Add a new protocol family definition AF_UB to support UB Memory based Socket(UMS) +- !18839 rseq: Protect event mask against membarrier IPI +- ub: ubase: support for activate/deactivate dev interface +- ub: ubase: pf supports debugfs query port/vl current. +- ub: ubase: adds the function to get ub port stats for unic and other aux driver +- ub: ubase: support pre-alloc 2M pages. +- ub: ubase: support ras function +- ub: ubase: Support to port reset +- ub: ubase: Added QoS and traffic management debugging features +- ub: ubase: Support reset count information statistics. +- ub: ubase: Support for ELR and entity reset. +- ub: ubase: Provide an entity_list query interface. +- ub: ubase: support create and delete UM TP/TPG +- ub: ubase: Support to config and query the sl scheduling mode and weight in ETS and TM modes. +- ub: ubase: support querying sl information in initialization phase +- ub: ubase: Supports for ctrl queue management. +- rseq: Protect event mask against membarrier IPI +- !18820 [OLK-6.6] backport important bugfix for lpfc driver from upstream linux-6.6.y +- !18817 [OLK-6.6] backport important bugfix for mpt3sas driver from upstream linux-6.6.y +- scsi: lpfc: Fix buffer free/clear order in deferred receive path +- !18800 net: hibmcge: support pagepool for rx +- scsi: mpt3sas: Fix crash in transport port remove by using ioc_info() +- !18745 *** Mainline PATCH backport *** +- !18807 smb: client: fix wrong index reference in smb2_compound_op() +- !18711 Bluetooth: hci_conn: Use disable_delayed_work_sync +- !17268 LoongArch: fix nvme/megaraid/stmmac/uprobe/rtc/kdump problems +- !18500 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.82-6.6.83 LTS Conflicts Patches +- ub: ubase: adds the function that query oor caps +- ub: ubase: Support for virtual register and unregister. +- ub: ubase: Add trace events to aeqe and ceqe. +- ub: ubase: Add the cmdq trace event print at the ubase layer. +- ub: ubase: Support for timer/ext_dtb buffer management. +- !18784 padata: Reset next CPU when reorder sequence wraps around +- !18806 perf: fix ddrc redundant value set +- !18802 KVM: x86: Don't (re)check L1 intercepts when completing userspace I/O +- !18739 fs: udf: fix OOB read in lengthAllocDescs handling +- !18710 ub: ubase: Add support of base driver for ub network +- smb: client: fix wrong index reference in smb2_compound_op() +- ub: ubase: Support for public Context memory allocation and release. +- ub: ubase: support for cdma auxiliary device +- ub: ubase: support for complete event process +- ub: ubase: support for async event process +- ub: ubase: add interrupt management framework and misc handler +- ub: ubase: add function to query device resource +- ub: ubase: add the function that query ueid +- ub: ubase: support debugfs public interface. +- ub: ubase: support for command process +- ub: ubase: add support of ubase driver for ub network +- perf: fix ddrc redundant value set +- KVM: x86: Don't (re)check L1 intercepts when completing userspace I/O +- net: hibmcge: support pagepool for rx +- net: hibmcge: reduce packet drop under stress testing +- padata: Reset next CPU when reorder sequence wraps around +- nbd: restrict sockets to TCP and UDP +- x86/umip: Fix decoding of register forms of 0F 01 (SGDT and SIDT aliases) +- x86/umip: Check that the instruction opcode is at least two bytes +- PM: sleep: core: Clear power.must_resume in noirq suspend error path +- ACPI: TAD: Add missing sysfs_remove_group() for ACPI_TAD_RT +- ACPI: processor: idle: Fix memory leak when register cpuidle device failed +- fs: udf: fix OOB read in lengthAllocDescs handling +- net: Add a new protocol family AF_UB +- Bluetooth: hci_conn: Use disable_delayed_work_sync +- selftests: net: test for lwtunnel dst ref loops +- arm64: dts: rockchip: fix pinmux of UART5 for PX30 Ringneck on Haikou +- arm64: dts: rockchip: Move uart5 pin configuration to px30 ringneck SoM +- net: ipv6: fix dst ref loop in ila lwtunnel +- net: ipv6: fix missing dst ref drop in ila lwtunnel +- arm64: dts: rockchip: add rs485 support on uart5 of px30-ringneck-haikou +- arm64: hugetlb: Fix huge_ptep_get_and_clear() for non-present ptes +- NFS: fix nfs_release_folio() to not deadlock via kcompactd writeback +- net: stmmac: dwmac-loongson: Set correct {tx,rx}_fifo_size +- LoongArch: Support mem= kernel parameter +- LoongArch: Reserve the EFI memory map region +- LoongArch: Add debugfs entries to switch SFB/TSO state +- LoongArch: uprobes: Remove redundant code about resume_era +- LoongArch: uprobes: Remove user_{en,dis}able_single_step() +- LoongArch: fix access mode of 7A2000 acpi gpio. +- LoongArch: configs: enable zstd compressed firmware support +- Loongarch: configs: Make EFI RTC built-in for correct vmcore timestamps +- megaraid: Add udelay to avoid data error caused by dma sequence +- nvme: Add udelay to avoid data error caused by dma sequence + +* Wed Nov 05 2025 Mingzheng Xing - 6.6.0-115.0.0.94 +- RISC-V kernel upgrade to 6.6.0-115.0.0 +- Remove CI-related file changes +- riscv: Backport steal-time support for RISC-V +- riscv: RISC-V SBI debug console extension support +- riscv: Add support for kernel mode vector + +* Wed Nov 05 2025 Li Nan - 6.6.0-115.0.0.93 +- !18713 iommu: Enabling Broadcast TLB Maintenance (BTM) for UMMU on ARM64 +- iommu: Fix kabi breakage due to opening KVM_PINNED_VMID +- vfio: Fix kabi breakage due to closing VFIO_CONTAINER +- iommu: Pass in kvm pointer to domain_alloc_user +- iommufd: Associate kvm pointer to iommufd ctx +- KVM: arm64: Introduce support to pin VMIDs +- KVM: Add generic infrastructure to support pinned VMIDs +- !18695 PCI: Run quirk_huawei_pcie_sva() before arm_smmu_probe_device() +- !18761 net: hns3: the multi-tcs VF can be configured only when the interface is down +- !18735 KVM: arm64: Prevent access to vCPU events before init +- !18598 arm64/watchdog_hld: Add a cpufreq notifier for update watchdog thresh +- net: hns3: the multi-tcs VF can be configured only when the interface is down +- net: hns3: remap vf's queue and qset when vport start +- net: hns3: support dpdk using vf multiple tcs +- net: hns3: add support for vf get dscp configuration from pf +- net: hns3: add support for vf multiple tcs +- !18597 iommu/ummu-core: Add ummu-core framework. +- !18316 jbd2: prevent softlockup in jbd2_log_do_checkpoint() +- KVM: arm64: Prevent access to vCPU events before init +- !18421 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.84-6.6.85 LTS Patches +- !18702 i40e: fix idx validation in i40e_validate_queue_map +- !18699 *** CVE-2025-39973 *** +- !18649 tty: n_gsm: Don't block input queue by waiting MSC +- PCI: Run quirk_huawei_pcie_sva() before arm_smmu_probe_device() +- iommu/ummu-core: core interfaces for ummu drivers +- iommu/ummu-core: impl ioctl interface for /dev/ummu device +- iommu/ummu-core: add pseudo Token Dev for Token ID abstraction +- iommu/ummu-core: add UMMU Token ID operation interfaces +- iommu/ummu-core: add UMMU EID operation interfaces +- iommu/ummu-core: support IOVA on demand mapping +- iommu/ummu-core: get iova_domain from iommu_domain +- !18659 ptp: Add a upper bound on max_vclocks +- !18680 futex: Prevent use-after-free during requeue-PI +- !18683 qed: Don't collect too many protection override GRC elements +- !18682 i40e: fix validation of VF state in get resources +- !18681 i40e: fix input validation logic for action_meta +- !18679 cnic: Fix use-after-free bugs in cnic_delete_task +- !18668 openvswitch: Fix unsafe attribute parsing in output_userspace() +- !18267 CVE-2025-39832 +- !18705 mm: fix compilation error due to MAX_PAGE_ORDER isn't declared +- arm64/watchdog_hld: Add a cpufreq notifier for update watchdog thresh +- watchdog/perf: Provide function for adjusting the event period +- Revert "watchdog/perf: Provide function for adjusting the event period" +- Revert "arm64/watchdog_hld: Add a cpufreq notifier for update watchdog thresh" +- !18690 [OLK-6.6] Fix CVE-2025-38290 +- !18672 [OLK-6.6] Fix CVE-2025-38286 +- !17950 net: atm: add lec_mutex +- iommu/ummu-core: introduce UMMU-CORE, basic configs +- iommu/ummu-core: support iommu ksva mode +- iommu/ummu-core: introduce iommu sva permission operation +- iommu/ummu-core: introduce iommu_sva_bind_isolated interface +- iommu/ummu-core: add the user invalidation type in iommufd file +- iommu/ummu-core: iommu_dev new min_pasids attribute +- mm: fix compilation error due to MAX_PAGE_ORDER isn't declared +- i40e: fix idx validation in i40e_validate_queue_map +- i40e: add validation for ring_len param +- i40e: increase max descriptors for XL710 +- !18372 sched/fair: remove qos_reweight logic under non-SMP scenario +- !18423 ext4: disambiguate the return value of ext4_dio_write_end_io() +- !18328 [OLK-6.6] fix txgbe link fec issues and enhance hot reset mechanism +- wifi: ath12k: fix node corruption in ar->arvifs list +- !18662 net: hibmcge: fix the inappropriate netif_device_detach() +- !18605 net: ubl: add depend to ARM64 and remove config in x86 +- qed: Don't collect too many protection override GRC elements +- i40e: fix validation of VF state in get resources +- i40e: fix input validation logic for action_meta +- futex: Prevent use-after-free during requeue-PI +- cnic: Fix use-after-free bugs in cnic_delete_task +- !18666 bpf-rvi: block: fix memleak issue in bpf_show_partitions_start() +- !18270 Bluetooth: Fix use-after-free in l2cap_sock_cleanup_listen() +- !18271 libceph: fix invalid accesses to ceph_connection_v1_info +- !18272 CVE-2025-39889 +- !18273 ppp: fix memory leak in pad_compress_skb +- !18268 RDMA: hfi1: fix possible divide-by-zero in find_hw_thread_mask() +- openvswitch: Fix unsafe attribute parsing in output_userspace() +- pinctrl: at91: Fix possible out-of-boundary access +- !18646 tracing: Fix race condition in kprobe initialization causing NULL pointer dereference +- !18630 tracing: dynevent: Add a missing lockdown check on dynevent +- bpf-rvi: block: fix memleak issue in bpf_show_partitions_start() +- iommu/ummu-core: New ub bus added to the iommu framework +- !18525 ub: add UBUS and UBFI driver framework +- !18486 ACPI: pfr_update: Fix the driver update version check +- !18363 comedi: pcl726: Prevent invalid irq number +- net: hibmcge: fix the inappropriate netif_device_detach() +- net: hibmcge: remove unnecessary check for np_link_fail in scenarios without phy. +- net: hibmcge: fix rx buf avl irq is not re-enabled in irq_handle issue +- ptp: Add a upper bound on max_vclocks +- tty: n_gsm: Don't block input queue by waiting MSC +- tracing: Fix race condition in kprobe initialization causing NULL pointer dereference +- tracing: dynevent: Add a missing lockdown check on dynevent +- ub:ubus: Supports decoder event processing +- ub:ubus: Support for sending decoder commands +- ub:ubus: Support for range table mapping and unmapping functions +- ub:ubus: Support for page-range table address mapping and unmapping +- ub:ubus: Support for Bus EID-UPI Table Configuration +- ub:ubus: Support for UB Character Device Driver +- ub:ubus: Support UBUS RAS capability +- ub:ubus: Creating and deleting ub decoder page table mappings +- ub:ubus: Adding a ue Creation Destruction Mechanism +- ub:ubus: Enable the ub decoder device support +- ub:ubus: Add UBUS RAS framework +- ub:ubus: Support configuration of device information interface +- ub:ubus: Support for enabling and disabling ue +- ub:ubus: Add MSI capability for UBUS driver +- ub:ubus: Add UBUS MSI basic functions +- ub:ubus: Support UBUS Interrupt framework +- ub:ubus: Add UBUS capability interfaces +- ub:ubus: Support device level and port level reset +- ub:ubus: Support for multiple mue enablement +- ub:ubus: Support UB device enabling basic interface +- ub:ubus: Supports device enumeration. +- ub:ubus: Support for UB Bus Controller Enumeration +- ub:ubus: Support for shortest path routing configuration +- ub:ubus: Supports network address configuration and querying +- ub:ubus: Support enumeration topology query commands +- ub:ubus: Add UBUS resource space basic functions +- ub:ubus: Add UBUS resource space framework +- ub:ubus: Support for UB bus enumeration message initialization +- ub:ubus: Support for UB routing table configuration function +- ub:ubus: Add CNA allocate and free interfaces by kernel +- ub:ubus: Add EID allocate and free interfaces by kernel +- ub:ubus: Support cc configuration and query +- ub:ubus: Supporting the UB Shared Port Function +- ub:ubus: Support for UB port sysfs attribute files +- ub:ubus: Support Ubus read/write configuration functions +- ub:ubus: Support for UB Bus Port Management Framework +- ub:ubus: Add Ubus setting configuration space function +- ub:ubus: Support for UB Bus DMA Configuration Function +- ub:ubfi: ubfi driver create ubc device +- ub:ubus: Support for ubus messaging communication framework +- ub:ubus: add ubus controller framework +- ub:ubfi: ubfi driver parse ubc information from ubrt +- ub:ubus: Support UB Bus Attribute Group +- ub:ubfi: Support ub firmware interface basic functions +- ub:ubus: Support for ub bus driver framework +- net: ubl: add depend to ARM64 and remove config in x86 +- net: ubl: modify the description in Kconfig of UB +- ACPI: pfr_update: Fix the driver update version check +- net: txgbe: fix invoke hot reset multi times unexpectedly +- net: txgbe: fix race condition when recover or close/suspend +- net: txgbe: fix fec mode issues +- net: txgbe: set pf promisc and vlan cannot receive packets +- net: txgbe: fix 25G link issues +- net: txgbe: change default tx queue depth +- net: txgbe: fix i2c issues +- net: txgbe: fix can't read rx_pb_n_pxoff +- ext4: disambiguate the return value of ext4_dio_write_end_io() +- net: lwtunnel: disable BHs when required +- netfilter: nft_counter: Use u64_stats_t for statistic. +- wifi: iwlwifi: mvm: ensure offloading TID queue exists +- wifi: iwlwifi: support BIOS override for 5G9 in CA also in LARI version 8 +- arm64: dts: rockchip: fix u2phy1_host status for NanoPi R4S +- mptcp: Fix data stream corruption in the address announcement +- drm/amd/display: Use HW lock mgr for PSR1 when only one eDP +- drm/amd/display: should support dmub hw lock on Replay +- drm/amdgpu: Fix JPEG video caps max size for navi1x and raven +- drm/amdgpu: Fix MPEG2, MPEG4 and VC1 video caps max size +- drm/sched: Fix fence reference count leak +- batman-adv: Ignore own maximum aggregation size during RX +- efi/libstub: Avoid physical address 0x0 when doing random allocation +- ARM: shmobile: smp: Enforce shmobile_smp_* alignment +- ARM: dts: imx6qdl-apalis: Fix poweroff on Apalis iMX6 +- mm: fix error handling in __filemap_get_folio() with FGP_NOWAIT +- mmc: atmel-mci: Add missing clk_disable_unprepare() +- mmc: sdhci-brcmstb: add cqhci suspend/resume to PM ops +- arm64: dts: rockchip: fix pinmux of UART0 for PX30 Ringneck on Haikou +- arm64: dts: freescale: imx8mm-verdin-dahlia: add Microphone Jack to sound card +- arm64: dts: freescale: imx8mp-verdin-dahlia: add Microphone Jack to sound card +- accel/qaic: Fix integer overflow in qaic_validate_req() +- regulator: dummy: force synchronous probing +- riscv: dts: starfive: Fix a typo in StarFive JH7110 pin function definitions +- drm/v3d: Don't run jobs that have errors flagged in its fence +- can: flexcan: disable transceiver during system PM +- can: flexcan: only change CAN state when link up in system PM +- can: ucan: fix out of bound read in strscpy() source +- can: rcar_canfd: Fix page entries in the AFL list +- i2c: omap: fix IRQ storms +- net/neighbor: add missing policy for NDTPA_QUEUE_LENBYTES +- net: lwtunnel: fix recursion loops +- ipv6: Set errno after ip_fib_metrics_init() in ip6_route_info_create(). +- ARM: davinci: da850: fix selecting ARCH_DAVINCI_DA8XX +- accel/qaic: Fix possible data corruption in BOs > 2G +- Bluetooth: hci_event: Fix connection regression between LE and non-LE adapters +- RDMA/bnxt_re: Avoid clearing VLAN_ID mask in modify qp path +- ARM: dts: bcm2711: Don't mark timer regs unconfigured +- ARM: OMAP1: select CONFIG_GENERIC_IRQ_CHIP +- RDMA/bnxt_re: Add missing paranthesis in map_qp_id_to_tbl_indx +- arm64: dts: rockchip: Remove undocumented sdmmc property from lubancat-1 +- ARM: dts: bcm2711: PL011 UARTs are actually r1p5 +- soc: imx8m: Unregister cpufreq and soc dev in cleanup path +- soc: imx8m: Use devm_* to simplify probe failure handling +- soc: imx8m: Remove global soc_uid +- xfrm_output: Force software GSO only in tunnel mode +- xfrm: fix tunnel mode TX datapath in packet offload mode +- arm64: dts: freescale: tqma8mpql: Fix vqmmc-supply +- firmware: imx-scu: fix OF node leak in .probe() +- net: txgbe: fix bond port rate is unknown when linkup +- net: txgbe: fix VF multicast address removal not taking effect +- net: txgbe: fix missing PFC mask configuration on AML +- net: txgbe: fix pktgen cannot stop +- sched/fair: remove qos_reweight logic under non-SMP scenario +- comedi: pcl726: Prevent invalid irq number +- jbd2: prevent softlockup in jbd2_log_do_checkpoint() +- ppp: fix memory leak in pad_compress_skb +- Bluetooth: hci_event: Fix not using key encryption size when its known +- Bluetooth: L2CAP: Fix not checking l2cap_chan security level +- Bluetooth: l2cap: Check encryption key size on incoming connection +- libceph: fix invalid accesses to ceph_connection_v1_info +- Bluetooth: Fix use-after-free in l2cap_sock_cleanup_listen() +- RDMA: hfi1: fix possible divide-by-zero in find_hw_thread_mask() +- net/mlx5: Fix lockdep assertion on sync reset unload event +- net/mlx5: Add support for sync reset using hot reset +- net/mlx5: Add device cap for supporting hot reset in sync reset flow +- net/mlx5: Reload auxiliary drivers on fw_activate +- net: txgbe: fix DAC cable link down +- net: txgbe: fix display of supports Wake-on in the ethtool +- net: atm: add lec_mutex + +* Fri Oct 31 2025 Li Nan - 6.6.0-114.0.0.92 +- !18640 v3 Errata management for VM Live migration +- !18638 CVE-2025-39982 +- !18564 perf: arm_spe: Prevent overflow in PERF_IDX2OFF() +- !18639 net: hns3: fix null pointer in debugfs issue +- KVM: arm64: fix kvm kabi conflict +- smccc: kvm_guest: Align with DISCOVER_IMPL_CPUS ABI +- smccc: kvm_guest: Fix kernel builds for 32 bit arm +- smccc/kvm_guest: Enable errata based on implementation CPUs +- arm64: Make  _midr_in_range_list() an exported function +- KVM: arm64: Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2 +- KVM: arm64: Specify hypercall ABI for retrieving target implementations +- arm64: Modify _midr_range() functions to read MIDR/REVIDR internally +- KVM: arm64: Allow userspace to change the implementation ID registers +- KVM: arm64: Load VPIDR_EL2 with the VM's MIDR_EL1 value +- KVM: arm64: Maintain per-VM copy of implementation ID regs +- KVM: arm64: Set HCR_EL2.TID1 unconditionally +- Bluetooth: hci_event: Fix UAF in hci_acl_create_conn_sync +- !18624 tools headers arm64: Sync arm64's cputype.h with the kernel sources +- !18622 support mempool feature +- net: hns3: fix null pointer in debugfs issue +- net: hns3: avoid bitwise copy during RX buffer replace leg when page pool enabled +- net: hns3: fixed vf get max channels bug +- net: hns3: `req_autoneg` for fiber is initialized to `support_autoneg` instead of OFF. +- net: hns3: use user configure after hardware reset when using kernel PHY. +- net: hns3: Return error code when function fails. +- net: hns3: using the num_tqps to check whether tqp_index is out of range when vf get ring info from mbx +- net: hns3: using the num_tqps in the vf driver to apply for resources +- net: hns3: add support to query tc map info of VF queues +- net: hns3: merge rss tc configuration +- net: hns3: add ieee_setmaxrate and ieee_getmaxrate support for pf +- !18550 cpu/SMT: recover global num_threads after disable smt switch fail +- !18589 net: hibmcge: support pci_driver.shutdown() +- !18579 regulator: core: fix NULL dereference on unbind due to stale coupling data +- !18415 sdei_watchdog: Add CPU_PM_ENTER_FAILED handling in sdei_watchdog_pm_notifier +- !18593 crypto: essiv - Check ssize for decryption and in-place encryption +- tools headers arm64: Sync arm64's cputype.h with the kernel sources +- tools headers arm64: Sync arm64's cputype.h with the kernel sources +- tools/include: Sync arm64 headers with the kernel sources +- tools/include: Sync arm64 asm/cputype.h with the kernel sources +- openeuler_defconfig: enable CONFIG_PFN_RANGE_ALLOC by default +- mm/pfn_range_alloc: check if folio is hwpoisoned in pfn_range_free +- mm/pfn_range_alloc: add a interface to show borrowable physical range +- mm/pfn_range_alloc: support update cachable attribue of linear mapping +- mm/pfn_range_alloc: introduce set_linear_mapping_invalid +- mm/pfn_range_alloc: introduce hugetlb_pool_alloc_size +- mm/pfn_range_alloc: add tracepoint for hugetlb_pool_alloc and hugetlb_pool_free +- mm/pfn_range_alloc: add tracepoint for pfn_range_alloc and pfn_range_free +- mm/pfn_range_alloc: support allocate and free from hugetlb pool +- mm/pfn_range_alloc: add interfaces to alloc/free for drivers +- mm/pfn_range_alloc: reserve memory for each node +- mm/contig_alloc: fix alloc_contig_range when __GFP_COMP and order < MAX_ORDER +- mm: page_alloc: speed up fallbacks in rmqueue_bulk() +- mm/page_alloc: clarify should_claim_block() commentary +- mm/page_alloc: clarify terminology in migratetype fallback code +- mm/page_alloc: add some detailed comments in can_steal_fallback +- mm: page_alloc: remove remnants of unlocked migratetype updates +- mm: page_alloc: don't steal single pages from biggest buddy +- mm/page_alloc: don't call pfn_to_page() on possibly non-existent PFN in split_large_buddy() +- mm/page_alloc: forward the gfp flags from alloc_contig_range() to post_alloc_hook() +- mm/page_alloc: sort out the alloc_contig_range() gfp flags mess +- mm/page_alloc: make __alloc_contig_migrate_range() static +- mm/page_isolation: don't pass gfp flags to isolate_single_pageblock() +- mm/contig_alloc: support __GFP_COMP +- mm: remove unused has_isolate_pageblock +- mm: page_alloc: simpify page del and expand +- mm: remove migration for HugePage in isolate_single_pageblock() +- mm/page_alloc: keep track of free highatomic +- mm: page_alloc: fix highatomic typing in multi-block buddies +- mm: page_alloc: batch vmstat updates in expand() +- mm: page_alloc: change move_freepages() to __move_freepages_block() +- mm: page_alloc: consolidate free page accounting +- mm: page_isolation: prepare for hygienic freelists +- mm: page_alloc: set migratetype inside move_freepages() +- mm: page_alloc: close migratetype race between freeing and stealing +- mm: page_alloc: fix freelist movement during block conversion +- mm: page_alloc: fix move_freepages_block() range error +- mm: page_alloc: move free pages when converting block during isolation +- mm: page_alloc: fix up block types when merging compatible blocks +- mm: page_alloc: optimize free_unref_folios() +- mm: page_alloc: remove pcppage migratetype caching +- mm: page_alloc: simplify __free_pages_ok() +- mm/page_alloc: remove unnecessary next_page in break_down_buddy_pages +- mm/page_alloc: remove unnecessary check in break_down_buddy_pages +- !18608 mm: fix __count_zid_vm_events when CONFIG_ZONE_EXTMEM is enabled +- !18588 add memory ras feature support +- mm: add memory reclaim notifier +- mm: fix __count_zid_vm_events when CONFIG_ZONE_EXTMEM is enabled +- panic: add notifier call before other cpu shutdown +- !18592 Make ID reg writable +- crypto: essiv - Check ssize for decryption and in-place encryption +- KVM: arm64: use KABI_EXTEND to revert struct kvm kabi change +- KVM: arm64: Make AA64PFR1_EL1.NMI writeable +- KVM: arm64: Expose S1PIE to guests +- KVM: arm64: Sanitise ID_AA64MMFR3_EL1 +- KVM: arm64: Do not allow ID_AA64MMFR0_EL1.ASIDbits to be overridden +- KVM: arm64: Make ID_AA64MMFR1_EL1.{HCX, TWED} writable from userspace +- KVM: arm64: Remove duplicated AA64MMFR1_EL1 XNX +- KVM: arm64: Make the L1Ip feature bits in CTR_EL0 writable from userspace +- KVM: arm64: rename functions for invariant sys regs +- KVM: arm64: show writable masks for feature registers +- KVM: arm64: Treat CTR_EL0 as a VM feature ID register +- KVM: arm64: Add helper for writing ID regs +- KVM: arm64: Use read-only helper for reading VM ID registers +- KVM: arm64: Only reset vCPU-scoped feature ID regs once +- KVM: arm64: Reset VM feature ID regs from kvm_reset_sys_regs() +- KVM: arm64: Rename is_id_reg() to imply VM scope +- KVM: arm64: Disable MPAM visibility by default and ignore VMM writes +- KVM: arm64: Make the exposed feature bits in AA64DFR0_EL1 writable from userspace +- KVM: arm64: Document vCPU feature selection UAPIs +- KVM: arm64: Allow userspace to change ID_AA64ZFR0_EL1 +- KVM: arm64: Allow userspace to change ID_AA64PFR0_EL1 +- KVM: arm64: Allow userspace to change ID_AA64MMFR{0-2}_EL1 +- KVM: arm64: Allow userspace to change ID_AA64ISAR{0-2}_EL1 +- KVM: arm64: Bump up the default KVM sanitised debug version to v8p8 +- KVM: arm64: Reject attempts to set invalid debug arch version +- KVM: arm64: Advertise selected DebugVer in DBGDIDR.Version +- KVM: arm64: Use guest ID register values for the sake of emulation +- KVM: arm64: Document KVM_ARM_GET_REG_WRITABLE_MASKS +- KVM: arm64: Allow userspace to get the writable masks for feature ID registers +- net: hibmcge: support pci_driver.shutdown() +- net: hibmcge: select FIXED_PHY +- !18568 support NUMA.remote feature +- !18498 serial: 8250: fix panic due to PSLVERR +- regulator: core: fix NULL dereference on unbind due to stale coupling data +- !18409 crypto: hisilicon - support for new ZIP features +- !18545 HID: multitouch: fix slab out-of-bounds access in mt_report_fixup() +- !18563 arm64/mpam: Fix L2 MBWU monitor multiplexing issue +- mm/numa_remote: add kernel doc for numa remote feature +- mm/numa_remote: extend cmdline numa_remote to limit the max number of remote node +- mm: memory-failure: use kill_accessing_process() in ghes +- mm: memory-failure: remove task_struct from kill_accessing_process() +- mm/memory-failure: fix infinite UCE for VM_PFNMAP pfn +- mm/hwpoison: do not send SIGBUS to processes with recovered clean pages +- arm64: configs: enable NUMA_REMOTE by default +- mm/numa_remote: enable oom_kill_cpuless_numa_allocating_task when numa_remote is enabled +- mm/oom_kill: kill current in OOM when binding to cpu-less nodes +- mm/numa_rmeote: add preonline interface for each memory device +- mm/numa_remote: add pre-online count in meminfo +- mm/numa_remote: add sysfs to distinguish whether a remote node +- mm/numa_remote: introduce hugetlb_nowatermark mode for remote node +- mm/numa_remote: undo isolation of remote memory asynchronously +- mm/numa_remote: introduce pre-online mode to support hotplug unready memory +- mm/numa_remote: introduce nofallback mode for remote node +- mm/numa_remote: support to set node distance +- mm/numa_remote: support to hotplug/hotremove remote memory +- mm/numa_remote: prepare numa node for remote memory +- mm/memory_hotplug: support to hotplug memory on ZONE_EXTMEM +- mm/hugetlb: fix folio is still mapped when deleted +- perf: arm_spe: Prevent overflow in PERF_IDX2OFF() +- arm64/mpam: Fix L2 MBWU monitor multiplexing issue +- !18559 arm64/mpam: Fix MBWU monitor overflow handling +- !18544 fs/resctrl: Re-allocate rmid for the monitor when migrating across control groups +- !18536 arm-smmu-v3: add 'HIP10C' for 162100602 errata +- !18130 Net: nebula_matrix: set nebula_matrix driver to a newer state +- !16777 RNPGBE: NET: Fix dmesg 'unknow speed' show when 1000M +- arm64/mpam: Print MPAM register operation for debug +- arm64/mpam: Fix MBWU monitor overflow handling +- cpu/SMT: recover global num_threads after disable smt switch fail +- HID: multitouch: fix slab out-of-bounds access in mt_report_fixup() +- fs/resctrl: Re-allocate rmid for the monitor when migrating across control groups +- !17676 [OLK-6.6] Backported patches of Transient Scheduler Attacks (TSA) +- arm-smmu-v3: add 'HIP10C' for 162100602 errata +- !18533 [OLK-6.6]:update patches for sw64 architecture +- !18516 arm64: Add support for HIP09 Spectre-BHB mitigation +- !18457 arm64: add cnp verification on Hisilicon erratum 162100125 +- !18513 Fix CVE-2025-39679 +- !18531 arm/syscalls: mark syscall invocation as likely in +- !11635 arm64/cpufeatures/kvm: Add ARMv8.9 FEAT_ECBHB bits in ID_AA64MMFR1 register +- !18507 Fix wrong CPU aff3 conversion between MPIDR and SYS_LSUDVMBM_EL2 +- sw64: add CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y to defconfig +- sw64: fix the condition for enabling hw_una_enabled +- sw64: efi: fix compile error when CONFIG_EFI=n +- sw64: powercap: fix misuse of an uninitialized temporary variable +- sw64: powercap: increase the interval of polling mode +- sw64: powercap: fix Kconfig dependency for SW64_POWERCAP +- sw64: powercap: avoid checking meaningless target freq +- sw64: smp: fix unaligned access before trap_init() +- sw64: irqchip: update effective affinity for INTx +- sw64: msi: fix affinity change of managed irq +- sw64: use orig_r0 as syscall nr +- sw64: revert ioremap() to __va() in ioport_map() +- sw64: fix unresolved reference error when close pci +- sw64: kvm: add the C4 kvm_regs structure to uapi kvm.h +- arm/syscalls: mark syscall invocation as likely in invoke_syscall +- !18517 media: rc: fix races with imon_disconnect() +- !18327 net: ubl: add CONFIG_UB_UBL definition and UBL interface +- !18488 RDMA/hns: Fix wrong WQE data when QP wraps around +- KVM: hisi_virt: tlbi: Fix wrong CPU aff3 conversion between MPIDR and SYS_LSUDVMBM_EL2 +- media: rc: fix races with imon_disconnect() +- arm64: Add support for HIP09 Spectre-BHB mitigation +- drm/nouveau/nvif: Fix potential memory leak in nvif_vmm_ctor(). +- !18495 CVE-2025-39967 +- !18258 perf: arm_pmuv3: Factor out PMCCNTR_EL0 use conditions and Don't use PMCCNTR_EL0 on SMT cores +- serial: 8250: fix panic due to PSLVERR +- fbcon: Fix OOB access in font allocation +- fbcon: fix integer overflow in fbcon_do_set_font +- !16738 perf vendor events arm64: Add N2, V2, N3, V3 events/metrics +- !18385 e1000e: fix heap overflow in e1000_set_eeprom +- !18395 scsi: target: target_core_configfs: Add length check to avoid buffer overflow +- config: enable CONFIG_MITIGATION_TSA and CONFIG_KVM_HYPERV by default on x86 +- x86/cpu: Shorten CPU matching macro +- platform/x86/intel/ifs: Switch to new Intel CPU model defines +- x86/microcode/AMD: Handle the case of no BIOS microcode +- KVM: SVM: Advertise TSA CPUID bits to guests +- tools headers: Sync x86 headers with the kernel source +- x86/microcode/AMD: Fix __apply_microcode_amd()'s return value +- x86/microcode/AMD: Add some forgotten models to the SHA check +- x86/CPU/AMD: Terminate the erratum_1386_microcode array +- x86/microcode/AMD: Split load_microcode_amd() +- x86/microcode/AMD: Pay attention to the stepping dynamically +- KVM: x86: hyper-v: Remove unused inline function kvm_hv_free_pa_page() +- x86/microcode/intel: Set new revision only after a successful update +- x86/platform/atom: Switch to new Intel CPU model defines +- x86/process: Move the buffer clearing before MONITOR +- x86/microcode/AMD: Add TSA microcode SHAs +- x86/microcode/AMD: Load only SHA256-checksummed patches +- x86/microcode/AMD: Add get_patch_level() +- x86/microcode/AMD: Merge early_apply_microcode() into its single callsite +- x86/microcode/AMD: Remove unused save_microcode_in_initrd_amd() declarations +- x86/microcode/AMD: Remove ret local var in early_apply_microcode() +- x86/microcode/AMD: Use the family,model,stepping encoded in the patch ID +- x86/microcode/AMD: Avoid -Wformat warning with clang-15 +- x86/microcode/AMD: Have __apply_microcode_amd() return bool +- x86/microcode/AMD: Flush patch buffer mapping after application +- x86/microcode: Rework early revisions reporting +- x86/bugs: Add a Transient Scheduler Attacks mitigation +- x86/bugs: Rename MDS machinery to something more generic +- x86/idle: Use MONITOR and MWAIT mnemonics in +- KVM: x86: Swap incoming guest CPUID into vCPU before massaging in KVM_SET_CPUID2 +- KVM: x86: Explicitly do runtime CPUID updates "after" initial setup +- KVM: x86: Rename kvm_cpu_cap_mask() to kvm_cpu_cap_init() +- x86/cpu: Expose only stepping min/max interface +- x86/cpu: Introduce new microcode matching helper +- cpufreq: intel_pstate: Switch to new Intel CPU model defines +- cpufreq: Switch to new Intel CPU model defines +- EDAC/skx: Switch to new Intel CPU model defines +- EDAC/i10nm: Switch to new Intel CPU model defines +- x86/aperfmperf: Switch to new Intel CPU model defines +- x86/apic: Switch to new Intel CPU model defines +- x86/bugs: Switch to new Intel CPU model defines +- x86/CPU/AMD: Improve the erratum 1386 workaround +- cpufreq: intel_pstate: Update default EPPs for Meteor Lake +- cpufreq: intel_pstate: Update hybrid scaling factor for Meteor Lake +- KVM: x86: Make Hyper-V emulation optional +- KVM: nVMX: Split off helper for emulating VMCLEAR on Hyper-V eVMCS +- KVM: VMX: Split off hyperv_evmcs.{ch} +- net: ubl: add CONFIG_UB_UBL definition and UBL interface +- RDMA/hns: Fix wrong WQE data when QP wraps around +- !18464 i40e: add max boundary check for VF filters +- !18433 dmaengine: qcom: bam_dma: Fix DT error handling for num-channels/ees +- !18256 Add EDAC basic support and RRL enhancement for CWF/SRF/GNR/GNR-D +- !18254 Add IFS support for ClearWater Forest +- !18455 KVM: arm64: vgic-v4: Restore pending state on host userspace write +- !18289 Expand scope of khugepaged anonymous collapse +- !18324 arm64: Enable vmalloc-huge with ptdump +- !18419 sched/core: Add sched_change_qos_group() in sched_cgroup_fork() +- i40e: add max boundary check for VF filters +- arm64: add cnp verification on Hisilicon erratum 162100125 +- !18151 ext4: do not BUG when INLINE_DATA_FL lacks system.data xattr +- !18152 fs/buffer: fix use-after-free when call bh_read() helper +- !18312 ext4: preserve SB_I_VERSION on remount +- !18313 fs: quota: create dedicated workqueue for quota_release_work +- !18311 ext4: fix an off-by-one issue during moving extents +- !17401 fscrypt: Don't use problematic non-inline crypto engines +- KVM: arm64: vgic-v4: Restore pending state on host userspace write +- !18237 fuse: support sysfs api for flush and +- !18422 net/hinic3: Modify driver version +- dmaengine: qcom: bam_dma: Fix DT error handling for num-channels/ees +- !18425 Fix CVE-2025-39989 +- !18427 i40e: fix idx validation in config queues msg +- i40e: fix idx validation in config queues msg +- x86/mce: use is_copy_from_user() to determine copy-from-user context +- x86/extable: Remove unused fixup type EX_TYPE_COPY +- net/hinic3: Modify driver version +- net:nebula-matrix:Add S1000 SNIC driver support +- !18414 Fix add define COMPAT_ELF_HWCAP3 +- !18406 pcmcia: Add error handling for add_interval() in do_validate_mem() +- sched/core: Add sched_change_qos_group() in sched_cgroup_fork() +- !18401 Bluetooth: btrtl: Prevent potential NULL dereference +- !18241 ras: ACPI: HED: Always initialize before evged +- sdei_watchdog: Add CPU_PM_ENTER_FAILED handling in sdei_watchdog_pm_notifier +- Fix add define COMPAT_ELF_HWCAP3 +- !17928 bus: fsl-mc: fix double-free on mc_dev +- !17908 fix CVE-2025-38630 +- !18224 fix CVE-2025-38624 +- !17871 f2fs: fix to trigger foreground gc during f2fs_map_blocks() in lfs mode +- !17877 fs/ntfs3: cancle set bad inode after removing name fails +- !17909 fix CVE-2025-38555 +- !17872 f2fs: fix to avoid panic in f2fs_evict_inode +- !17760 Fix CVE-2025-38414 +- !17611 net: ravb: Fix missing rtnl lock in suspend/resume path +- !17859 f2fs: fix to bail out in get_new_segment() +- !17354 net/atm/resources: Release atm_dev_mutex after removing procfs in atm_… +- !18032 [OLK-6.6] Fix CVE-2025-37897 +- !18030 [OLK-6.6] Fix CVE-2025-37914 +- !18037 [OLK-6.6] Fix CVE-2025-37917 +- !18036 [OLK-6.6] Fix CVE-2025-37922 +- !18035 [OLK-6.6] Fix CVE-2025-37935 +- !18034 [OLK-6.6] Fix CVE-2025-37931 +- !18033 [OLK-6.6] Fix CVE-2025-37901 +- !18028 [OLK-6.6] Fix CVE-2025-37987 +- !18027 [OLK-6.6] Fix CVE-2025-37983 +- crypto: hisilicon/qm - clear all VF configurations in the hardware +- Revert "crypto: hisilicon/qm - fix vfs_num error" +- crypto: hisilicon/qm - invalidate queues in use +- Revert "crypto: hisilicon/qm - invalidate queues in use" +- Revert "crypto: hisilicon/qm - clear the memory before enabling the device" +- crypto: hisilicon/zip - add hashjoin, gather, and UDMA data move features +- crypto: hisilicon/zip - add lz4 and lz77_only to algorithm sysfs +- crypto: hisilicon/sec2 - Fix false-positive warning of uninitialised qp_ctx +- !17807 Add workaround for HIP09/HIP10/HIP10C/HIP12 erratum 162100803/162200807/162400807/165010802 +- pcmcia: Add error handling for add_interval() in do_validate_mem() +- !18326 [OLK-6.6] Support Hygon TKM (Trusted Key Management) Guest live migration +- !18199 block: avoid possible overflow for chunk_sectors check in blk_stack_limits() +- !18131 loop: Avoid updating block size under exclusive owner +- !18006 scsi: libiscsi: Initialize iscsi_conn->dd_data only if memory is allocated +- !18402 kernel/sys.c: fix the racy usage of task_lock(tsk->group_leader) in sys_prlimit64() paths +- !18249 enfs: add new path state PM_STATE_UNSTABLE +- kernel/sys.c: fix the racy usage of task_lock(tsk->group_leader) in sys_prlimit64() paths +- Bluetooth: btrtl: Prevent potential NULL dereference +- scsi: target: target_core_configfs: Add length check to avoid buffer overflow +- nfs/enfs: cleanups in pm_set_path_state() +- nfs/enfs: prefer normal rpc transport over unstable one +- nfs/enfs: introduce reconnect time KUnit tests +- nfs/enfs: set PM_STATE_UNSTABLE if path is unstable +- nfs/enfs: introduce enum PM_STATE_UNSTABLE +- !18357 *** OLK-6.6-fix-CVE-2025-39787 *** +- !18374 locking/qspinlock: Fix CNA NUMA node setup before numa_store_cpu_info() +- !18362 umount performance fix +- e1000e: fix heap overflow in e1000_set_eeprom +- locking/qspinlock: Fix CNA NUMA node setup before numa_store_cpu_info() +- do_make_slave(): choose new master sanely +- change_mnt_propagation(): do_make_slave() is a no-op unless IS_MNT_SHARED() +- change_mnt_propagation() cleanups, step 1 +- soc: qcom: mdt_loader: Deal with zero e_shentsize +- soc: qcom: mdt_loader: Fix error return values in mdt_header_valid() +- soc: qcom: mdt_loader: Ensure we don't read past the ELF header +- crypto: ccp: ensure kvm_bind_vtkm is only executed once for csv guest +- crypto: ccp: support vtkm live migration +- arm64: Enable vmalloc-huge with ptdump +- arm64: Restrict pagetable teardown to avoid false warning +- fs: quota: create dedicated workqueue for quota_release_work +- ext4: preserve SB_I_VERSION on remount +- ext4: fix an off-by-one issue during moving extents +- mm: drop all references of writable and SCAN_PAGE_RO +- mm: enable khugepaged anonymous collapse on non-writable regions +- khugepaged: pass folio instead of head page to trace events +- perf: arm_pmuv3: Don't use PMCCNTR_EL0 on SMT cores +- perf: arm_pmuv3: Factor out PMCCNTR_EL0 use conditions +- ras: ACPI: HED: Always initialize before evged +- fuse: fix issues in fuse_resend_pqueue() implementation +- fuse: Introduce sysfs API for resend pending requests +- fuse: Introduce sysfs API for flushing pending requests +- PCI: pnv_php: Clean up allocated IRQs on unplug +- block: avoid possible overflow for chunk_sectors check in blk_stack_limits() +- EDAC/{skx_common,i10nm}: Use scnprintf() for safer buffer handling +- EDAC/i10nm: Add Intel Granite Rapids-D support +- EDAC/i10nm: Fix the bitwise operation between variables of different sizes +- EDAC/{skx_common,i10nm}: Add RRL support for Intel Granite Rapids server +- EDAC/{skx_common,i10nm}: Refactor show_retry_rd_err_log() +- EDAC/{skx_common,i10nm}: Refactor enable_retry_rd_err_log() +- EDAC/{skx_common,i10nm}: Structure the per-channel RRL registers +- EDAC/i10nm: Explicitly set the modes of the RRL register sets +- EDAC/{i10nm,skx,skx_common}: Support UV systems +- EDAC/i10nm: Add Intel Clearwater Forest server support +- EDAC/{skx_common,i10nm}: Remove the AMAP register for determing DDR5 +- EDAC/{skx_common,skx,i10nm}: Move the common debug code to skx_common +- EDAC/{skx_common,i10nm}: Fix the loss of saved RRL for HBM pseudo channel 0 +- EDAC/{skx_common,i10nm}: Fix some missing error reports on Emerald Rapids +- EDAC/i10nm: Add Intel Grand Ridge micro-server support +- platform/x86/intel/ifs: Add Clearwater Forest to CPU support list +- fs/buffer: fix use-after-free when call bh_read() helper +- ext4: do not BUG when INLINE_DATA_FL lacks system.data xattr +- loop: Avoid updating block size under exclusive owner +- openeuler_defconfig: Add CONFIG_HISILICON_ERRATUM_162100803 +- irqchip/gicv3-its: Add workaround for HIP09/HIP10/HIP10C/HIP12 erratum 162100803/162200807/162400807/165010802 +- net/atm/resources: Release atm_dev_mutex after removing procfs in atm_dev_deregister(). +- net: ethernet: mtk-star-emac: fix spinlock recursion issues on rx/tx poll +- book3s64/radix : Align section vmemmap start address to PAGE_SIZE +- net: ethernet: mtk_eth_soc: fix SER panic with 4GB+ RAM +- btrfs: adjust subpage bit start based on sectorsize +- irqchip/qcom-mpm: Prevent crash when trying to handle non-wake GPIOs +- wifi: plfxlc: Remove erroneous assert in plfxlc_mac_release +- net_sched: ets: Fix double list add in class with netem as child qdisc +- pds_core: Prevent possible adminq overflow/stuck condition +- qibfs: fix _another_ leak +- scsi: libiscsi: Initialize iscsi_conn->dd_data only if memory is allocated +- bus: fsl-mc: fix double-free on mc_dev +- usb: gadget : fix use-after-free in composite_dev_cleanup() +- fbdev: imxfb: Check fb_add_videomode to prevent null-ptr-deref +- fs/ntfs3: cancle set bad inode after removing name fails +- f2fs: fix to bail out in get_new_segment() +- net: ravb: Fix missing rtnl lock in suspend/resume path +- wifi: ath12k: fix GCC_GCC_PCIE_HOT_RST definition for WCN7850 +- f2fs: fix to trigger foreground gc during f2fs_map_blocks() in lfs mode +- f2fs: fix to avoid panic in f2fs_evict_inode +- fscrypt: Don't use problematic non-inline crypto engines +- RNPGBE: NET: Fix dmesg 'unknow speed' show when 1000M +- perf vendor events arm64: Add V3 events/metrics +- perf vendor events arm64: Add N3 events/metrics +- perf vendor events arm64: Add FUJITSU-MONAKA PMU event +- perf vendor events arm64: Update N2/V2 events from source +- arm64/cpufeatures/kvm: Add ARMv8.9 FEAT_ECBHB bits in ID_AA64MMFR1 register + +* Fri Oct 17 2025 Mingzheng Xing - 6.6.0-112.0.0.91 +- RISC-V kernel upgrade to 6.6.0-112.0.0 +- Sync patches from the rvck-olk repository, including: + ACPI, IOMMU drivers; + Key RISC-V extensions support; + Add support for K1, SG2044, and DP1000. + +* Wed Oct 15 2025 Li Nan - 6.6.0-112.0.0.90 +- !18320 workqueue: Fix kabi broken of enum WORK_OFFQ_POOL_SHIFT +- !18336 soc cache: support L3 cache lock in framework +- !18352 net/oenetcls: Fix memleak when clean flow table +- !18306 vxlan: Fix NPD in {arp,neigh}_reduce() when using nexthop objects +- !18342 tracing: Silence warning when chunk allocation fails in trace_pid_write +- net/oenetcls: Fix memleak when clean flow table +- tracing: Silence warning when chunk allocation fails in trace_pid_write +- !18331 ACPI/IORT: Fix memory leak in iort_rmr_alloc_sids() +- !18330 cpufreq: Initialize cpufreq-based invariance before subsys +- soc cache: L3 cache lockdown support for HiSilicon SoC +- soc cache: support L3 cache lock in framework +- ACPI/IORT: Fix memory leak in iort_rmr_alloc_sids() +- cpufreq: Initialize cpufreq-based invariance before subsys +- !18275 [OLK-6.6] Optimized the kernel driver of HYGON trusted functions +- !18322 Fix CVE-2025-39715 +- !18293 CVE-2025-39886 +- !18290 vxlan: Fix NPD when refreshing an FDB entry with a nexthop object +- parisc: Revise gateway LWS calls to probe user read access +- workqueue: Fix kabi broken of enum WORK_OFFQ_POOL_SHIFT +- vxlan: Fix NPD in {arp,neigh}_reduce() when using nexthop objects +- !18286 mm: shmem: fix the strategy for the tmpfs 'huge=' options +- !18282 v2: shmem cleanup +- bpf: Tell memcg to use allow_spinning=false path in bpf_timer_init() +- memcg: skip cgroup_file_notify if spinning is not allowed +- !18291 CVE-2025-39844 +- !18031 Introduce fwctl subystem framework +- x86/mm/64: define ARCH_PAGE_TABLE_SYNC_MASK and arch_sync_kernel_mappings() +- mm: introduce and use {pgd,p4d}_populate_kernel() +- mm: move page table sync declarations to linux/pgtable.h +- vxlan: Fix NPD when refreshing an FDB entry with a nexthop object +- !18280 samples/bpf: fix cpuinfo_arm64 prog load issue +- crypto: ccp: optimize PSP driver code logic to reduce modifications to native code +- crypto: ccp: add "tdm_guard" as kernel and module parameter +- crypto: ccp: optimize TDM kernel driver +- mm: shmem: fix the strategy for the tmpfs 'huge=' options +- docs: update THP documentation to clarify sysfs "never" setting +- mm: shmem: drop the unnecessary folio_nr_pages() +- mm: shmem: use 'folio' for shmem_partial_swap_usage() +- samples/bpf: fix cpuinfo_arm64 prog load issue +- !18264 memory tiers: use default_dram_perf_ref_source in log message +- !18259 cgroup: split cgroup_destroy_wq into 3 workqueues +- !18231 ipv6: Fix memleak of nhc_pcpu_rth_output in fib_check_nh_v6_gw(). +- !18230 net: ch9200: fix uninitialised access during mii_nway_restart +- !18227 i40e: Fix potential invalid access when MAC list is empty +- memory tiers: use default_dram_perf_ref_source in log message +- !18250 mm/memory-failure: fix VM_BUG_ON_PAGE(PagePoisoned(page)) when unpoison memory +- !18253 samples/bpf: Fix print issue in bpf_rvi_diskstats +- !18234 samples/bpf: Fix print issue in dump_partitions() +- cgroup: split cgroup_destroy_wq into 3 workqueues +- fwctl: Fix repeated device word in log message +- fwctl: Add documentation +- fwctl: FWCTL_RPC to execute a Remote Procedure Call to device firmware +- taint: Add TAINT_FWCTL +- fwctl: FWCTL_INFO to return basic information about the device +- fwctl: Basic ioctl dispatch for the character device +- fwctl: Add basic structure for a class subsystem with a cdev +- samples/bpf: Fix print issue in bpf_rvi_diskstats +- mm/memory-failure: fix VM_BUG_ON_PAGE(PagePoisoned(page)) when unpoison memory +- !18173 usb: core: config: Prevent OOB read in SS endpoint companion parsing +- samples/bpf: Fix print issue in dump_partitions() +- ipv6: Fix memleak of nhc_pcpu_rth_output in fib_check_nh_v6_gw(). +- net: ch9200: fix uninitialised access during mii_nway_restart +- i40e: Fix potential invalid access when MAC list is empty +- !18179 Implement disable/enable for (delayed) work +- !18038 v3 arm64: Do not enable hardware xcall/xint in guest temporarily +- !18195 arm64/mpam: Update MB hardlimit and priority default value forcely +- !17636 [OLK-6.6] [Intel-SIG] crypto: iaa - Remove potential infinite loop in check_completion() +- !17858 [OLK-6.6] [Intel-SIG] Enable FLR for IDXD halt +- !17994 perf: backport community patches +- !18183 CVE-2025-38725 +- arm64/mpam: Update MB hardlimit and priority default value forcely +- dmaengine: idxd: Enable Function Level Reset (FLR) for halt +- dmaengine: idxd: Refactor halt handler +- dmaengine: idxd: Add idxd_device_config_save() and idxd_device_config_restore() helpers +- dmaengine: idxd: Binding and unbinding IDXD device and driver +- dmaengine: idxd: Add idxd_pci_probe_alloc() helper +- bitmap: Define a cleanup function for bitmaps +- !15977 use refcount+RCU method to implement lockless slab shrink +- !18166 Updates of HiSilicon Uncore L3C PMU +- net: usb: asix_devices: Fix PHY address mask in MDIO bus initialization +- net: usb: asix_devices: add phy_mask for ax88772 mdio bus +- !18156 ARM: rockchip: fix kernel hang during smp initialization +- workqueue: Fix UBSAN 'subtraction overflow' error in shift_and_mask() +- workqueue: Implement disable/enable for (delayed) work items +- workqueue: Preserve OFFQ bits in cancel[_sync] paths +- workqueue: Make @flags handling consistent across set_work_data() and friends +- workqueue: Remove clear_work_data() +- workqueue: Factor out work_grab_pending() from __cancel_work_sync() +- workqueue: Clean up enum work_bits and related constants +- workqueue: Introduce work_cancel_flags +- workqueue: Rename __cancel_work_timer() to __cancel_timer_sync() +- mm: add KABI_* macros to preserve KABI +- mm: shrinker add a copyright notice +- mm: shrinker: convert shrinker_rwsem to mutex +- mm: shrinker: hold write lock to reparent shrinker nr_deferred +- mm: shrinker: make memcg slab shrink lockless +- mm: shrinker: make global slab shrink lockless +- mm: shrinker: rename {prealloc|unregister}_memcg_shrinker() to shrinker_memcg_{alloc|remove}() +- mm: shrinker: add a secondary array for shrinker_info::{map, nr_deferred} +- mm: shrinker: remove old APIs +- mm: shrinker: vmalloc.c: use the new shrinker kAPI +- fs: super: dynamically allocate the s_shrink +- zsmalloc: dynamically allocate the mm-zspool shrinker +- xfs: dynamically allocate the xfs-qm shrinker +- xfs: dynamically allocate the xfs-inodegc shrinker +- xfs: dynamically allocate the xfs-buf shrinker +- nfsd: dynamically allocate the nfsd-reply shrinker +- nfsd: dynamically allocate the nfsd-client shrinker +- jbd2,ext4: dynamically allocate the jbd2-journal shrinker +- ext4: dynamically allocate the ext4-es shrinker +- mbcache: dynamically allocate the mbcache shrinker +- virtio_balloon: dynamically allocate the virtio-balloon shrinker +- vmw_balloon: dynamically allocate the vmw-balloon shrinker +- bcache: dynamically allocate the md-bcache shrinker +- md/raid5: dynamically allocate the md-raid5 shrinker +- dm zoned: dynamically allocate the dm-zoned-meta shrinker +- dm: dynamically allocate the dm-bufio shrinker +- drm/panfrost: dynamically allocate the drm-panfrost shrinker +- drm/msm: dynamically allocate the drm-msm_gem shrinker +- drm/i915: dynamically allocate the i915_gem_mm shrinker +- mm: workingset: dynamically allocate the mm-shadow shrinker +- sunrpc: dynamically allocate the sunrpc_cred shrinker +- mm: thp: dynamically allocate the thp-related shrinkers +- rcu: dynamically allocate the rcu-kfree shrinker +- rcu: dynamically allocate the rcu-lazy shrinker +- ubifs: dynamically allocate the ubifs-slab shrinker +- quota: dynamically allocate the dquota-cache shrinker +- nfsd: dynamically allocate the nfsd-filecache shrinker +- nfs: dynamically allocate the nfs-acl shrinker +- NFSv4.2: dynamically allocate the nfs-xattr shrinkers +- gfs2: dynamically allocate the gfs2-qd shrinker +- gfs2: dynamically allocate the gfs2-glock shrinker +- f2fs: dynamically allocate the f2fs-shrinker +- erofs: dynamically allocate the erofs-shrinker +- xenbus/backend: dynamically allocate the xen-backend shrinker +- drm/ttm: dynamically allocate the drm-ttm_pool shrinker +- binder: dynamically allocate the android-binder shrinker +- kvm: mmu: dynamically allocate the x86-mmu shrinker +- mm: shrinker: add infrastructure for dynamically allocating shrinker +- drm/ttm: introduce pool_shrink_rwsem +- mm: shrinker: remove redundant shrinker_rwsem in debugfs operations +- mm: vmscan: move shrinker-related code into a separate file +- mm: move some shrinker-related function declarations to mm/internal.h +- x86/boot: Compile boot code with -std=gnu11 too +- usb: core: config: Prevent OOB read in SS endpoint companion parsing +- !17949 crypto: hisilicon - some bugfix +- !18144 Backport fs/pipe performance optimization patches +- !17961 usb: gadget: configfs: Fix OOB read on empty string write +- !17960 drm/amd/pp: Fix potential NULL pointer dereference in atomctrl_initial… +- !17958 jfs: Prevent copying of nlink with value 0 from disk inode +- !17957 fs/ntfs3: Fix a couple integer overflows on 32bit systems +- !17956 drm/tegra: Fix a possible null pointer dereference +- !17954 mtd: spinand: fix memory leak of ECC engine conf +- !17953 regulator: gpio: Fix the out-of-bounds access to drvdata::gpiods +- !17952 rose: fix dangling neighbour pointers in rose_rt_device_down() +- Documentation: hisi-pmu: Add introduction to HiSilicon V3 PMU +- Documentation: hisi-pmu: Fix of minor format error +- drivers/perf: hisi: Add support for L3C PMU v3 +- Revert "drivers/perf: hisi: Add support for L3C PMU v3" +- Revert "Documentation: hisi-pmu: Fix of minor format error" +- Revert "Documentation: hisi-pmu: Add introduction to HiSilicon V3 PMU" +- !17973 enfs: recreate shard info when re-adding enfs module +- !18114 tee: fix NULL pointer dereference in tee_shm_put +- ARM: rockchip: fix kernel hang during smp initialization +- !18133 iommu/arm-smmu-qcom: Add SM6115 MDSS compatible +- fs/pipe: use spinlock in pipe_read() only if there is a watch_queue +- fs/pipe: remove unnecessary spinlock from pipe_write() +- !18129 soc: hisilicon: kunpeng_hccs: Fix incorrect log information +- iommu/arm-smmu-qcom: Add SM6115 MDSS compatible +- !18073 HID: asus: fix UAF via HID_CLAIMED_INPUT validation +- !18118 samples/bpf: Fix print issue in dump_swaps() +- soc: hisilicon: kunpeng_hccs: Fix incorrect log information +- !18123 arm64/mpam: Add quirk for hisi cpbm_wd field +- !18041 fs/smb: Fix inconsistent refcnt update +- !17932 [OLK-6.6] add support for WangXun 10G/25G/40G +- arm64/mpam: Add quirk for hisi cpbm_wd field +- !17665 cpufreq: intel_pstate: Support Clearwater Forest OOB mode +- !17663 platform/x86: ISST: Add Clearwater Forest to support list +- !17668 cpufreq: intel_pstate: Add Granite Rapids support in no-HWP mode +- !17681 cpufreq: intel_pstate: Update Balance-performance EPP for Granite Rapids +- !17468 [Intel-SIG] backporting KVM: x86: Advertise AVX10.1 CPUID to userspace +- !17571 [OLK-6.6] Intel-SIG: Add new Intel CPU# definition +- samples/bpf: Fix print issue in dump_swaps() +- !18024 trace/fgraph: Fix error handling +- !18061 net: bridge: fix soft lockup in br_multicast_query_expired() +- tee: fix NULL pointer dereference in tee_shm_put +- !18026 vsock/virtio: Validate length in packet header before skb_put() +- !18088 net/oenetcls: Fix crash when removing module +- net/oenetcls: Fix crash when removing module +- nfs/enfs: remove duplicate EOPNOTSUPP definition +- nfs/enfs: remove enfs_tp_common.h +- nfs/enfs: remove ping_execute_workq_lock +- nfs/enfs: remove lookupcache_workq_lock +- nfs/enfs: remove redundant flush_workqueue() before destroy_workqueue() +- nfs/enfs: return more nuanced error in NfsExtendProcInfoExtendEncode() and NfsExtendProcInfoExtendDecode() +- nfs/enfs: handle error returned by NfsExtendProcInfoExtendEncode() +- nfs/enfs: fix possible null-ptr-deref in exten_call.c +- nfs/enfs: free memory uniformly at the end of function in exten_call.c +- nfs/enfs: fix possible memory leak in exten_call.c +- nfs/enfs: reload config when re-adding enfs module +- nfs/enfs: recreate shard info when re-adding enfs module +- !18071 media: dvb-frontends: dib7090p: fix null-ptr-deref in dib7090p_rw_on_apb() +- HID: asus: fix UAF via HID_CLAIMED_INPUT validation +- media: dvb-frontends: dib7090p: fix null-ptr-deref in dib7090p_rw_on_apb() +- net: bridge: fix soft lockup in br_multicast_query_expired() +- !18054 arm64 kvm: On the Adaptation of CCA and virtCCA. +- arm64 kvm: On the Adaptation of CCA and virtCCA. +- usb: gadget: configfs: Fix OOB read on empty string write +- drm/amd/pp: Fix potential NULL pointer dereference in atomctrl_initialize_mc_reg_table +- jfs: Prevent copying of nlink with value 0 from disk inode +- fs/ntfs3: Fix a couple integer overflows on 32bit systems +- drm/tegra: Fix a possible null pointer dereference +- mtd: spinand: fix memory leak of ECC engine conf +- regulator: gpio: Fix the out-of-bounds access to drvdata::gpiods +- !17972 arm64/mpam: Add quirk for L3 CSU counters +- rose: fix dangling neighbour pointers in rose_rt_device_down() +- fs/smb: Fix inconsistent refcnt update +- !18016 media: uvcvideo: Fix 1-byte out-of-bounds read in uvc_parse_format() +- arm64: Do not enable hardware xcall/xint in guest temporarily +- !16579 Do not shatter hugezeropage on wp-fault +- !17839 [OLK-6.6] Fix CVE-2025-37838 +- !17887 [OLK-6.6] Fix CVE-2025-38637 +- !17889 [OLK-6.6] Fix CVE-2025-37840 +- !17850 [OLK-6.6] Fix CVE-2025-38173 +- !17854 [OLK-6.6] Fix CVE-2025-38143 +- !17861 [OLK-6.6] Fix CVE-2025-38194 +- vsock/virtio: Validate length in packet header before skb_put() +- arm64: cca_base: On the Adaptation of CCA and virtCCA. +- trace/fgraph: Fix error handling +- perf: Always feature test reallocarray +- perf comm str: Avoid sort during insert +- crypto: hisilicon - use kcalloc() instead of kzalloc() +- crypto: hisilicon/zip - enable literal length in stream mode compression +- crypto: hisilicon - check the sva module status while enabling or disabling address prefetch +- crypto: hisilicon - re-enable address prefetch after device resuming +- crypto: hisilicon/zip - remove unnecessary validation for high-performance mode configurations +- crypto: hisilicon/sec2 - implement full backlog mode for sec +- Revert "crypto: hisilicon/sec2 - fix memory use-after-free issue" +- Revert "crypto: hisilicon/sec2 - fix for gcm spec check" +- crypto: hisilicon - Use fine grained DMA mapping direction +- Revert "crypto: hisilicon - implement full backlog support for sec" +- crypto: hisilicon/qm - replace devm_kzalloc with devm_kcalloc +- crypto: hisilicon/qm - remove sizeof(char) +- !18015 arm64: cca_base: On the Adaptation of CCA and virtCCA. +- !18017 pNFS: Fix uninited ptr deref in block/scsi layout +- pNFS: Fix uninited ptr deref in block/scsi layout +- !18000 gfs2: Validate i_depth for exhash directories +- media: uvcvideo: Fix 1-byte out-of-bounds read in uvc_parse_format() +- !17997 hv_netvsc: Fix panic during namespace deletion with VF +- !17947 migration/hisilicon: some bugfix +- !17919 nfsd: handle get_client_locked() failure in nfsd4_setclientid_confirm() +- perf comm: Fix comm_str__put() for reference count checking +- perf comm: Add reference count checking to 'struct comm_str' +- gfs2: Validate i_depth for exhash directories +- hv_netvsc: Fix panic during namespace deletion with VF +- !17985 Updates of HiSilicon Uncore L3C PMU +- !17977 media: dvb-frontends: w7090p: fix null-ptr-deref in w7090p_tuner_write_serpar and w7090p_tuner_read_serpar +- !17987 virtCCA CoDA: Prevent secure PFs/VFs load or probe native drivers. +- virtCCA CoDA: Prevent secure PFs/VFs load or probe native drivers. +- Documentation: hisi-pmu: Add introduction to HiSilicon V3 PMU +- Documentation: hisi-pmu: Fix of minor format error +- drivers/perf: hisi: Add support for L3C PMU v3 +- !17974 iommu: Fix the incorrect exception process of the failed memory allocation +- !17983 fix oenetcls bugs +- net/oenetcls: clean resource when init ntuple/flow fail +- net/oenetcls: use workqueue for ntuple cfg +- net/oenetcls: remove oenetcls trace hook +- net/oenetcls: use raw_smp_processor_id() instead of smp_processor_id() +- !17898 enfs: fix some bugs and optimize performance +- media: dvb-frontends: w7090p: fix null-ptr-deref in w7090p_tuner_write_serpar and w7090p_tuner_read_serpar +- !17976 LoongArch: BPF: Fix jump offset calculation in tailcall +- iommu: Fix the incorrect exception process of the failed memory allocation +- LoongArch: BPF: Fix jump offset calculation in tailcall +- !17602 [OLK-6.6] drm/phytium: Fix make allmodconfig build fail +- jffs2: check that raw node were preallocated before writing summary +- net_sched: skbprio: Remove overly strict queue assertions +- crypto: marvell/cesa - Handle zero-length skcipher requests +- mtd: rawnand: brcmnand: fix PM resume warning +- HSI: ssi_protocol: Fix use after free vulnerability in ssi_protocol Driver Due to Race Condition +- backlight: pm8941: Add NULL check in wled_configure() +- !17792 virtCCA: Improve cvm performance. +- !17869 [OLK-6.6]Backport upstream CC Measurement support in the efi libstub +- !17862 [OLK-6.6]Hygon: Support RTMR and Attestation that includes RTMR for Hygon CSV3 VM +- !17946 mm/shmem, swap: bugfix and improvement of mTHP swap in +- arm64/mpam: Add quirk for L3 CSU counters +- !17969 arm64/mpam: Add quirk for cmax and cmin +- !17967 sched: Fix sched_numa_find_nth_cpu() if mask offline +- !17955 arm64: RME: handle RIPAS changes before kvm_rec_enter +- arm64: RME: handle RIPAS changes before kvm_rec_enter +- arm64: RME: Introduce kvm_rec_pre_enter() called before entering an atomic section +- !17962 tools/mpam: Add MPAM dynamic adjustment and sampling scripts +- arm64/mpam: Add quirk for cmax and cmin +- !17940 net/tls: fix kernel panic when alloc_page failed +- !17941 OLK-6.6-fix-CVE-2025-38147 +- sched: Fix sched_numa_find_nth_cpu() if mask offline +- tools/mpam: Add MPAM dynamic adjustment and sampling scripts +- !17939 pinctrl: qcom: msm: mark certain pins as invalid for interrupts +- !17945 Fix CVE-2025-38687 +- hisi_acc_vfio_pci: update device driver status +- migration: resolve duplicate migration states +- migration: fix VF reset timeout issue +- migration: bugfix live migration function without VF device driver +- mm/shmem, swap: fix major fault counting +- mm/shmem, swap: rework swap entry and index calculation for large swapin +- !17936 padata: Fix pd UAF once and for all +- mm/shmem, swap: simplify swapin path and result handling +- mm/shmem, swap: never use swap cache and readahead for SWP_SYNCHRONOUS_IO +- mm/shmem, swap: tidy up swap entry splitting +- !17879 sched: apply custom cpu range for idle cpu +- mm/shmem, swap: tidy up THP swapin checks +- mm/shmem, swap: avoid redundant Xarray lookup during swapin +- mm/shmem, swap: improve cached mTHP handling and fix potential hang +- mm/shmem, swap: fix softlockup with mTHP swapin +- x86/cpu/vfm: Fix x86_vfm kabi +- x86/cpu/vfm: Update arch/x86/include/asm/intel-family.h +- x86/cpu/vfm: Add/initialize x86_vfm field to struct cpuinfo_x86 +- x86/cpu: Add model number for another Intel Arrow Lake mobile processor +- net: wangxun: txgbe: add support for wangxun 10G/25G/40G +- comedi: fix race between polling and detaching +- !17937 clk: davinci: Add NULL check in davinci_lpsc_clk_register() +- Add IPI direct inject method for virtCCA CVM. +- Improve virtCCA CVM vtimer irq inject method. +- Add poll method for nfs for virtCCA CVM. +- calipso: unlock rcu before returning -EAFNOSUPPORT +- calipso: Don't call calipso functions for AF_INET sk. +- net/tls: fix kernel panic when alloc_page failed +- pinctrl: qcom: msm: mark certain pins as invalid for interrupts +- clk: davinci: Add NULL check in davinci_lpsc_clk_register() +- padata: Fix pd UAF once and for all +- !17853 v3 arm64: Fix actlr_el1 can not set problem in guest +- !17882 CVE-2025-38502 +- net: wangxun: remove wangxun txgbe inbox drivers +- !17922 x86/sgx: Prevent attempts to reclaim poisoned pages +- !17923 media: vidtv: Terminating the subsequent process of initialization failure +- !17924 [backport]KVM: arm64: Select default PMU in KVM_ARM_VCPU_INIT handler +- !17913 Bluetooth: hci_sync: fix double free in 'hci_discovery_filter_clear()' +- !17915 arm64/mpam: Half the number of RMIDs for all resources under CDP mode +- [backport]KVM: arm64: Select default PMU in KVM_ARM_VCPU_INIT handler +- media: vidtv: Terminating the subsequent process of initialization failure +- x86/sgx: Prevent attempts to reclaim poisoned pages +- nfsd: handle get_client_locked() failure in nfsd4_setclientid_confirm() +- !17848 [OLK-6.6]:update patches for sw64 architecture +- nfs/enfs: remove duplicate definitions +- nfs/enfs: fix some cleanup issues +- nfs/enfs: fix finding root uuid issue +- nfs/enfs: introduce is_enfs_debug() +- nfs/enfs: get rpc procedure number from rpc_procinfo in get_uuid_from_task() +- sunrpc: do not set enfs transport in rpc_task_set_client() +- virtCCA CoDA: Expose I/O hook interfaces. +- !17731 arm64/mpam: Add mb_only boot option +- !17911 CVE-2025-38305 +- !17244 Fix netns reference count issue +- !17914 virtCCA CoDA: Expose I/O hook interfaces. +- arm64/mpam: Half the number of RMIDs for all resources under CDP mode +- Bluetooth: hci_sync: fix double free in 'hci_discovery_filter_clear()' +- ptp: fix breakage after ptp_vclock_in_use() rework +- ptp: remove ptp->n_vclocks check logic in ptp_vclock_in_use() +- !17868 scsi: Remove scsi device no_start_on_resume flag +- !17866 md: make rdev_addable usable for rcu mode +- !17865 scsi: core: ufs: Fix a hang in the error handler +- !17842 [OLK-6.6] Fix CVE-2025-37924 +- !17836 [OLK-6.6] Fix CVE-2025-38312 +- !17902 [OLK-6.6] perf/zhaoxin/uncore: Enhance uncore support and fix related bugs +- !17880 [OLK-6.6] backport important bugfix for lpfc driver from upstream linux-6.6.y +- perf/zhaoxin/uncore: Enhance uncore support and fix related bugs +- !17890 LeapIOraid: fix strscpy size argument +- !17892 pptp: Fix skb length handling and error path in pptp_xmit() +- !17815 v2 ext4: better scalability for ext4 block allocation +- pptp: fix pptp_xmit() error path +- pptp: ensure minimal skb length in pptp_xmit() +- LeapIOraid: fix strscpy size argument +- !17883 net/mlx5e: Remove skb secpath if xfrm state is not found +- net/mlx5e: Remove skb secpath if xfrm state is not found +- Fix kabi breakage for bpf_map by using KABI_BROKEN_REPLACE and KABI_FILL_HOLE. +- bpf: Fix oob access in cgroup local storage +- bpf: Move cgroup iterator helpers to bpf.h +- bpf: Move bpf map owner out of common struct +- bpf: Add cookie object to bpf maps +- sched: apply custom cpu range for idle cpu +- !17756 [OLK-6.6] Provide host-side administrators/users with real-time access to the usage status of CSV3 CMA memory. +- !17846 VMCI: fix race between vmci_host_setup_notify and vmci_ctx_unset_notify +- !17442 scsi: target: Fix NULL pointer dereference in core_scsi3_decode_spec_i_port() +- scsi: lpfc: Check for hdwq null ptr when cleaning up lpfc_vport structure +- !17813 net/mlx5: Check device memory pointer before usage +- scsi: fix kabi broken for no_start_on_resume flag in struct scsi_device +- scsi: Remove scsi device no_start_on_resume flag +- md: make rdev_addable usable for rcu mode +- scsi: core: ufs: Fix a hang in the error handler +- efi/libstub: Add get_event_log() support for CC platforms +- efi/libstub: Measure into CC protocol if TCG2 protocol is absent +- efi/libstub: Add Confidential Computing (CC) measurement typedefs +- efi/tpm: Use symbolic GUID name from spec for final events table +- efi/libstub: Use TPM event typedefs from the TCG PC Client spec +- !17835 v2 perf/aux: Fix pending disable flow when the AUX ring buffer overruns +- virt: csv-guest: Add support for extended attestation aware request in the CSV3 guest +- virt: csv-guest: Implement CSV RTMR ioctl interfaces +- virt: csv-guest: Introduce CSV RTMR uapi interface +- x86/csv: Introduce RTMR interfaces +- sw64: fix for userspace asking for syscall(-1) +- sw64: update junzhang default config file +- sw64: kvm: introduce pmd_thp_or_huge for PMD hugepage checking +- sw64: add cross-page check in software unaligned write handling +- sw64: ptrace: fail system calls on invalid address access in PEEKUSR +- sw64: ptrace: fix a bug in return value handling in PEEKUSR AND POKEUSR +- sw64: perf: Add support for get_cpuid_str function +- sw64: kvm: use read/write spin lock for KVM MMU +- sw64: rethook: fix a bug in rethook_trampoline +- sw64: perf unwind: Do not put libunwind-sw_64 in FEATURE_TESTS_BASIC +- !17856 sdei_watchdog: add percpu flag to fix sdei watchdog state in lpi mode +- !17635 mm: madvise: use per_vma lock for MADV_FREE +- sdei_watchdog: add percpu flag to fix sdei watchdog state in lpi mode +- !17841 fs/resctrl: Fix max_threshold_occupancy write operation +- !17844 CVE-2025-37752 +- !17845 xfrm: interface: fix use-after-free after changing collect_md xfrm interface +- !17733 proc: use the same treatment to check proc_lseek as ones for proc_read_iter et.al +- arm64: Fix actlr_el1 can not set problem in guest +- !17834 cpufreq: scpi: Fix null-ptr-deref in scpi_cpufreq_get_rate() +- VMCI: fix race between vmci_host_setup_notify and vmci_ctx_unset_notify +- xfrm: interface: fix use-after-free after changing collect_md xfrm interface +- net_sched: sch_sfq: move the limit validation +- net_sched: sch_sfq: use a temporary work area for validating configuration +- !17840 arm64/mpam: Fix num_rmids when enable CDP mode +- ksmbd: fix use-after-free in kerberos authentication +- ksmbd: Fix dangling pointer in krb_authenticate +- fs/resctrl: Fix max_threshold_occupancy write operation +- arm64/mpam: Fix num_rmids when enable CDP mode +- !17838 mm/hugetlb: fix set_max_huge_pages() when there are surplus pages +- mm/hugetlb: fix set_max_huge_pages() when there are surplus pages +- !17587 CVE-2025-38406: wifi: ath6kl: remove WARN on bad firmware input +- !17585 CVE-2025-38408： genirq/irq_sim: Initialize work context pointers properly +- !17586 CVE-2025-38409: drm/msm: Fix another leak in the submit error path +- !17584 CVE-2025-38410: drm/msm: Fix a fence leak in submit error path +- !17582 NFC: nci: uart: Set tty->disc_data only in success path +- !17590 wifi: carl9170: do not ping device which has failed to load firmware +- !17593 net: lan743x: Modify the EEPROM and OTP size for PCI1xxxx devices +- !17594 CVE-2025-38425: i2c: tegra: check msg length in SMBUS block read +- !17595 Input: ims-pcu - check record size in ims_pcu_flash_firmware() +- !17596 usb: gadget: u_serial: Fix race condition in TTY wakeup +- !17583 CVE-2025-38467：drm/exynos: exynos7_drm_decon: add vblank check in IRQ handling +- fbdev: core: fbcvt: avoid division by 0 in fb_cvt_hperiod() +- perf/aux: Fix pending disable flow when the AUX ring buffer overruns +- cpufreq: scpi: Fix null-ptr-deref in scpi_cpufreq_get_rate() +- !17789 bpf, sockmap: Fix CVE-2025-38154 +- !17712 wifi: brcm80211: fmac: Add error handling for brcmf_usb_dl_writeimage() +- !17790 page_pool: Fix use-after-free in page_pool_recycle_in_ring +- !17788 netfilter: nft_set_pipapo: clamp maximum map bucket size to INT_MAX +- !17786 net: xfrm: Fix CVE-2025-38097 +- !17784 bpf, sockmap: Fix panic when calling skb_linearize +- !17785 bpf: fix ktls panic with sockmap +- !17783 net/sched: fix use-after-free in taprio_dev_notifier +- !17781 mlxsw: spectrum_router: Fix use-after-free when deleting GRE net devices +- !17782 net/mlx5e: Disable MACsec offload for uplink representor profile +- !17780 RDMA/core: Fix "KASAN: slab-use-after-free Read in ib_register_device" problem +- !17779 RDMA/rxe: Fix slab-use-after-free Read in rxe_queue_cleanup bug +- !17778 phy: tegra: xusb: remove a stray unlock +- !17777 net/mlx5e: Avoid WARN_ON when configuring MQPRIO with HTB offload enabled +- !17776 dmaengine: ti: k3-udma-glue: Drop skip_fdq argument from k3_udma_glue_reset_rx_chn +- !17775 espintcp: fix skb leaks +- !17774 can: bcm: add locking for bcm_op runtime updates +- !17711 net: lan743x: Fix memleak issue when GSO enabled +- !17708 net: phy: Fix CVE-2025-37945 +- !17707 bpf: Scrub packet on bpf_redirect_peer +- !17703 9p/net: fix improper handling of bogus negative read/write replies +- !17699 netfilter: socket: Lookup orig tuple for IPv6 SNAT +- !17685 gtp: Fix CVE-2025-21678 +- !17816 vmci: Prevent the dispatching of uninitialized payloads +- !17824 tracing/probes: Avoid using params uninitialized in parse_btf_arg() +- !17609 cpufreq: Exit governor when failed to start old governor +- Add dynamic wfx trap method for virtCCA CVM. +- !17823 arm64/mpam: Fix the logic that determines whether an MPAM MSC should initialize +- tracing/probes: Avoid using params uninitialized in parse_btf_arg() +- !17821 mm/hugetlb: fix surplus pages in dissolve_free_huge_page() +- arm64/mpam: Fix the logic that determines whether an MPAM MSC should initialize +- mm/hugetlb: fix surplus pages in dissolve_free_huge_page() +- !17818 fix some CVEs +- !17798 benet: fix BUG when creating VFs +- mm/vmscan: fix hwpoisoned large folio handling in shrink_folio_list +- mm/vmscan: don't try to reclaim hwpoison folio +- platform/x86: dell-wmi-sysman: Fix WMI data block retrieval in sysfs callbacks +- mm/hugetlb: fix huge_pmd_unshare() vs GUP-fast race +- memblock: Accept allocated memory before use in memblock_double_array() +- KVM: x86: Reset IRTE to host control if *new* route isn't postable +- scsi: mpi3mr: Fix possible crash when setting up bsg fails +- Input: synaptics - use guard notation when pausing serio port +- vmci: Prevent the dispatching of uninitialized payloads +- ext4: fix potential null deref in ext4_mb_init() +- ext4: implement linear-like traversal across order xarrays +- ext4: refactor choose group to scan group +- ext4: open coding repeated check in next_linear_group +- ext4: use correct criteria name instead stale integer number in comment +- ext4: remove unused parameter ngroup in ext4_mb_choose_next_group_*() +- ext4: convert free groups order lists to xarrays +- ext4: factor out ext4_mb_scan_group() +- ext4: factor out ext4_mb_might_prefetch() +- ext4: keep "prefetch_grp" and "nr" consistent +- ext4: remove unused ext4_allocation_context::ac_groups_considered +- ext4: factor out __ext4_mb_scan_group() +- ext4: convert EXT4_B2C(sbi->s_stripe) users to EXT4_NUM_B2C +- ext4: fallback to complex scan if aligned scan doesn't work +- ext4: fix largest free orders lists corruption on mb_optimize_scan switch +- ext4: fix zombie groups in average fragment size lists +- ext4: merge freed extent with existing extents before insertion +- ext4: convert sbi->s_mb_free_pending to atomic_t +- ext4: make running and commit transaction have their own freed_data_list +- ext4: fix typo in CR_GOAL_LEN_SLOW comment +- ext4: get rid of some obsolete EXT4_MB_HINT flags +- ext4: utilize multiple global goals to reduce contention +- ext4: remove unnecessary s_md_lock on update s_mb_last_group +- ext4: remove unnecessary s_mb_last_start +- ext4: separate stream goal hits from s_bal_goals for better tracking +- Revert "ext4: move mb_last_[group|start] to ext4_inode_info" +- ext4: add ext4_try_lock_group() to skip busy groups +- net/mlx5: Check device memory pointer before usage +- !17804 pinmux: fix race causing mux_owner NULL with active mux_usecount +- !17800 gic: increase the arch_timer +- !17794 perf/core: Prevent VMA split of buffer mappings +- !17574 ROH: Add crc_err_cnt and retry_cnt mib statistics +- pinmux: fix race causing mux_owner NULL with active mux_usecount +- !17337 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.64-6.6.66 LTS Conflicts Patches +- !17797 v3 HiSock Redirect Framework +- gic: increase the arch_timer priority to avoid hardlockup +- benet: fix BUG when creating VFs +- !17768 net/oenetcls: introduce oenetcls for network optimization +- ROH: Add crc_err_cnt and retry_cnt mib statistics +- samples/bpf: Add HiSock Redirect sample +- openeuler_defconfig: Enable CONFIG_HISOCK +- bpf: Add bpf_*_change_dev kfunc +- bpf: Add bpf_xdp_set_ingress_dst kfunc +- bpf: Add hisock_xdp_buff wrapper for xdp_buff +- bpf: Add bpf_skops_get_ingress_dst kfunc +- bpf: Enable generic kfuncs for BPF_CGROUP_* programs +- bpf: Add HISOCK_EGRESS hook on network egress path +- bpf: Add BPF_PROG_TYPE_HISOCK prog type +- bpf: Add XDP_HISOCK_REDIRECT action +- bpf: Add CONFIG_HISOCK +- !17470 fix CVE-2025-38351 +- !17766 RDMA/hns: Fix bond not cleared when the configuration becomes invalid +- !17772 two CVE fix +- perf/core: Prevent VMA split of buffer mappings +- !17659 v2 sched: More flexible use of CPU quota when CPU is idle +- !17748 fix bug in hdbss +- page_pool: Fix use-after-free in page_pool_recycle_in_ring +- bpf, sockmap: Avoid using sk_socket after free when sending +- bpf, sockmap: fix duplicated data transmission +- netfilter: nft_set_pipapo: clamp maximum map bucket size to INT_MAX +- net: xfrm: Fix kabi breakage of struct xfrm_state +- espintcp: remove encap socket caching to avoid reference leak +- x86/csv: Add function to get free size of CSV3 CMA +- !17294 [OLK-6.6] backport important bugfix for lpfc driver from upstream linux-6.6.y +- !17714 can: bcm: add missing rcu read protection for procfs content +- !17713 net: phy: leds: fix memory leak +- !17717 ipvs: fix uninit-value for saddr in do_output_route4 +- !17706 RDMA/cma: Fix CVE-2025-37772 +- !17709 bnxt_en: Fix CVE-2025-37911 +- !17705 igc: fix PTM cycle trigger logic +- !17718 ice: Check VF VSI Pointer Value in ice_vc_add_fdir_fltr() +- !17701 net: openvswitch: fix nested key length validation in the set() action +- !17702 net: ppp: Add bound checking for skb data on ppp_sync_txmung +- !17704 page_pool: avoid infinite loop to schedule delayed worker +- !17700 wifi: mac80211: Purge vif txq in ieee80211_do_stop() +- !17697 netfilter: nft_tunnel: fix geneve_opt type confusion addition +- !17715 netlabel: Fix NULL pointer exception caused by CALIPSO on IPv4 sockets +- !17696 net: decrease cached dst counters in dst_release +- !17695 netfilter: nf_tables: don't unregister hook when table is dormant +- !17694 RDMA/mlx5: Fix mlx5_poll_one() cur_qp update flow +- !17693 drm/connector: Fix CVE-2025-22089 +- !17692 vmxnet3: unregister xdp rxq info in the reset path +- !17690 Bluetooth: Fix error code in chan_alloc_skb_cb() +- !17689 tcp: Fix CVE-2025-21864 +- !17741 Fix trace may OOB when trace_get_user failed +- !17742 i2c: qup: jump out of the loop in case of timeout +- !17678 dm cache: prevent BUG_ON by blocking retries on failed device resumes +- !17679 media: cx231xx: set device_caps for 417 +- bpf: fix ktls panic with sockmap +- bpf, sockmap: Fix panic when calling skb_linearize +- net/sched: fix use-after-free in taprio_dev_notifier +- net/mlx5e: Disable MACsec offload for uplink representor profile +- mlxsw: spectrum_router: Fix use-after-free when deleting GRE net devices +- RDMA/core: Fix "KASAN: slab-use-after-free Read in ib_register_device" problem +- RDMA/rxe: Fix slab-use-after-free Read in rxe_queue_cleanup bug +- phy: tegra: xusb: remove a stray unlock +- net/mlx5e: Avoid WARN_ON when configuring MQPRIO with HTB offload enabled +- dmaengine: ti: k3-udma-glue: Drop skip_fdq argument from k3_udma_glue_reset_rx_chn +- espintcp: fix skb leaks +- can: bcm: add locking for bcm_op runtime updates +- net: libwx: fix the using of Rx buffer DMA +- HID: quirks: Add quirk for 2 Chicony Electronics HP 5MP Cameras +- !9218 [OLK-6.6] i2c/zhaoxin: I2C controller driver enhancement and optimization +- !17755 arm64: perf: Add support for HIP12 hw metric +- !17763 mm/smaps: fix race between smaps_hugetlb_range and migration +- net/oenetcls: introduce oenetcls for network optimization +- RDMA/hns: Fix bond not cleared when the configuration becomes invalid +- !17744 trace/fgraph: Fix the warning caused by missing unregister notifier +- mm/smaps: fix race between smaps_hugetlb_range and migration +- fs/proc/task_mmu: convert smaps_hugetlb_range() to work on folios +- !17737 soc cache: Fix incorrect error path of ioctl +- !17660 cgroup/cpuset: Support dynamic affinity feature +- !17293 [OLK-6.6] backport important bugfix for Broadcom BCM57414 nic driver from upstream linux-6.6.y +- i2c: zhaoxin: Add support for Zhaoxin I2C controller +- !17747 fix some CVE +- !17751 iBMA: Fix veth soft lockup by adding DMA-pre-readl sync +- !17730 arm64/entry: Mask DAIF in cpu_switch_to(), call_on_irq_stack() +- !16545 KVM: arm64: fix memory leak in TLBI +- arm64: perf: Add support for HIP12 hw metric +- iBMA: Fix veth soft lockup by adding DMA-pre-readl sync +- !17716 hrtimers: Update new CPU's next event in hrtimers_cpu_dying() +- !17669 ftrace: Fix potential warning in trace_printk_seq during ftrace_dump +- !17674 ftrace: Also allocate and copy hash for reading of filter files +- !17725 perf/core: Don't leak AUX buffer refcount on allocation failure +- KVM: arm64: fix memory leak in HDBSS +- KVM: arm64: do not support hdbss in nvhe +- usb: xhci: Fix invalid pointer dereference in Etron workaround +- thunderbolt: Do not double dequeue a configuration request +- powerpc/powernv/memtrace: Fix out of bounds issue in memtrace mmap +- mm: memory-failure: update ttu flag inside unmap_poisoned_folio +- Bluetooth: btnxpuart: Fix kernel panic during FW release +- !17592 ice: add NULL check in eswitch lag check +- trace/fgraph: Fix the warning caused by missing unregister notifier +- i2c: qup: jump out of the loop in case of timeout +- tracing: Limit access to parser->buffer when trace_get_user failed +- tracing: Remove unneeded goto out logic +- !17720 crypto: lzo - Fix compression buffer overrun +- !17719 spi: spi-imx: Add check for spi_imx_setupxfer() +- !17687 ipmi:msghandler: Fix potential memory corruption in ipmi_create_user() +- soc cache: Fix incorrect error path of ioctl +- soc cache: Fix incorrect size validation +- soc cache: Enforce maintain type check +- !17683 v2 Fix for bpf samples and selftests +- !17724 mm/dpool: mark dpool hugetlb page as dirty in free_huge_folio() +- !17438 [OLK-6.6]:update patches for sw64 architecture +- !17652 wifi: mac80211: Set n_channels after allocating struct cfg80211_scan_request +- !17728 CVE-2025-38006 +- !17677 v2 Fix static check warning reported by coccinelle +- !17633 dmaengine: idxd: Check availability of workqueue allocated by idxd wq driver before using +- !17455 raid10: cleanup memleak at raid10_make_request +- !17453 md/raid1: Fix stack memory use after return in raid1_reshape +- proc: use the same treatment to check proc_lseek as ones for proc_read_iter et.al +- arm64/mpam: Add mb_only boot option +- arm64/entry: Mask DAIF in cpu_switch_to(), call_on_irq_stack() +- net: mctp: Don't access ifa_index when missing +- mctp: no longer rely on net->dev_index_head[] +- perf/core: Don't leak AUX buffer refcount on allocation failure +- mm/dpool: mark dpool hugetlb page as dirty in free_huge_folio() +- platform/x86: ISST: Add Clearwater Forest to support list +- cpufreq: intel_pstate: Support Clearwater Forest OOB mode +- cpufreq: intel_pstate: Support Granite Rapids and Sierra Forest OOB mode +- cpufreq: intel_pstate: Support Emerald Rapids OOB mode +- cpufreq: intel_pstate: Add Granite Rapids support in no-HWP mode +- cpufreq: intel_pstate: Update Balance-performance EPP for Granite Rapids +- !17606 drm/sched: Increment job count before swapping tail spsc queue +- !17671 bpf-rvi: pidns: fix null-pointer-derefence in pidns_update_load_tasks +- crypto: lzo - Fix compression buffer overrun +- spi: spi-imx: Add check for spi_imx_setupxfer() +- ice: Check VF VSI Pointer Value in ice_vc_add_fdir_fltr() +- ipvs: fix uninit-value for saddr in do_output_route4 +- hrtimers: Update new CPU's next event in hrtimers_cpu_dying() +- netlabel: Fix NULL pointer exception caused by CALIPSO on IPv4 sockets +- can: bcm: add missing rcu read protection for procfs content +- net: phy: leds: fix memory leak +- wifi: brcm80211: fmac: Add error handling for brcmf_usb_dl_writeimage() +- net: lan743x: Fix memleak issue when GSO enabled +- bnxt_en: Fix out-of-bound memcpy() during ethtool -w +- bnxt_en: Fix coredump logic to free allocated buffer +- genirq/irq_sim: Initialize work context pointers properly +- net: phy: allow MDIO bus PM ops to start/stop state machine for phylink-controlled PHY +- net: phy: move phy_link_change() prior to mdio_bus_phy_may_suspend() +- bpf: Scrub packet on bpf_redirect_peer +- RDMA/cma: Fix hang when cma_netevent_callback fails to queue_work +- RDMA/cma: Fix workqueue crash in cma_netevent_work_handler +- igc: fix PTM cycle trigger logic +- page_pool: avoid infinite loop to schedule delayed worker +- 9p/net: fix improper handling of bogus negative read/write replies +- net: ppp: Add bound checking for skb data on ppp_sync_txmung +- net: openvswitch: fix nested key length validation in the set() action +- wifi: mac80211: Purge vif txq in ieee80211_do_stop() +- netfilter: socket: Lookup orig tuple for IPv6 SNAT +- netfilter: nft_tunnel: fix geneve_opt type confusion addition +- net: decrease cached dst counters in dst_release +- !17632 perf/core: Exit early on perf_mmap() fail +- netfilter: nf_tables: don't unregister hook when table is dormant +- RDMA/mlx5: Fix mlx5_poll_one() cur_qp update flow +- drm/connector: fix kabi broken in struct ib_device +- RDMA/core: Don't expose hw_counters outside of init net namespace +- vmxnet3: unregister xdp rxq info in the reset path +- Bluetooth: Fix error code in chan_alloc_skb_cb() +- net: use __GENKSYMS__ to revert the kabi change +- tcp: drop secpath at the same time as we currently drop dst +- ipmi:msghandler: Fix potential memory corruption in ipmi_create_user() +- gtp: Suppress list corruption splat in gtp_net_exit_batch_rtnl(). +- gtp: Destroy device along with udp socket's netns dismantle. +- samples/bpf: Move bpf_rvi_* progs to bpf_rvi directory +- selftests/bpf: fix perf_event link info name_len assertion +- Revert "selftests/bpf: Use bpf_link__destroy in fill_link_info tests" +- Revert "selftests/bpf: Add cookies check for perf_event fill_link_info test" +- Revert "selftests/bpf: fix perf_event link info name_len assertion" +- !17650 net: phy: motorcomm: Add support for PHY LEDs on YT8521 +- !17634 HID: pidff: Fix null pointer dereference in pidff_find_fields +- media: cx231xx: set device_caps for 417 +- dm cache: prevent BUG_ON by blocking retries on failed device resumes +- ACPI / MPAM: Fix wrong IS_ERR() check in _parse_table() +- hiroce3: Use IS_ERR() to check the return value of kthread_run() +- ftrace: Also allocate and copy hash for reading of filter files +- bpf-rvi: pidns: fix null-pointer-derefence in pidns_update_load_tasks +- !17642 tipc: fix null-ptr-deref when acquiring remote ip of ethernet bearer +- !17641 net: clear the dst when changing skb protocol +- !17653 wifi: iwlwifi: fix debug actions order +- !17651 wifi: mt76: disable napi on driver removal +- !17647 CVE-2025-38052 +- ftrace: Fix potential warning in trace_printk_seq during ftrace_dump +- cgroup/cpuset: Support dynamic affinity feature +- sched: More flexible use of CPU quota when CPU is idle +- !17646 sunrpc: handle SVC_GARBAGE during svc auth processing as auth error +- !17648 CVE-2025-37920 +- !17643 net: Fix TOCTOU issue in sk_is_readable() +- !17644 bridge: mcast: Fix use-after-free during router port configuration +- !17645 CVE-2025-38465 +- !17600 two bugfix for arm64 support HVO +- !17649 Fix CVE-2025-21839 +- !17628 Fix CVE-2025-37780 +- !17629 Fix CVE-2025-37767 +- !17637 netfilter: flowtable: account for Ethernet header in nf_flow_pppoe_proto() +- !17638 tcp: Correct signedness in skb remaining space calculation +- !17639 tipc: Fix use-after-free in tipc_conn_close(). +- !17640 seg6: Fix validation of nexthop addresses +- wifi: iwlwifi: fix debug actions order +- wifi: mac80211: Set n_channels after allocating struct cfg80211_scan_request +- wifi: mt76: disable napi on driver removal +- net: phy: motorcomm: Add support for PHY LEDs on YT8521 +- !17608 mm/util: make memdup_user_nul() similar to memdup_user() +- net: hns3: change the function return type from int to bool +- KVM: x86: Load DR6 with guest value only before entering .vcpu_run() loop +- xsk: Fix kabi broken +- xsk: Fix race condition in AF_XDP generic RX path +- net: tipc: fix refcount warning in tipc_aead_encrypt +- net/tipc: fix slab-use-after-free Read in tipc_aead_encrypt_done +- sunrpc: handle SVC_GARBAGE during svc auth processing as auth error +- netlink: avoid infinite retry looping in netlink_unicast() +- netlink: make sure we allow at least one dump skb +- netlink: Fix rmem check in netlink_broadcast_deliver(). +- netlink: Fix wraparounds of sk->sk_rmem_alloc. +- bridge: mcast: Fix use-after-free during router port configuration +- net: Fix TOCTOU issue in sk_is_readable() +- tipc: fix null-ptr-deref when acquiring remote ip of ethernet bearer +- net: clear the dst when changing skb protocol +- seg6: Fix validation of nexthop addresses +- tipc: Fix use-after-free in tipc_conn_close(). +- tcp: Correct signedness in skb remaining space calculation +- netfilter: flowtable: account for Ethernet header in nf_flow_pppoe_proto() +- mm: fix the race between collapse and PT_RECLAIM under per-vma lock +- HID: pidff: Fix null pointer dereference in pidff_find_fields +- mm: madvise: use per_vma lock for MADV_FREE +- dmaengine: idxd: Check availability of workqueue allocated by idxd wq driver before using +- !17597 Delete PHYTIUM_CPU_PART_FTC862 macro +- perf/core: Exit early on perf_mmap() fail +- mm: use per_vma lock for MADV_DONTNEED +- drm/amd/pm/swsmu/smu13/smu_v13_0: Prevent division by zero +- isofs: Prevent the use of too small fid +- mm: madvise: use walk_page_range_vma() instead of walk_page_range() +- cpufreq: Exit governor when failed to start old governor +- cpufreq: Move the check of cpufreq_driver->get into cpufreq_verify_current_freq() +- cpufreq: Init policy->rwsem before it may be possibly used +- cpufreq: Initialize cpufreq-based frequency-invariance later +- cpufreq: Hold cpufreq_driver_lock when assigning cpufreq_driver->set_boost +- cpufreq: Remove duplicate check in __cpufreq_offline() +- cpufreq: Contain scaling_cur_freq.attr in cpufreq_attrs +- cpufreq: CPPC: Remove forward declaration of cppc_cpufreq_register_em() +- cpufreq: CPPC: Do not return a value from populate_efficiency_class() +- cpufreq: CPPC: Remove cpu_data_list +- cpufreq: CPPC: Remove forward declaration of hisi_cppc_cpufreq_get_rate() +- !17307 scsi: megaraid_sas: Fix invalid node index +- !17309 irqchip/gic-v2m: Prevent use after free of gicv2m_get_fwnode() +- mm/util: make memdup_user_nul() similar to memdup_user() +- drm/sched: Increment job count before swapping tail spsc queue +- !17575 net/sched: Fix CVE-2025-37890 +- cpufreq: cppc: Fix invalid return value in hisi_cppc_cpufreq_get_rate() +- cpufreq: cppc: Fix invalid return value in .get() callback +- drm/phytium: Fix Phytium DRM build fail +- arm64: mm: HVO: fix deadlock in split vmemmap pmd +- mm: HVO: fix hard lockup in split_vmemmap_huge_pmd under x86 +- cputype: phytium: Delete PHYTIUM_CPU_PART_FTC862 macro +- ice: add NULL check in eswitch lag check +- i2c: tegra: check msg length in SMBUS block read +- drm/msm: Fix another leak in the submit error path +- wifi: ath6kl: remove WARN on bad firmware input +- !16528 [OLK-6.6] dma: phytium: Add PSWIOTLB mechanism to improve DMA performance +- !17570 tracing: Add down_write(trace_event_sem) when adding trace event +- !17565 Enhance soft hwpoison handling and injection +- net/sched: ets: use old 'nbands' while purging unused classes +- net/sched: Always pass notifications when child class becomes empty +- net_sched: hfsc: Address reentrant enqueue adding class to eltree twice +- sch_hfsc: Fix qlen accounting bug when using peek in hfsc_enqueue() +- net_sched: hfsc: Fix a UAF vulnerability in class with netem as child qdisc +- !17569 drm/vkms: Fix use after free and double free on init error +- tracing: Add down_write(trace_event_sem) when adding trace event +- drm/vkms: Fix use after free and double free on init error +- !17564 bpf: Reject %p% format string in bprintf-like helpers +- !17510 PM / devfreq: Add HiSilicon uncore frequency scaling driver +- !17547 net_sched: drr: Fix double list add in class with netem as child qdisc +- !17556 filemap: move prefaulting out of hot write path +- mm/memory-failure: send SIGBUS in the event of thp split fail +- mm/memory-failure: move hwpoison_filter() higher up +- mm/memory-failure: improve memory failure action_result messages +- mm/madvise: add MF_ACTION_REQUIRED to madvise(MADV_HWPOISON) +- mm/memory-failure: try to send SIGBUS even if unmap failed +- PM / devfreq: Add HiSilicon uncore frequency scaling driver +- !17529 mm: optimize mremap() by PTE batching +- !17481 HID: core: fix __hid_request when no report ID is used +- bpf: Reject %p% format string in bprintf-like helpers +- crypto: iaa - Remove potential infinite loop in check_completion() +- !17495 v2 mm/mem_sampling: preserve kernel subsystem SPE state across perf sampling +- !17546 net_sched: qfq: Fix double list add in class with netem as child qdisc +- !17542 net: sch: Fix CVE-2025-37798 +- !17538 wifi: at76c50x: fix use after free access in at76_disconnect +- !17537 net: vlan: fix VLAN 0 refcount imbalance of toggling filtering during runtime +- !17553 RDMA/hns: Fix dip entries leak on devices newer than hip09 +- filemap: move prefaulting out of hot write path +- RDMA/hns: Fix dip entries leak on devices newer than hip09 +- RDMA/hns: Fix querying wrong SCC context for DIP algorithm +- !17479 fix some bugs of enfs +- !17530 sched/psi: change permissions of /proc/pressure/stat +- net_sched: drr: Fix double list add in class with netem as child qdisc +- net_sched: qfq: Fix double list add in class with netem as child qdisc +- mm/mremap: avoid expensive folio lookup on mremap folio pte batch +- codel: remove sch->q.qlen check before qdisc_tree_reduce_backlog() +- sch_htb: make htb_deactivate() idempotent +- sch_htb: make htb_qlen_notify() idempotent +- sch_drr: make drr_qlen_notify() idempotent +- sch_hfsc: make hfsc_qlen_notify() idempotent +- sch_qfq: make qfq_qlen_notify() idempotent +- sch_ets: make est_qlen_notify() idempotent +- wifi: at76c50x: fix use after free access in at76_disconnect +- net: vlan: Fix kabi breakage of struct vlan_info +- net: vlan: fix VLAN 0 refcount imbalance of toggling filtering during runtime +- sched/psi: change permissions of /proc/pressure/stat +- mm: optimize mremap() by PTE batching +- !17526 v7 High Performance Container Resource View Isolation -- Stage 2 +- !17515 gpiolib: Get rid of never false gpio_is_valid() calls +- mm: call pointers to ptes as ptep +- !17519 CVE-2025-38473 +- !17524 CVE-2025-38375 +- !17517 CVE-2025-38251 +- !17523 CVE-2025-38211 +- samples/bpf: Add iterator program for meminfo +- bpf-rvi: Add bpf_x86_direct_pages kfunc +- bpf-rvi: Add bpf_mem_vmalloc_{used,total} kfunc +- bpf-rvi: Add bpf_mem_commit_limit & bpf_mem_committed kfunc +- bpf-rvi: Add bpf_mem_percpu kfunc +- bpf-rvi: Add bpf_mem_failure kfunc +- bpf-rvi: Add bpf_hugetlb_report_meminfo kfunc +- bpf-rvi: Add bpf_mem_{total,free}cma kfunc +- bpf-rvi: Add bpf_mem_{file_{hugepage,pmdmapped},kreclaimable} kfunc +- samples/bpf: Add iterator program for stat +- bpf-rvi: stat: Add stat iterator target +- bpf-rvi: Add kstat_ & kcpustat_ kfuncs +- bpf-rvi: Add cpu runqueue related kfuncs +- bpf-rvi: Add stat-related misc kfuncs +- bpf-rvi: Add bpf_seq_file_append() kfunc +- samples/bpf: Add iterator program for swaps +- bpf-rvi: Add bpf_page_counter_read() kfunc +- bpf-rvi: Add bpf_si_memswinfo() kfunc +- samples/bpf: Add iterator program for uptime +- bpf-rvi: cpuacct: Add bpf_task_ca_cpuusage() kfunc +- samples/bpf: Add iterator program for loadavg +- bpf-rvi: pidns: Add for_each_task_in_pidns and loadavg-related kfuncs +- bpf-rvi: pidns: Calculate loadavg for each pid namespace +- samples/bpf: Add iterator program for partitions +- bpf-rvi: block: Look up /dev in reaper's fs->root and filter partitions +- bpf-rvi: block: Add partitions iterator target +- samples/bpf: Add iterator program for diskstats +- bpf-rvi: blk-cgroup: Add bpf_blkcg_get_dev_iostat() kfunc +- bpf-rvi: block: Add diskstats iterator target +- samples/bpf: Add iterator program for cpuinfo_arm64 +- bpf-rvi: arm64: Add bpf_arch_flags kunc for arm64 +- bpf-rvi: arm64: Add cpuinfo_arm64 iterator target +- bpf-rvi: arm64: Add bpf_arm64_cpu_have_feature() kfunc +- bpf-rvi: cpuacct: Add task_cpuacct() +- bpf-rvi: cpuacct: Add bpf_cpuacct_kcpustat_cpu_fetch kfunc +- bpf-rvi: proc: add bpf_get_{idle,iowait}_time kfunc +- bpf-rvi: cgroup: Add cgroup_rstat_flush_atomic() kfunc +- bpf-rvi: Add bpf_mem_cgroup_from_task() kfunc +- bpf-rvi: cpuset: Fix missing of return for !tsk in task_effective_cpumask() +- virtio-net: ensure the received length does not exceed allocated size +- RDMA/iwcm: Fix use-after-free of work objects after cm_id destruction +- Bluetooth: Fix null-ptr-deref in l2cap_sock_resume_cb() +- nfs/enfs: make some functions static in enfs_multipath_client.c +- nfs/enfs: remove enfs_init() and enfs_fini() +- nfs/enfs: format get_ip_to_str() in shard_route.c +- nfs/enfs: unlock uniformly at the end of function in shard_route.c +- nfs/enfs: support debugging ip and dns list +- nfs/enfs: fix error when showing dns list +- nfs/enfs: fix alignment between struct rpc_clnt and rpc_clnt_reserve +- nfs/enfs: set CONFIG_SUNRPC_ENFS=y by default +- nfs/enfs: fix memory leak of shard_view_ctrl when removing nfs module +- atm: clip: prevent NULL deref in clip_push() +- !17513 sched-ebpf-add-for-exec-init +- gpiolib: Get rid of never false gpio_is_valid() calls +- !17487 arm64/mpam: Support partial-core boot for MPAM +- !16844 LeapIOraid: add device and support fw log +- sched/bpf: Fix kabi breakage for including sched bpf header file +- sched/ebpf: Add kfunc to set the preferred NUMA node for the task +- sched/ebpf: Add bpf_sched_cfs_exec_set_cpumask hook +- !17512 CVE-2025-21868 +- !17498 sched: Support NUMA parallel scheduling for multiple processes +- !17508 Huawei iBMA: Added support for Hi1712 Chip +- hv_netvsc: Preserve contiguous PFN grouping in the page buffer array +- net: use __GENKSYMS__ to revert the kabi change +- net: allow small head cache usage with large MAX_SKB_FRAGS values +- Revert "PM / devfreq: Add HiSilicon uncore frequency scaling driver" +- Huawei iBMA: Added support for Hi1712 Chip +- !17501 net: hibmcge: fix rtnl deadlock issue +- net: hibmcge: fix rtnl deadlock issue +- net: hibmcge: fix the division by zero issue +- net: hibmcge: fix the np_link_fail error reporting issue +- !17459 ROH: fix variable type inconsistency issue +- sched: Support NUMA parallel scheduling for multiple processes +- !17183 rtnetlink: Allocate vfinfo size for VF GUIDs when supported +- !17180 atm: Fix NULL pointer dereference +- !16807 Support RME feature for CCA host +- mm/mem_sampling: preserve kernel subsystem SPE state across perf sampling +- !17488 arm64/mpam: Set 1 as the minimum setting value for CMAX +- arm64/mpam: Set 1 as the minimum setting value for CMAX +- arm64/mpam: Support partial-core boot for MPAM +- HID: core: do not bypass hid_hw_raw_request +- HID: core: ensure __hid_request reserves the report ID as the first byte +- HID: core: ensure the allocated report buffer can contain the reserved report ID +- !17440 sdei_watchdog: use lockup_detector_retry_init() to init sdei watchdog +- VirtCCA: Compatible with virtcca macro definitions +- CCA: Fix cca kabi conflict +- VirtCCA: Modify openeuler_defconfig +- Reapply "VirtCCA: cvm support UEFI boot" +- VirtCCA: Adapt virtcca operations by cca_base. +- VirtCCA: Adapt realm operations by cca_base. +- VirtCCA: Add CCA base operations. +- [v8-43-43]KVM: arm64: Allow activating realms +- [v8-42-43]KVM: arm64: Expose KVM_ARM_VCPU_REC to user space +- [v8-40-43]arm64: RME: Provide accurate register list +- [v8-39-43]arm64: RME: Provide register list for unfinalized RME RECs +- [v8-38-43]rm64: RME: Configure max SVE vector length for a Realm +- [v8-37-43]arm64: RME: Propagate max SVE vector length from RMM +- [v8-36-43]arm64: RME: Initialize PMCR.N with number counter supported by RMM +- [v8-35-43]arm64: RME: Set breakpoint parameters through SET_ONE_REG +- [v8-34-43]arm64: RME: Propagate number of breakpoints and watchpoints to userspace +- [v8-33-43]arm64: RME: Hide KVM_CAP_READONLY_MEM for realm guests +- [v8-32-43]arm64: rme: Enable PMU support with a realm guest +- [backport]KVM: arm64: PMU: Introduce helpers to set the guest's PMU +- [backport]KVM: arm64: PMU: Allow userspace to limit PMCR_EL0.N for the guest +- !17463 crypto: hisilicon/qm - clear the memory before enabling the device +- sw64: ptrace: return -EIO in invalid PEEKUSR/POKEUSR requests +- sw64: kvm: fix missing kvm_unregister_perf_callbacks +- sw64: kvm: fix compile error of kvm.ko +- sw64: kvm: remove the duplicated Kconfig selection of KVM +- sw64: cpuidle: add dedicated driver +- sw64: fix missing characters for sw64_rrk +- sw64: selftests/net: add sw64 support for io_uring_zerocopy_tx +- sw64: crypto: implement crc32 assembly optimization +- sw64: efi: remove meaningless code from asm/efi.h +- sw64: fix incorrect macro definitions for sw64 in bpf_tracing.h +- sw64: cpufreq: using PLL0 to provide clock for idle cores +- sw64: kvm: save the virtual address of irqs_pending in vcpucb +- sw64: kvm: fix undefined error with ftrace disabled +- KVM: x86/hyper-v: Skip non-canonical addresses during PV TLB flush +- KVM: x86: model canonical checks more precisely +- KVM: x86: Add X86EMUL_F_MSR and X86EMUL_F_DT_LOAD to aid canonical checks +- KVM: x86: Route non-canonical checks in emulator through emulate_ops +- KVM: x86: drop x86.h include from cpuid.h +- KVM: x86: Bury guest_cpuid_is_amd_or_hygon() in cpuid.c +- KVM: SVM: Emulate SYSENTER RIP/RSP behavior for all Intel compat vCPUs +- KVM: x86: Inhibit code #DBs in MOV-SS shadow for all Intel compat vCPUs +- KVM: x86: Apply Intel's TSC_AUX reserved-bit behavior to Intel compat vCPUs +- KVM: x86/pmu: Squash period for checkpointed events based on host HLE/RTM +- [backport]KVM: arm64: PMU: Set PMCR_EL0.N for vCPU based on the associated PMU +- [v8-31-43]arm_pmu: Provide a mechanism for disabling the physical IRQ +- [v8-30-43]arm64: RME: Prevent Device mappings for Realms +- [v8-29-43]arm64: RME: Always use 4k pages for realms +- [v8-28-43]arm64: RME: Allow checking SVE on VM instance +- [v8-27-43]arm64: RME: support RSI_HOST_CALL +- [v8-26-43]arm64: RME: allow userspace to inject aborts +- [v8-25-43]arm64: Don't expose stolen time for realm guests +- [v8-24-43]KVM: arm64: WARN on injected undef exceptions +- [v8-23-43]KVM: arm64: Handle Realm PSCI requests +- [v8-22-43]KVM: arm64: Validate register access for a Realm VM +- [v8-21-43]KVM: arm64: Handle realm VCPU load +- [v8-20-43]arm64: RME: Runtime faulting of memory +- [backport]KVM: arm64: Move pagetable definitions to common header +- [backport]rme: populate guest memory region without guest_memfd +- [v8-19-43]arm64: RME: Allow populating initial contents +- [v8-18-43]KVM: arm64: Handle realm MMIO emulation +- [v8-17-43]arm64: RME: Handle RMI_EXIT_RIPAS_CHANGE +- [v8-16-43]arm64: RME: Handle realm enter/exit +- [v8-15-43]arm64: RME: Allow VMM to set RIPAS +- [backport]KVM: Add member to struct kvm_gfn_range to indicate private/shared +- [v8-14-43]KVM: arm64: Support timers in realm RECs +- [v8-13-43]arm64: RME: Support for the VGIC in realms +- [backport]KVM: arm64: Force GICv3 trap activation when no irqchip is configured on VHE +- [v8-12-43]KVM: arm64: vgic: Provide helper for number of list registers +- [v8-11-43]arm64: RME: Allocate/free RECs to match vCPUs +- [backport]KVM: arm64: Add generic check for system-supported vCPU features +- !17467 arm-smmu-v3: adjust ecmdq log level +- KVM: x86: Advertise AVX10.1 CPUID to userspace +- arm-smmu-v3: adjust ecmdq log level +- crypto: hisilicon/qm - clear the memory before enabling the device +- [v8-10-43]arm64: RME: RTT tear down +- [v8-09-43]KVM: arm64: Allow passing machine type in KVM creation +- [v8-08-43]kvm: arm64: Don't expose debug capabilities for realm guests +- [v8-07-43]arm64: RME: ioctls to create and configure realms +- [v8-06-43]arm64: RME: Define the user ABI +- [v8-05-43]arm64: RME: Check for RME support at KVM init +- [v8-04-43]arm64: RME: Add wrappers for RMI calls +- [v8-03-43]arm64: RME: Add SMC definitions for calling the RMM +- [v8-02-43]arm64: RME: Handle Granule Protection Faults (GPFs) +- [v8-01-43]kvm: arm64: Include kvm_emulate.h in kvm/arm_psci.h +- Revert "kvm: add virtcca cvm host feature" +- Revert "VirtCCA: cvm support UEFI boot" +- Revert "pmu: enable pmu phys irq inject for cvm" +- !16720 [OLK-6.6] Not print information of SMMU 0x10 event and add phytium PS24080 SoC platform judgement +- !17458 v2 posix-cpu-timers: fix race between handle_posix_cpu_timers() and posix_cpu_timer_del() +- ROH: fix variable type inconsistency issue +- !17441 net: hibmcge: support for statistics of reset failures +- !17435 arm64/mm: Optimize loop to reduce redundant operations of contpte_ptep_get +- posix-cpu-timers: fix race between handle_posix_cpu_timers() and posix_cpu_timer_del() +- raid10: cleanup memleak at raid10_make_request +- md/raid1: Fix stack memory use after return in raid1_reshape +- !17385 mptcp: pm: only set fullmesh for subflow endp +- !17445 xdp: Reset bpf_redirect_info before running a xdp's BPF prog. +- !17444 Backport mainline patch +- !17395 Squashfs: check return result of sb_min_blocksize +- xdp: Reset bpf_redirect_info before running a xdp's BPF prog. +- bonding: check xdp prog when set bond mode +- bonding: return detailed error when loading native XDP fails +- scsi: target: Fix NULL pointer dereference in core_scsi3_decode_spec_i_port() +- net: hibmcge: support for statistics of reset failures +- sw64: kvm: enable dirty log gradually in small chunks for core4 +- sw64: fix guest kdump function +- sw64: ftrace: add HAVE_DYNAMIC_FTRACE_WITH_ARGS support +- sw64: use wait/complete for smp boot +- sw64: fix sw64_rrk_store() wrap handling +- sdei_watchdog: use lockup_detector_retry_init() to init sdei watchdog +- !17436 i2c/designware: Fix an initialization issue +- !17432 ACPICA: Refuse to evaluate a method if arguments are missing +- !17421 ACPICA: fix acpi operand cache leak in dswstate.c +- !17422 soc: aspeed: Add NULL check in aspeed_lpc_enable_snoop() +- !17423 net: stmmac: make sure that ptp_rate is not 0 before configuring EST +- !17301 fs/nfs/read: fix double-unlock bug in nfs_return_empty_folio() +- !17304 nfsd: nfsd4_spo_must_allow() must check this is a v4 compound request +- !17331 dm-bufio: fix sched in atomic context +- !17413 CVE-2025-37957 +- !17412 v3 livepatch: support for super-long symbol repositioning +- !16933 virtio_ring: Fix data race by tagging event_triggered as racy for KCSAN +- !16932 dm: fix unconditional IO throttle caused by REQ_PREFLUSH +- !17430 drm/v3d: Disable interrupts before resetting the GPU +- i2c/designware: Fix an initialization issue +- !17254 irqchip/gic-v3-its: Remove DevID Pool's restriction +- !17406 usb: net: sierra: check for no status endpoint +- arm64/mm: Optimize loop to reduce redundant operations of contpte_ptep_get +- ACPICA: Refuse to evaluate a method if arguments are missing +- drm/v3d: Disable interrupts before resetting the GPU +- !17393 selftests/tracing: Fix false failure of subsystem event test +- net: stmmac: make sure that ptp_rate is not 0 before configuring EST +- soc: aspeed: Add NULL check in aspeed_lpc_enable_snoop() +- ACPICA: fix acpi operand cache leak in dswstate.c +- KVM: SVM: Forcibly leave SMM mode on SHUTDOWN interception +- KVM: SVM: Update SEV-ES shutdown intercepts with more metadata +- livepatch: support for super-long symbol repositioning +- usb: net: sierra: check for no status endpoint +- !17380 bnxt_en: Set DMA unmap len correctly for XDP_REDIRECT +- mptcp: pm: only set fullmesh for subflow endp +- !17115 xfs: fix attr inactive issue +- Squashfs: check return result of sb_min_blocksize +- selftests/tracing: Fix false failure of subsystem event test +- !17390 ipvlan: Fix out-of-bound bugs in ipvlan_xmit_mode_l2e() +- !17387 ipvlan: ensure network headers are in skb linear part +- ipvlan: Fix out-of-bound bugs in ipvlan_xmit_mode_l2e() +- !17205 fix some panic bugs and memory leak bugs of enfs +- ipvlan: ensure network headers are in skb linear part +- !17271 LoongArch: KVM: Solve the problem of starting up the upstream qemu +- !17251 migration: update status after RAS error +- !17278 nvme: update firmware version after commit +- !17330 livepatch: fix insn length check +- irqchip/gic-v3-its: Remove DevID Pool's restriction +- !17372 drm/amd/display: Add null pointer check for get_first_active_display() +- !17349 bnxt: properly flush XDP redirect lists +- !17375 RDMA/mlx5: Initialize obj_event->obj_sub_list before xa_insert +- bnxt_en: Set DMA unmap len correctly for XDP_REDIRECT +- !17367 fd/proc: modify xcall proc file permissions to 640 +- !17346 calipso: Fix null-ptr-deref in calipso_req_{set,del}attr(). +- !17222 fs: Add additional checks for block devices during mount +- !17168 xfrm: state: fix out-of-bounds read during lookup +- RDMA/mlx5: Initialize obj_event->obj_sub_list before xa_insert +- !17141 CVE-2025-37992：net_sched: Flush gso_skb list too during ->change() +- !17364 net: usb: lan78xx: fix WARN in __netif_napi_del_locked on disconnect +- !17167 eth: bnxt: always recalculate features after XDP clearing, fix null-deref +- !17371 ext4: inline: fix len overflow in ext4_prepare_inline_data +- !17362 ucounts: fix mq_perf test error. +- drm/amd/display: Add null pointer check for get_first_active_display() +- ext4: inline: fix len overflow in ext4_prepare_inline_data +- fd/proc: modify xcall proc file permissions to 640 +- !17361 scsi: lpfc: Use memcpy() for BIOS version +- !17360 net: Fix CVE-2025-21884 +- !17356 serial: jsm: fix NPE during jsm_uart_port_init +- net: usb: lan78xx: fix WARN in __netif_napi_del_locked on disconnect +- ucounts: fix mq_perf test error. +- scsi: lpfc: Use memcpy() for BIOS version +- net: better track kernel sockets lifetime +- net: Add net_passive_inc() and net_passive_dec(). +- !17269 MPTCP UPTREAM PART26 +- !17358 serial: mctrl_gpio: split disable_ms into sync and no_sync APIs +- !17322 Fix CVE-2025-38424 +- serial: mctrl_gpio: split disable_ms into sync and no_sync APIs +- !17342 tls: always refresh the queue when reading sock +- !17343 Fix WARN in perf_cgroup_switch() +- !17080 serial: Fix potential null-ptr-deref in mlb_usio_probe() +- !17351 mm/dynamic_pool: Fix free_huge_pages undefflow problem +- !17319 mm/dpool: do not use uninitialized variable +- serial: jsm: fix NPE during jsm_uart_port_init +- nfs/enfs: remove unnecessary shard_should_stop in shard_route.c +- mm/dynamic_pool: Fix free_huge_pages undefflow problem +- !17345 net: hns3: use seq_file for files in tx_bd_info/ and rx_bd_info/ in debugfs +- bnxt: properly flush XDP redirect lists +- calipso: Fix null-ptr-deref in calipso_req_{set,del}attr(). +- net: hns3: use seq_file for files in tx_bd_info/ and rx_bd_info/ in debugfs +- net: hns3: use seq_file for files in common/ of hclge layer +- net: hns3: use seq_file for files in fd/ in debugfs +- !17328 perf: Revert to requiring CAP_SYS_ADMIN for uprobes +- perf/core: Fix the WARN_ON_ONCE is out of lock protected region +- perf/core: Fix WARN in perf_cgroup_switch() +- !17324 video: screen_info: Relocate framebuffers behind PCI bridges +- tls: always refresh the queue when reading sock +- !17341 v7 arm64: Add hardware xcall framework support +- !17336 fs/resctrl: Enqueue mon_event_count() forcefully for MPAM +- !17308 arm64: Fix guest app segment fault when disable_sdei_nmi_watchdog +- nvme-host: fix the updating of the firmware version +- nvme: update firmware version after commit +- arm64: Add hardware xcall framework support +- arm64: Reserve a kabi in task_struct exclusively for xcall +- arm64: Refactor the xcall proc code +- !17259 mpls: Use rcu_dereference_rtnl() in mpls_route_input_rcu(). +- !17335 ROH: fix security issues and eliminate redundancies +- ROH: fix security issues and eliminate redundancies +- ALSA: hda: Fix build error without CONFIG_SND_DEBUG +- ALSA: hda/conexant: Use the new codec SSID matching +- ALSA: hda: Use own quirk lookup helper +- arm64/mpam: Ensure the err variable is initialized +- fs/resctrl: Enqueue mon_event_count() forcefully for MPAM +- net: hns3: use seq_file for files in reg/ in debugfs +- net: hns3: use seq_file for files in mac_list/ in debugfs +- net: hns3: use seq_file for files in tm/ in debugfs +- net: hns3: use seq_file for files in common/ of hns3 layer +- net: hns3: use seq_file for files in queue/ in debugfs +- net: hns3: clean up the build warning in debugfs by use seq file +- net: hns3: remove tx spare info from debugfs. +- net: hns3: use string choices helper +- dm-bufio: fix sched in atomic context +- livepatch: fix insn length check +- perf: Revert to requiring CAP_SYS_ADMIN for uprobes +- video: screen_info: Relocate framebuffers behind PCI bridges +- LoongArch: KVM: Enhance the robustness of extioi +- perf/core: Fix WARN in perf_sigtrap() +- perf: Fix sample vs do_exit() +- !17261 Fixed the wrong debugfs node name in hisi_spi debugfs initialization +- mm/dpool: do not use uninitialized variable +- !17296 nvmet-tcp: don't restore null sk_state_change +- !17299 dmaengine: idxd: fix memory leak in error handling path of idxd_alloc +- !17310 usb: typec: altmodes/displayport: do not index invalid pin_assignments +- usb: typec: altmodes/displayport: do not index invalid pin_assignments +- mptcp: sysctl: blackhole timeout: avoid using current->nsproxy +- mptcp: blackhole only if 1st SYN retrans w/o MPC is accepted +- doc: mptcp: sysctl: blackhole_timeout is per-netns +- mptcp: sysctl: avail sched: remove write access +- !17305 CVE-2025-38264 +- !17303 sched/fair: Only increment deadline once on yield +- mptcp: remove unneeded lock when listing scheds +- nfs/enfs: remove enfs_uuid_debug in shard_route.c +- nfs/enfs: remove usage of list_entry_is_head() in shard_route.c +- sunrpc: remove redundant rpc_task_release_xprt() of enfs +- nfs/enfs: remove judgement about enfs_option in nfs_multipath_client_info_init() +- nfs/enfs: remove nfs_multipath_client_info_free_work() +- nfs/enfs: fix typos in shard_route.c +- nfs/enfs: remove unused functions in shard_route.c +- nfs/enfs: make some functions static in shard_route.c +- nfs/enfs: introduce DEFINE_PARSE_FH_FUNC to define parse_fh() +- nfs/enfs: fix memory leak when free view_table +- nfs/enfs: use DEFINE_CLEAR_LIST_FUNC to define enfs_clear_shard_view() +- nfs/enfs: introduce DEFINE_CLEAR_LIST_FUNC to define enfs_clear_fs_info() +- nfs/enfs: fix memory leak in enfs_delete_fs_info() +- nfs/enfs: fix double free of multipath_client_info +- nfs/enfs: fix null-ptr-deref in shard_update_work() +- irqchip/gic-v2m: Prevent use after free of gicv2m_get_fwnode() +- !17266 unify log function usage of enfs +- arm64: Fix guest app segment fault when disable_sdei_nmi_watchdog +- scsi: megaraid_sas: Fix invalid node index +- nvme-tcp: sanitize request list handling +- llist: add interface to check if a node is on a list. +- nfsd: nfsd4_spo_must_allow() must check this is a v4 compound request +- !17255 inspector: hold module refcnt when cpu inspector is running +- !17291 libbpf: Fix null pointer dereference in btf_dump__free on allocation failure +- sched/fair: Only increment deadline once on yield +- nfs/enfs: use enfs_log_error() instead of pr_err() in enfs +- nfs/enfs: use enfs_log_info() instead of pr_info() in enfs +- nfs/enfs: cleanups in enfs/shard_route.c +- nfs/enfs: use enfs_log_debug() instead of pr_debug() to debug enfs +- nfs/enfs: remove unused code in enfs/dns_internal.h +- nfs/enfs: use enfs_log_debug() instead of dfprintk() to debug enfs +- nfs: use dfprintk() to debug enfs +- fs/nfs/read: fix double-unlock bug in nfs_return_empty_folio() +- !17262 v2 ifs: Defer cgroup ifs enable +- !17287 i40e: fix MMIO write access to an invalid page in i40e_clear_hw +- dmaengine: idxd: fix memory leak in error handling path of idxd_alloc +- nvmet-tcp: don't restore null sk_state_change +- scsi: lpfc: Use memcpy() for BIOS version +- bnxt: properly flush XDP redirect lists +- LoongArch: KVM: Solve the problem of starting up the upstream qemu +- drivers/iommu: Fix the initialization failure issue of the iommu module +- LoongArch: KVM: Fix the extioi simulation error +- dma: phyitum: enable CONFIG_PHYTIUM_ERRATUM_FT3386 +- libbpf: Fix null pointer dereference in btf_dump__free on allocation failure +- dma: Fix kabi breakage due to "dma: pswiotlb: Move platform-identification to cpu errata" +- !17275 fs: export anon_inode_make_secure_inode() and fix secretmem LSM bypass +- i40e: fix MMIO write access to an invalid page in i40e_clear_hw +- !17273 atm: Revert atm_account_tx() if copy_from_iter_full() fails. +- fs: export anon_inode_make_secure_inode() and fix secretmem LSM bypass +- atm: Revert atm_account_tx() if copy_from_iter_full() fails. +- netlink: specs: mptcp: fix port endianness +- mptcp: disable active MPTCP in case of blackhole +- mptcp: include inet_common in mib.h +- mptcp: move mptcp_pm_gen.h's include +- mptcp: remove unnecessary else statements +- mptcp: prefer strscpy over strcpy +- mptcp: add net.mptcp.available_schedulers +- mptcp: sockopt: info: stop early if no buffer +- mptcp: fallback to TCP after SYN+MPC drops +- mptcp: export mptcp_subflow_early_fallback() +- selftests: mptcp: connect: remove duplicated spaces in TAP output +- selftests: mptcp: diag: remove trailing whitespace +- selftests: mptcp: test last time mptcp_info +- mptcp: add last time fields in mptcp_info +- selftests: mptcp: reset the last TS before the first test +- selftests: mptcp: netlink: drop disable=SC2086 +- selftests: mptcp: ip_mptcp option for more scripts +- selftests: mptcp: use pm_nl endpoint ops +- selftests: mptcp: export pm_nl endpoint ops +- selftests: mptcp: join: update endpoint ops +- selftests: mptcp: netlink: add change_address helper +- selftests: mptcp: add {get,format}_endpoint(s) helpers +- selftests: mptcp: netlink: add 'limits' helpers +- selftests: mptcp: add tc check for check_tools +- selftests: mptcp: connect: remote time in TAP output +- selftests: mptcp: lib: add time per subtests in TAP output +- !17246 MPTCP UPTREAM PART25 +- ifs: Defer cgroup ifs enable +- Fixed the wrong debugfs node name in hisi_spi debugfs initialization +- mpls: Use rcu_dereference_rtnl() in mpls_route_input_rcu(). +- inspector: hold module refcnt when cpu inspector is running +- selftests: mptcp: join: validate event numbers(part 2) +- selftests: mptcp: pm_nl_ctl: remove re-definition +- selftests: mptcp: join: simplify checksum_tests +- selftests: mptcp: join: mute errors when ran in the background +- selftests: mptcp: join: specify host being checked +- selftests: mptcp: join: more explicit check name +- selftests: mptcp: join: validate MPJ SYN TX MIB counters +- selftests: mptcp: join: one line for join check +- selftests: mptcp: join: reduce join_nr params +- mptcp: MIB counters for sent MP_JOIN +- mptcp: pm: reduce entries iterations on connect +- mptcp: pm: fix backup support in signal endpoints +- mptcp: pm: rename helpers linked to 'flush' +- mptcp: Remove unused declaration mptcp_sockopt_sync() +- !17250 arm64/mpam: Fix incorrect CPU-to-partid binding under CDP mode +- !17253 mm: fix uprobe pte be overwritten when expanding vma +- !16602 【OLK-6.6】修复虚拟机下使用海光TKM功能可能返回TKM_UNSUPPORTED_CMD错误 +- !16797 【OLK-6.6】修复psp_mutex_lock_timeout超时触发soft lockup警告的问题 +- mm: fix uprobe pte be overwritten when expanding vma +- arm64/mpam: Fix incorrect CPU-to-partid binding under CDP mode +- arm64/mpam: Fix typo in resctrl_arch_set_cdp_enabled() +- migration: update status after RAS error +- Revert "migration: update status after RAS error" +- !17076 smb: client: fix use-after-free in crypt_message when using async crypto +- !17102 smb: client: fix use-after-free in cifs_oplock_break +- smb: client: fix netns refcount leak after net_passive changes +- Revert "smb: client: fix TCP timers deadlock after rmmod" +- Revert "smb: client: Fix netns refcount imbalance causing leaks and use-after-free" +- !17240 some fix for spe hotcold +- !17159 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.72-6.6.75 LTS Conflicts Patches +- !17206 driver core: Use kasprintf() instead of fixed buffer formatting +- mm_monitor/mm_spe: Embed boost events within SPE packets +- mm_monitor/mm_spe: Fix SPE boost enablement and continuation handling +- !17161 olk-6.6-backport-bugfix-for-dynamic-affinity +- !17150 HID: usbhid: Eliminate recurrent out-of-bounds bug in usbhid_parse() +- !17153 f2fs: prevent kernel warning due to negative i_nlink from corrupted image +- !17154 media: vivid: Change the siize of the composing +- !17238 v2 net: Fix kabi breakage for defer_free_list in struct net +- !17237 RDMA/hns: Fix HW configurations not cleared in error flow +- net: Fix kabi breakage for defer_free_list in struct net +- RDMA/hns: Fix HW configurations not cleared in error flow +- RDMA/hns: Fix -Wframe-larger-than issue +- RDMA/hns: Drop GFP_NOWARN +- RDMA/hns: Fix accessing uninitialized resources +- RDMA/hns: Remove redundant hnae3.h in bond and sysfs +- RDMA/hns: Use __free_page() to free pages allocated with alloc_page() +- RDMA/hns: Fix divide-by-zero error in dca debugfs +- RDMA/hns: Add check between dca_min_size and dca_max_size +- RDMA/hns: Get messgae length of ack_req from FW +- !17229 net: hns3: fix spelling mistake "reg_um" -> "reg_num" +- !17032 fbcon: Make sure modelist not set on unregistered console +- !16958 scsi: target: iscsi: Fix timeout on deleted connection +- drivers/crypto/ccp: fix soft lockup warning in psp_mutex_lock_timeout() +- net: hns3: fix spelling mistake "reg_um" -> "reg_num" +- net: hns3: add complete parentheses for some macros +- net: hns3: use hns3_get_ops() helper to reduce the unnecessary middle layer conversion +- net: hns3: use hns3_get_ae_dev() helper to reduce the unnecessary middle layer conversion +- net: hns3: delete redundant address before the array +- net: hns3: clear hns alarm: comparison of integer expressions of different signedness +- net: hns3: add +- Revert "net: hns3: add +- dma: pswiotlb: Move platform-identification to cpu errata +- !17201 net: hns3: reduce stack usage in hclge_dbg_dump_tm_pri() +- !17215 net_sched: sch_sfq: fix a potential crash on gso_skb handling +- !17214 netfilter: nf_set_pipapo_avx2: fix initial map fill +- !17213 net: fix udp gso skb_segment after pull from frag_list +- !17212 ice: fix Tx scheduler error handling in XDP callback +- !17207 net: pktgen: fix access outside of user given buffer in pktgen_thread_write() +- !17200 xen-netfront: handle NULL returned by xdp_convert_buff_to_frame() +- !17199 tipc: fix NULL pointer dereference in tipc_mon_reinit_self() +- !17198 bpf: consider that tail calls invalidate packet pointers +- !17197 net: stmmac: Fix accessing freed irq affinity_hint +- !17196 mptcp: fix NULL pointer in can_accept_new_subflow +- !17194 tipc: fix memory leak in tipc_link_xmit +- !17193 net: dsa: mv88e6xxx: avoid unregistering devlink regions which were never registered +- !17192 cxgb4: fix memory leak in cxgb4_init_ethtool_filters() error path +- !17184 sctp: add mutual exclusion in proc_sctp_do_udp_port() +- !17181 net: fix NULL pointer dereference in l3mdev_l3_rcv +- fs: Add additional checks for block devices during mount +- !17101 jbd2: fix data-race and null-ptr-deref in jbd2_journal_dirty_metadata() +- !17191 net: mctp: Set SOCK_RCU_FREE +- !17190 usbnet:fix NPE during rx_complete +- !17189 net: ibmveth: make veth_pool_store stop hanging +- !17188 arcnet: Add NULL check in com20020pci_probe() +- !17187 net: fix geneve_opt length integer overflow +- !17186 udp: Fix memory accounting leak. +- !17185 net: mvpp2: Prevent parser TCAM memory corruption +- !17182 net: Remove RTNL dance for SIOCBRADDIF and SIOCBRDELIF. +- !17179 net: dsa: free routing table on probe failure +- !17178 sctp: detect and prevent references to a freed transport in sendmsg +- !17177 net_sched: hfsc: Fix a UAF vulnerability in class handling +- !17176 net_sched: hfsc: Fix a potential UAF in hfsc_dequeue() too +- !17175 net: dsa: mv88e6xxx: fix -ENOENT when deleting VLANs and MST is unsupported +- !17174 net: dsa: clean up FDB, MDB, VLAN entries on unbind +- !17172 xsk: fix an integer overflow in xp_create_and_assign_umem() +- !17171 net: atm: fix use after free in lec_send() +- !17170 eth: bnxt: fix truesize for mb-xdp-pass case +- net_sched: sch_sfq: fix a potential crash on gso_skb handling +- netfilter: nf_set_pipapo_avx2: fix initial map fill +- net: fix udp gso skb_segment after pull from frag_list +- ice: fix Tx scheduler error handling in XDP callback +- net: pktgen: fix access outside of user given buffer in pktgen_thread_write() +- !17016 [OLK-6.6]:update patches for sw64 architecture +- driver core: Use kasprintf() instead of fixed buffer formatting +- !17074 ext4: only dirty folios when data journaling regular files +- !16951 wifi: ath9k_htc: Abort software beacon handling if disabled +- net: hns3: reduce stack usage in hclge_dbg_dump_tm_pri() +- net: hns3: Resolved the issue that the debugfs query result is inconsistent. +- Revert "net: hns3: Resolved the issue that the debugfs query result is inconsistent." +- net: hns3: initialize reset_timer before hclgevf_misc_irq_init() +- Revert "net: hns3: initialize reset_timer before hclgevf_misc_irq_init()" +- net: hns3: Use ARRAY_SIZE() to improve readability +- Revert "net: hns3: fix spelling mistake "reg_um" -> "reg_num"" +- xen-netfront: handle NULL returned by xdp_convert_buff_to_frame() +- tipc: fix NULL pointer dereference in tipc_mon_reinit_self() +- bpf: consider that tail calls invalidate packet pointers +- net: stmmac: Fix accessing freed irq affinity_hint +- mptcp: fix NULL pointer in can_accept_new_subflow +- tipc: fix memory leak in tipc_link_xmit +- net: dsa: mv88e6xxx: avoid unregistering devlink regions which were never registered +- cxgb4: fix memory leak in cxgb4_init_ethtool_filters() error path +- net: mctp: Set SOCK_RCU_FREE +- usbnet:fix NPE during rx_complete +- net: ibmveth: make veth_pool_store stop hanging +- arcnet: Add NULL check in com20020pci_probe() +- net: fix geneve_opt length integer overflow +- !16850 [OLK-6.6] Backport TDX HLT/SAFE-HLT fix +- udp: Fix memory accounting leak. +- net: mvpp2: Prevent parser TCAM memory corruption +- sctp: add mutual exclusion in proc_sctp_do_udp_port() +- rtnetlink: Allocate vfinfo size for VF GUIDs when supported +- net: Remove RTNL dance for SIOCBRADDIF and SIOCBRDELIF. +- net: fix NULL pointer dereference in l3mdev_l3_rcv +- atm: Fix NULL pointer dereference +- net: dsa: free routing table on probe failure +- sctp: detect and prevent references to a freed transport in sendmsg +- net_sched: hfsc: Fix a UAF vulnerability in class handling +- net_sched: hfsc: Fix a potential UAF in hfsc_dequeue() too +- net: dsa: mv88e6xxx: fix -ENOENT when deleting VLANs and MST is unsupported +- net: dsa: clean up FDB, MDB, VLAN entries on unbind +- Input: ims-pcu - check record size in ims_pcu_flash_firmware() +- usb: gadget: u_serial: Fix race condition in TTY wakeup +- net: lan743x: Modify the EEPROM and OTP size for PCI1xxxx devices +- wifi: carl9170: do not ping device which has failed to load firmware +- drm/msm: Fix a fence leak in submit error path +- !17137 [olk 6.6] net: hibmcge: backport some feature patches +- xsk: fix an integer overflow in xp_create_and_assign_umem() +- net: atm: fix use after free in lec_send() +- eth: bnxt: fix truesize for mb-xdp-pass case +- !17156 v3:add perpcu stock for kmem +- xfrm: state: fix out-of-bounds read during lookup +- eth: bnxt: always recalculate features after XDP clearing, fix null-deref +- scsi: ufs: core: Set default runtime/system PM levels before ufshcd_hba_init() +- libfs: Fix simple_offset_rename_exchange() +- shmem: Fix shmem_rename2() +- mac802154: check local interfaces before deleting sdata list +- hfs: Sanity check the root record +- !17109 efi: efi_get_memory_map -- increase map headroom +- !17148 Bluetooth: MGMT: Protect mgmt_pending list with its own lock +- sched/dynamic_affinity: Calculate cpu capacity in real time when realtime tasks are running on this cpu +- sched/dynamic_affinity: fix CI test alway select prefer_cpu problem +- sched/dynamic_affinity: fix preffered_cpu offline problem +- sched/fair: reset idlest_cpu if not meet the conditions for preferred CPU +- memcg: add stock for kmem [un]charging +- memcg: add CONFIG_MEMCG_KMEM_STOCK +- scsi: ufs: core: Honor runtime/system PM levels if set by host controller drivers +- selftests: tc-testing: reduce rshift value +- media: vivid: Change the siize of the composing +- f2fs: prevent kernel warning due to negative i_nlink from corrupted image +- !17124 crypto: algif_hash - fix double free in hash_accept +- HID: usbhid: Eliminate recurrent out-of-bounds bug in usbhid_parse() +- Bluetooth: MGMT: Protect mgmt_pending list with its own lock +- drm/exynos: exynos7_drm_decon: add vblank check in IRQ handling +- NFC: nci: uart: Set tty->disc_data only in success path +- net: hibmcge: configure FIFO thresholds according to the MAC controller documentation +- net: hibmcge: adjust the burst len configuration of the MAC controller to improve TX performance. +- net: hibmcge: support scenario without PHY +- net: phy: fixed_phy: constify status argument where possible +- net: phy: fixed_phy: remove irq argument from fixed_phy_register +- efi/libstub: Bump up EFI_MMAP_NR_SLACK_SLOTS to 32 +- net: phy: fixed_phy: remove irq argument from fixed_phy_add +- net: phy: fixed_phy: remove fixed_phy_register_with_gpiod +- !17140 v2 Cgroup-Based Interference Statistics followup +- net_sched: Flush gso_skb list too during ->change() +- !17041 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.70-6.6.72 LTS Conflicts Patches +- interference: Add distribution histogram support +- interference: Use hardware timer counter +- interference: Add mutex interference track support +- interference: Add spinlock interference track support +- interference: Add cgroup_ifs_enter/leave_lock helper +- interference: Disable irq track when irqtime disabled +- sw64: delete useless I/O access in VM and emulator +- sw64: kvm: fix perf kvm support +- sw64: Kconfig: enable HAS_IOPORT by default +- sw64: pci: remove zx200 dma mask reset +- sw64: fix for executing syscall(-1) +- sw64: acpi: select HAVE_ACPI_APEI when ACPI is set +- sw64: pci: add base address of devmn +- sw64: adjust the notrace in the code +- !17126 EDAC/skx_common: Fix general protection fault +- !17136 xcall xint 644 +- modify xcall proc file permissions to 640 +- modify xint proc file permissions to 640 +- dma: pswiotlb: Move pswiotlb dma functions behind dma_map_ops +- !17070 arm64/mpam: Add quirk for MPAM MSMON_MBWU monitor NRDY bit +- !17106 arm64/ptrace: Fix stack-out-of-bounds read in regs_get_kernel_stack_nth() +- !17120 v3 mm: batched unmap lazyfree large folios +- EDAC/skx_common: Fix general protection fault +- crypto: algif_hash - fix double free in hash_accept +- mm/rmap: fix potential out-of-bounds page table access during batched unmap +- mm: avoid splitting pmd for lazyfree pmd-mapped THP in try_to_unmap +- mm: support batched unmap for lazyfree large folios during reclamation +- mm: support tlbbatch flush for a range of PTEs +- mm: set folio swapbacked iff folios are dirty in try_to_unmap_one +- !16998 arm64/mpam: Fix L2 monitor issue under CDP mode +- xfs: handle attr node/leaf blocks atomically during inactive +- xfs: factor out xfs_attr3_leaf_init +- xfs: factor out xfs_da3_node_entry_remove +- Revert "xfs: factor out __xfs_da3_node_read()" +- Revert "xfs: atomic drop extent entries when inactiving attr" +- dma: phyitum: enable CONFIG_PSWIOTLB +- dma: Fix kabi breakage due to "dma: phytium: Add PSWIOTLB mechanism to improve DMA performance" +- dma: phytium: Add PSWIOTLB mechanism to improve DMA performance +- !17107 CVE-2025-38322 +- !16947 [OLK-6.6] Fix patches for X2avic to backport for OLK-6.6 +- !17092 drm/amd/display: Fix && vs || typos +- perf/x86/intel: Fix crash in icl_update_topdown_event() +- perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read +- perf/x86/intel: Apply static call for drain_pebs +- arm64/ptrace: Fix stack-out-of-bounds read in regs_get_kernel_stack_nth() +- !16884 fs: Backport four namespace fix patches +- !17094 config: Set TSM_REPORTS=m and ARM_CCA_GUEST=m +- smb: client: fix use-after-free in cifs_oplock_break +- jbd2: fix data-race and null-ptr-deref in jbd2_journal_dirty_metadata() +- config: Set TSM_REPORTS=m and ARM_CCA_GUEST=m +- drm/amd/display: Fix && vs || typos +- !17071 usb: dwc3: gadget: check that event count does not exceed event buffer length +- !17068 mcb: fix a double free bug in chameleon_parse_gdd() +- !17062 bnxt_en: Mask the bd_cnt field in the TX BD properly +- serial: Fix potential null-ptr-deref in mlb_usio_probe() +- !16956 v3 config: Enable CONFIG_VIRT_DRIVERS +- smb: client: fix use-after-free in crypt_message when using async crypto +- !16950 perf: Add support for SPE Data Source packet on HiSilicon HIP12 +- ext4: only dirty folios when data journaling regular files +- usb: dwc3: gadget: check that event count does not exceed event buffer length +- arm64/mpam: Add quirk for MPAM MSMON_MBWU monitor NRDY bit +- mcb: fix a double free bug in chameleon_parse_gdd() +- bnxt_en: Mask the bd_cnt field in the TX BD properly +- !17055 bpf lts backport +- !17054 bpf verifier fixes +- !17049 platform/x86: dell_rbu: Fix list usage +- !17059 media: cxusb: no longer judge rbuf when the write fails +- !16462 fix CVE-2025-22101 +- media: cxusb: no longer judge rbuf when the write fails +- bpf: Improve check_raw_mode_ok test for MEM_UNINIT-tagged types +- bpf: Fix bpf_strtol and bpf_strtoul helpers for 32bit +- selftests/bpf: Add tests with stack ptr register in conditional jmp +- bpf: Do not include stack ptr register in precision backtracking bookkeeping +- selftests/bpf: precision tracking test for BPF_NEG and BPF_END +- Fix kabi for TDX halt/safe-halt +- !17003 net_sched: red: fix a race in __red_change() +- !16747 LoongArch: Fix bugs in huge_pte_offset/__alloc_pages_slowpath/iommu +- platform/x86: dell_rbu: Fix list usage +- !17026 CVE-2025-37842 +- !17022 spi-rockchip: Fix register out of bounds access +- !17039 iio: light: opt3001: fix deadlock due to concurrent flag access +- !17036 parisc: Fix double SIGFPE crash +- !17001 Fix CVE-2025-38182 +- fs/Kconfig: make hugetlbfs a menuconfig +- !17034 fix CVE-2025-38118 +- !17033 LoongArch: Fix panic caused by NULL-PMD in huge_pte_offset() +- !17035 drm/amdkfd: Fix mode1 reset crash issue +- workqueue: Update lock debugging code +- memblock: use numa_valid_node() helper to check for invalid node ID +- !16792 v3 Support CCA Guest +- iio: light: opt3001: fix deadlock due to concurrent flag access +- !17023 bpf: Check rcu_read_lock_trace_held() in bpf_map_lookup_percpu_elem() +- parisc: Fix double SIGFPE crash +- drm/amdkfd: Fix mode1 reset crash issue +- Bluetooth: MGMT: Fix sparse errors +- Bluetooth: MGMT: Fix UAF on mgmt_remove_adv_monitor_complete +- perf mem: Count L2 HITM for c2c statistic +- perf arm-spe: Add support for SPE Data Source packet on HiSilicon HIP12 +- LoongArch: Fix panic caused by NULL-PMD in huge_pte_offset() +- arm64: cputype: Add cputype definition for HIP12 +- Revert "perf arm-spe: Add support for SPE Data Source packet on HiSilicon HIP12" +- fbcon: Make sure modelist not set on unregistered console +- spi: fsl-qspi: Fix double cleanup in probe error path +- spi: fsl-qspi: use devm function instead of driver remove +- !16991 hwmon: (asus-ec-sensors) check sensor index in read_string() +- !17014 v2 CVE-2025-38099 +- !17013 crypto: null - Use spin lock instead of mutex +- bpf: Check rcu_read_lock_trace_held() in bpf_map_lookup_percpu_elem() +- spi-rockchip: Fix register out of bounds access +- !16983 crypto: ecdsa - Harden against integer overflows in DIV_ROUND_UP() +- !16982 KVM: x86: Acquire SRCU in KVM_GET_MP_STATE to protect guest memory accesses +- !16957 f2fs: fix to do sanity check on sbi->total_valid_block_count +- !16999 octeon_ep: Fix host hang issue during device reboot +- !16995 f2fs: fix to avoid out-of-bounds access in f2fs_truncate_inode_blocks() +- !17000 v2 mm: respect mmap hint address when aligning for THP +- !17006 drivers/rapidio/rio_cm.c: prevent possible heap overwrite +- sw64: irqchip: update effective affinity for PINTC +- sw64: irqchip: add support for PINTC on all NUMA nodes +- sw64: cpufreq: optimize the granularity of cpufreq +- sw64: ftrace: restore the register storing direct call address +- sw64: kvm: fix undefined compile issues for kvm module +- sw64: pintc: fix guest pintc init +- sw64: set module region to the 2GB space below vmalloc area +- Bluetooth: btusb: Fix regression in the initialization of fake Bluetooth controllers +- Bluetooth: Disable SCO support if READ_VOICE_SETTING is unsupported/broken +- Bluetooth: Add quirk for broken READ_PAGE_SCAN_TYPE +- Bluetooth: Add quirk for broken READ_VOICE_SETTING +- crypto: null - Use spin lock instead of mutex +- !16467 drm/virtio: implement virtio_gpu_shutdown +- !16565 [OLK-6.6] backport important bugfix for nvme driver from upstream linux-6.6.y +- !16743 [OLK-6.6] backport important bugfix for lpfc driver from upstream linux-6.6.y +- !16902 cachefiles: Fix the incorrect return value in cachefiles_ondemand_fd_write_iter() +- !16994 net: cadence: macb: Fix a possible deadlock in macb_halt_tx. +- !16996 wifi: rtw88: fix the 'para' buffer size to avoid reading out of bounds +- drivers/rapidio/rio_cm.c: prevent possible heap overwrite +- net_sched: red: fix a race in __red_change() +- ublk: santizize the arguments from userspace when adding a device +- mm: respect mmap hint address when aligning for THP +- octeon_ep: Fix host hang issue during device reboot +- arm64/mpam: Fix L2 monitor issue under CDP mode +- wifi: rtw88: fix the 'para' buffer size to avoid reading out of bounds +- f2fs: fix to avoid out-of-bounds access in f2fs_truncate_inode_blocks() +- net: cadence: macb: Fix a possible deadlock in macb_halt_tx. +- !16988 Fix compile problem for psi_grained +- hwmon: (asus-ec-sensors) check sensor index in read_string() +- Fix compile problem for psi_grained +- crypto: ecdsa - Harden against integer overflows in DIV_ROUND_UP() +- KVM: x86: Acquire SRCU in KVM_GET_MP_STATE to protect guest memory accesses +- !16969 Fix CVE-2025-38170 +- !16967 net: openvswitch: Fix the dead loop of MPLS parse +- !16964 dma-buf: insert memory barrier before updating num_fences +- arm64/fpsimd: Discard stale CPU state when handling SME traps +- !16891 fix some build errors of enfs +- KVM: x86: Fix a comment inside __kvm_set_or_clear_apicv_inhibit() +- net: openvswitch: Fix the dead loop of MPLS parse +- dma-buf: insert memory barrier before updating num_fences +- !16557 [OLK-6.6] PCI/P2PDMA: Add Zhaoxin Zhaoxin bridges to the whitelist +- !16919 Backport mainline patches +- scsi: target: iscsi: Fix timeout on deleted connection +- f2fs: fix to do sanity check on sbi->total_valid_block_count +- config: Enable CONFIG_VIRT_DRIVERS +- !16567 [OLK-6.6]:update patches for sw64 architecture +- wifi: ath9k_htc: Abort software beacon handling if disabled +- KVM: x86: Only set APICV_INHIBIT_REASON_ABSENT if APICv is enabled +- iommu/amd: Do not flush IRTE when only updating isRun and destination fields +- !16897 CVE-2025-37784 +- !16922 v2 irqchip/gic-v3: Fix xint sleeping function called from invalid context +- !16926 nfs: Clean up /proc/net/rpc/nfs when nfs_fs_proc_net_init() fails. +- !16912 Unbinding VF drivers in virtCCA CoDA SR-IOV scenarios. +- !16908 OLK-6.6 Bperf +- !16603 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.83-6.6.84 LTS Patches +- virtio_ring: Fix data race by tagging event_triggered as racy for KCSAN +- dm: fix unconditional IO throttle caused by REQ_PREFLUSH +- nfs: Clean up /proc/net/rpc/nfs when nfs_fs_proc_net_init() fails. +- irqchip/gic-v3: Fix xint sleeping function called from invalid context +- !16906 rseq: Fix segfault on registration when rseq_cs is non-zero +- xfrm: Clear low order bits of ->flowi4_tos in decode_session4(). +- xfrm: policy: fix layer 4 flowi decoding +- xfrm: policy: replace session decode with flow dissector +- xfrm: move mark and oif flowi decode into common code +- xfrm: pass struct net to xfrm_decode_session wrappers +- !16898 libnvdimm/labels: Fix divide error in nd_label_data_init() +- !16903 some optimization patches for inet_diag dump +- Unbinding VF drivers in virtCCA CoDA SR-IOV scenarios. +- !16895 virtCCA supports SR-IOV in CoDA scenarios. +- perf stat: Fix incorrect display of bperf when event count is 0 +- perf stat: Increase perf_attr_map entries +- perf test: Use sqrtloop workload to test bperf event +- perf test stat_bpf_counter.sh: Stabilize the test results +- perf test bpf-counters: Add test for BPF event modifier +- perf stat: Support inherit events during fork() for bperf +- !16896 drm/v3d: Add job to pending list if the reset was skipped +- rseq: Fix segfault on registration when rseq_cs is non-zero +- inet_diag: skip over empty buckets +- sock_diag: remove sock_diag_mutex +- inet_diag: allow concurrent operations +- inet_diag: add module pointer to "struct inet_diag_handler" +- cachefiles: Fix the incorrect return value in cachefiles_ondemand_fd_write_iter() +- !16780 v2 mm/mem_sampling: add trace event for spe based damon record +- !16781 v4 mm/mem_sampling: fix some mem sampling issues +- libnvdimm/labels: Fix divide error in nd_label_data_init() +- net: ti: icss-iep: Fix possible NULL pointer dereference for perout request +- net: ti: icss-iep: Add phase offset configuration for perout signal +- net: ti: icss-iep: Add pwidth configuration for perout signal +- drm/v3d: Add job to pending list if the reset was skipped +- !16753 nfs: handle failure of nfs_get_lock_context in unlock path +- !16237 pds_core: handle unsupported PDS_CORE_CMD_FW_CONTROL result +- virtCCA supports SR-IOV in CoDA scenarios. +- !16556 [OLK-6.6] PCI: Supplement ACS quirk for more Zhaoxin Root Ports +- !16559 mm: mincore: use pte_batch_hint() to batch process large folios +- !16689 mm/madvise: batch tlb flushes for MADV_DONTNEED and MADV_FREE +- !16426 FIX CVE-2025-40014 +- !16601 add virtcca capability and kae vf interface +- sunrpc, nfs: CONFIG_ENFS should select CONFIG_SUNRPC_ENFS +- sunrpc, nfs: fix build errors when CONFIG_SUNRPC_ENFS=m && CONFIG_ENFS=m && CONFIG_NFS=y +- !16866 v2 mempolicy: optimize queue_folios_pte_range by PTE batching +- nfs: fix build errors when CONFIG_ENFS=m && CONFIG_NFS_FS=y +- nfs: fix enfs mount failure when CONFIG_ENFS=y +- clone_private_mnt(): make sure that caller has CAP_SYS_ADMIN in the right userns +- do_change_type(): refuse to operate on unmounted/not ours mounts +- fix propagation graph breakage by MOVE_MOUNT_SET_GROUP move_mount(2) +- path_overmount(): avoid false negatives +- !16865 pci:vga fix race condition in vga_arb_write +- !16770 [OLK-6.6] Intel: Backport to fix QuickAssist Technology(QAT) in-tree driver from v6.10 to v6.16 +- !16773 mm:userswap: change VM_USWAP_BIT to bit 61 +- x86/tdx: Fix arch_safe_halt() execution for TDX VMs +- x86/paravirt: Move halt paravirt calls under CONFIG_PARAVIRT +- sw64: adapt to the upstream changes in huge_ptep_get_and_clear +- sw64: sort arch/sw64/Kconfig +- sw64: iommu: implement map_pages/unmap_pages API +- !16820 v2 Fix some smmu bugs +- !16803 CVE-2025-38060 +- !16828 orangefs: Do not truncate file size +- !16812 perf hisi-ptt: Fix memory leak in lseek failure handling +- !16827 btrfs: correct the order of prelim_ref arguments in btrfs__prelim_ref +- !16775 [OLK-6.6]fix enfs bug +- !16861 arm64: kaslr: fix nokaslr cmdline parsing +- mempolicy: optimize queue_folios_pte_range by PTE batching +- !16804 HID: uclogic: Add NULL check in uclogic_input_configured() +- !16806 vhost-scsi: protect vq->log_used with vq->mutex +- !14552 [OLK-6.6]Arm64:KVM: Auto get the virtual_dev irq bypass's devid +- !16573 Backport 6.6.64-6.6.72 LTS +- !16757 objtool, media: dib8000: Prevent divide-by-zero in dib8000_set_dds() +- !16756 mm: pcp: increase pcp->free_count threshold to trigger free_high +- pci:vga fix race condition in vga_arb_write +- !16656 merge CVE patches into OLK-6.6 +- !16512 perf/x86/intel: Only check the group flag for X86 leader +- !16617 SELinux: Add check for the user data passed to kcalloc in hashtab_init +- !16748 arm64: xcall: Fix GPR touch as the original step +- !16778 drm/radeon: fix uninitialized size issue in radeon_vce_cs_parse() +- !16848 tracing: probes: Fix a possible race in trace_probe_log APIs +- !16842 ftrace: Fix UAF when lookup kallsym after ftrace disabled +- !16837 ftrace: Fix preemption accounting for stacktrace filter command +- !16845 bpf: Avoid __bpf_prog_ret0_warn when jit fails +- !16652 v2 CVE-2025-37948 +- !16791 ALSA: pcm: Fix race of buffer access at PCM OSS layer +- arm64: kaslr: fix nokaslr cmdline parsing +- !16851 drm/amd/display: Increase block_sequence array size +- drm/amd/display: Increase block_sequence array size +- tracing: probes: Fix a possible race in trace_probe_log APIs +- bpf: Avoid __bpf_prog_ret0_warn when jit fails +- LeapIOraid: add device and support fw log +- ftrace: Fix UAF when lookup kallsym after ftrace disabled +- ftrace: Fix preemption accounting for stacktrace filter command +- orangefs: Do not truncate file size +- btrfs: correct the order of prelim_ref arguments in btrfs__prelim_ref +- iommu/arm-smmu-v3: Fix the problematic platform detection logic +- iommu/arm-smmu-v3: Fix the extra atc tlb flush +- fix review issue +- vhost-scsi: protect vq->log_used with vq->mutex +- perf hisi-ptt: Fix memory leak in lseek failure handling +- HID: uclogic: Add NULL check in uclogic_input_configured() +- bpf: abort verification if env->cur_state->loop_entry != NULL +- bpf: copy_verifier_state() should copy 'loop_entry' field +- gicv3: add lpi support for virtcca cvm guest +- MAINTAINERS: Add CCA and pKVM CoCO guest support to the ARM64 entry +- configfs-tsm-report: Fix NULL dereference of tsm_ops +- arm64: Document Arm Confidential Compute +- virt: arm-cca-guest: TSM_REPORT support for realms +- mm/slab: Add __free() support for kvfree +- configfs-tsm: Introduce a shared ABI for attestation reports +- virt: coco: Add a coco/Makefile and coco/Kconfig +- arm64: realm: Use aliased addresses for device DMA to shared buffers +- dma: Introduce generic dma_addr_*crypted helpers +- dma: Fix encryption bit clearing for dma_to_phys +- rme: make sure realm guest map memory in page granularity +- arm64: realm: ioremap: Allow mapping memory as encrypted +- parisc: Delay write-protection until mark_rodata_ro() call +- jump_label,module: Don't alloc static_key_mod for __ro_after_init keys +- irqchip/gic-v3-its: Rely on genpool alignment +- irqchip/gic-v3-its: Fix over allocation in itt_alloc_pool() +- irqchip/gic-v3-its: Share ITS tables with a non-trusted hypervisor +- arm64: Enable memory encrypt for Realms +- arm64: mm: Avoid TLBI when marking pages as valid +- arm64: Enforce bounce buffers for realm DMA +- efi: arm64: Map Device with Prot Shared +- arm64: rsi: Map unprotected MMIO as decrypted +- arm64: rsi: Add support for checking whether an MMIO is protected +- arm64: realm: Query IPA size from the RMM +- arm64: Detect if in a realm and set RIPAS RAM +- arm64: rsi: Add RSI definitions +- arm64: mm: Add confidential computing hook to ioremap_prot() +- arm64: mm: Add top-level dispatcher for internal mem_encrypt API +- Revert "gicv3: add lpi support for cvm guest" +- Revert "virtcca feature : disable swiotlb for passthrough device" +- ALSA: pcm: Fix race of buffer access at PCM OSS layer +- fix enfs bug +- mm/mem_sampling: Fix inaccurate sampling for NUMA balancing and DAMON +- mm/mem_sampling: Prevent mem_sampling from being enabled if SPE init failed +- mm/mem_sampling: add trace event for spe based damon record +- drm/radeon: fix uninitialized size issue in radeon_vce_cs_parse() +- mm:userswap: change VM_USWAP_BIT to bit 61 +- crypto: qat - switch to standard pattern for PCI IDs +- crypto: qat - remove BITS_IN_DWORD() +- crypto: qat - remove initialization in device class +- crypto: qat - add shutdown handler to qat_c3xxx +- crypto: qat - remove redundant prototypes in qat_c3xxx +- crypto: qat - add shutdown handler to qat_c62x +- crypto: qat - remove redundant prototypes in qat_c62x +- crypto: qat - add shutdown handler to qat_dh895xcc +- crypto: qat - remove redundant prototypes in qat_dh895xcc +- crypto: qat - add shutdown handler to qat_420xx +- crypto: qat - add shutdown handler to qat_4xxx +- crypto: qat - Remove dst_null support +- crypto: qat - remove access to parity register for QAT GEN4 +- crypto: qat - set parity error mask for qat_420xx +- crypto: qat - optimize allocations for fw authentication +- crypto: qat - remove redundant FW image size check +- crypto: qat - remove unused members in suof structure +- crypto: qat - introduce fuse array +- crypto: qat - add macro to write 64-bit values to registers +- crypto: qat - refactor service parsing logic +- crypto: qat - do not export adf_cfg_services +- crypto: qat - reorder objects in qat_common Makefile +- crypto: qat - fix object goals in Makefiles +- crypto: qat - set command ids as reserved +- crypto: drivers - Use str_enable_disable-like helpers +- vfio/qat: fix overflow check in qat_vf_resume_write() +- crypto: qat - Fix missing destroy_workqueue in adf_init_aer() +- crypto: qat - Fix typo "accelaration" +- crypto: qat - Constify struct pm_status_row +- crypto: qat - remove unused adf_devmgr_get_first +- lib/string_choices: Add str_up_down() helper +- crypto: qat - Remove trailing space after +- crypto: qat - fix "Full Going True" macro definition +- crypto: qat - Use static_assert() to check struct sizes +- crypto: qat - allow disabling SR-IOV VFs +- crypto: qat - ensure correct order in VF restarting handler +- crypto: qat - fix recovery flow for VFs +- crypto: qat - disable IOV in adf_dev_stop() +- crypto: qat - preserve ADF_GENERAL_SEC +- crypto: qat - initialize user_input.lock for rate_limiting +- crypto: qat - make adf_ctl_class constant +- crypto: qat - Fix typo +- crypto: qat - fix linking errors when PCI_IOV is disabled +- objtool, media: dib8000: Prevent divide-by-zero in dib8000_set_dds() +- mm: pcp: increase pcp->free_count threshold to trigger free_high +- mm: page_alloc: remove redundant READ_ONCE +- nfs: handle failure of nfs_get_lock_context in unlock path +- arm64: xcall: Fix GPR touch as the original step +- iommu/loongarch: fix potential hung task when error occurred +- LoongArch: Fix panic caused by NULL-PMD in huge_pte_offset() +- mm/page_alloc.c: avoid infinite retries caused by cpuset race +- LoongArch: Fix early_numa_add_cpu() usage for FDT systems +- !16740 KVM: arm64: check if IPIV is enabled in BIOS +- scsi: lpfc: Free phba irq in lpfc_sli4_enable_msi() when pci_irq_vector() fails +- !16737 drivers/perf: hisi: Add events and rename event "cycle" for pa_pmu +- KVM: arm64: check if IPIV is enabled in BIOS +- drivers/perf: hisi: Add events and rename event "cycle" for pa_pmu +- !16694 RDMA/hns: Fix double destruction of rsv_qp +- !16730 uacce: removal of the Shared Memory Feature for Queues +- !16727 perf: Remove unstable events for uncore L3C PMU +- !16732 drivers/perf: hisi: Fixes the incorrect bitmask limit for the CPA event sysfs interface +- !16728 drivers/perf: hisi: Add cacheable option for L3C PMU +- !16724 drivers/perf: hisi: Clarifying event names and fix event ID for pa_pmu +- !16723 perf iostat: hisi: Fix port range retrival +- mm/madvise: handle madvise_lock() failure during race unwinding +- mm/madvise: batch tlb flushes for MADV_DONTNEED[_LOCKED] +- mm/memory: split non-tlb flushing part from zap_page_range_single() +- mm/madvise: batch tlb flushes for MADV_FREE +- mm/madvise: define and use madvise_behavior struct for madvise_do_behavior() +- mm/madvise: remove len parameter of madvise_do_behavior() +- mm/madvise: remove redundant mmap_lock operations from process_madvise() +- mm/madvise: split out madvise() behavior execution +- mm/madvise: split out madvise input validity check +- mm/madvise: split out mmap locking operations for madvise() +- mm/madvise: unrestrict process_madvise() for current process +- !16729 arm64: Fix unmet direct dependencies for DEBUG_FEATURE_BYPASS +- drivers/perf: hisi: Fixes the incorrect bitmask limit for the CPA event sysfs interface +- crypto: hisilicon/qm - check whether the input parameters and device PF match +- uacce: removal of the Shared Memory Feature for Queues +- arm64: Fix unmet direct dependencies for DEBUG_FEATURE_BYPASS +- !16721 cpufreq: CPPC: Don't warn on failing to read perf counters on offline cpus +- drivers/perf: hisi: Add cacheable option for L3C PMU +- perf: Remove unstable events for uncore L3C PMU +- drivers/perf: hisi: Clarifying event names and fix event ID for pa_pmu +- perf iostat: hisi: Fix port range retrival +- !16701 Add HiSilicon uncore frequency scaling driver +- !16712 crypto: hisilicon - support algorithm fallback v2 +- iommu: smmuv3: Not print information of SMMU 0x10 event +- arm64: topology: Setup amu fie when cpu hotplugging +- cpufreq: CPPC: Fix error handling in cppc_scale_freq_workfn() +- cpufreq: CPPC: Don't warn on failing to read perf counters on offline cpus +- topology: Set capacity_freq_ref in all cases +- arm64/amu: Use capacity_ref_freq() to set AMU ratio +- cpufreq/cppc: Set the frequency used for computing the capacity +- energy_model: Use a fixed reference frequency +- cpufreq/schedutil: Use a fixed reference frequency +- cpufreq: Use the fixed and coherent frequency for scaling capacity +- !16717 Fix issues of nfs mounting +- !16714 v2 Fix file content inconsistency issue and incorporate subsequent fix patches +- sched/topology: Add a new arch_scale_freq_ref() method +- !16696 KVM: arm64: Check whether GICv4.0 is enabled +- nfs: ignore SB_RDONLY when remounting nfs +- nfs: clear SB_RDONLY before getting superblock +- vfs: generate FS_CREATE before FS_OPEN when ->atomic_open used. +- NFS: abort nfs_atomic_open_v23 if name is too long. +- NFS: add atomic_open for NFSv3 to handle O_TRUNC correctly. +- !16680 Fix the problem of NFS client mount read/write permission failure +- crypto: hisilicon/qm - check the value obtained from the hardware +- crypto: hisilicon/hpre - support the hpre algorithm fallback +- crypto: hisilicon - implement full backlog support for sec +- uacce: add queue lock for uacce_get_ss_dma +- uacce: use WARN_ON_ONCE instead of WARN_ON +- !16705 fs/resctrl: Fix return value in rdtgroup_pseudo_locked_in_hierarchy() +- !16691 v3 Rewrite smt interference account logic +- mm/madvise: don't perform madvise VMA walk for MADV_POPULATE_(READ|WRITE) +- !16662 Fix watchdog false positive problem +- !16711 af_unix: Read 0 bytes with MSG_PEEK loops if last skb is OOB +- af_unix: Read 0 bytes with MSG_PEEK loops if last skb is OOB +- fs/resctrl: Fix return value in rdtgroup_pseudo_locked_in_hierarchy() +- !16287 Update the watchdog period according to real CPU frequency +- !16679 fs/resctrl: L2_MON does not support the limbo mechanism +- openeuler_defconfig: Enable devfreq and hisi_uncore_freq by default +- PM / devfreq: Add HiSilicon uncore frequency scaling driver +- PM / devfreq: Allow devfreq driver to add custom sysfs ABIs +- PM / devfreq: Check governor before using governor->name +- PM / devfreq: Remove redundant devfreq_get_freq_range() calling in devfreq_add_device() +- PM / devfreq: Limit max_freq with scaling_min_freq +- PM / devfreq: governor: Replace sscanf() with kstrtoul() in set_freq_store() +- mailbox: pcc: Refactor and simplify check_and_ack() +- mailbox: pcc: Always map the shared memory communication address +- mailbox: pcc: Refactor error handling in irq handler into separate function +- mailbox: pcc: Use acpi_os_ioremap() instead of ioremap() +- mailbox: pcc: Return early if no GAS register from pcc_mbox_cmd_complete_check +- mailbox: pcc: Drop unnecessary endianness conversion of pcc_hdr.flags +- !16653 [OLK-6.6] Revert "USB: xHCI: fix resume issue of some ZHAOXIN hosts" +- !16699 mm: provide vm_normal_(page|folio)_pmd() with CONFIG_PGTABLE_HAS_HUGE_LEAVES +- mm: provide vm_normal_(page|folio)_pmd() with CONFIG_PGTABLE_HAS_HUGE_LEAVES +- KVM: arm64: Check whether GICv4.0 is enabled +- RDMA/hns: Fix double destruction of rsv_qp +- interference: Rewrite smt interference account logic +- interference: Fix the compilation warning of sched_task_is_throttled() +- !16686 v2 Soft domain improves and bugfixes +- sched: Fix might sleep in atomic section issue +- sched: Consider task affinity in wake_soft_domain() +- sched: Fix soft domain group memleak +- sched: Rework cpu.soft_domain_nr_cpu +- sched: Add cmdline sched_soft_domain switch for soft domain feature +- !16639 arm64/mpam: Add MPAM manual +- Revert "nfs: ignore SB_RDONLY when mounting nfs" +- Revert "nfs: pass flags to second superblock" +- Revert "nfs: fix the loss of superblock's initialized flags" +- !16651 iommu/arm-smmu-v3: Fix global-out-of-bounds access in arm_smmu_group_get_mpam() +- fs/resctrl: L2_MON does not support the limbo mechanism +- !16670 mm/mm_spe.c: remove redundant checks and debug prints in mm_spe probe +- !16668 Remove useless file +- mm/mm_spe.c: remove redundant checks and debug prints in mm_spe probe +- Remove useless file +- !15819 [OLK-6.6]drm/phytium: Fix some Bugs in Phytium Display Engine +- watchdog: fix the SOFTLOCKUP_DETECTOR=n case +- watchdog: fix watchdog may detect false positive of softlockup +- crypto: iaa - Fix potential use after free bug +- mm/migrate_device: don't add folio to be freed to LRU in migrate_device_finalize() +- mm: migrate_device: use more folio in migrate_device_finalize() +- Revert "remoteproc: core: Clear table_sz when rproc_shutdown" +- remoteproc: core: Clear table_sz when rproc_shutdown +- jfs: fix slab-out-of-bounds read in ea_get() +- tpm: do not start chip while suspended +- KVM: arm64: Tear down vGIC on failed vCPU creation +- drm/nouveau: Fix WARN_ON in nouveau_fence_context_kill() +- Revert "USB: xHCI: fix resume issue of some ZHAOXIN hosts" +- arm64: proton-pack: Add new CPUs 'k' values for branch mitigation +- arm64: bpf: Only mitigate cBPF programs loaded by unprivileged users +- arm64: bpf: Add BHB mitigation to the epilogue for cBPF programs +- arm64: proton-pack: Expose whether the branchy loop k value +- arm64: proton-pack: Expose whether the platform is mitigated by firmware +- arm64: insn: Add support for encoding DSB +- arm64: errata: Add missing sentinels to Spectre-BHB MIDR arrays +- arm64: errata: Add newer ARM cores to the spectre_bhb_loop_affected() lists +- arm64: cputype: Add MIDR_CORTEX_A76AE +- arm64: errata: Add KRYO 2XX/3XX/4XX silver cores to Spectre BHB safe list +- arm64: errata: Assume that unknown CPUs _are_ vulnerable to Spectre BHB +- arm64: errata: Add QCOM_KRYO_4XX_GOLD to the spectre_bhb_k24_list +- iommu/arm-smmu-v3: Fix global-out-of-bounds access in arm_smmu_group_get_mpam() +- fs/resctrl: Prevent idle RMIDs from not being released in time from limbo +- arm64/mpam: Update the chip condition judgment in resctrl_arch_would_mbm_overflow() +- add virtcca capability and kae vf interface +- arm64/watchdog_hld: Add a cpufreq notifier for update watchdog thresh +- watchdog/perf: Provide function for adjusting the event period +- !16627 fix move_pages return value +- arm64/mpam: Add MPAM manual +- ksmbd: fix r_count dec/increment mismatch +- Input: xpad - fix two controller table values +- cifs: Ensure that all non-client-specific reparse points are processed by the server +- nvme: unblock ctrl state transition for firmware update +- vboxsf: Add __nonstring annotations for unterminated strings +- netfilter: nf_tables: allow clone callbacks to sleep +- netfilter: nf_tables: bail out if stateful expression provides no .clone +- netfilter: nf_tables: use timestamp to check for set element timeout +- rust: lockdep: Remove support for dynamically allocated LockClassKeys +- HID: apple: disable Fn key handling on the Omoton KB066 +- nvme-fc: rely on state transitions to handle connectivity loss +- Bluetooth: L2CAP: Fix corrupted list in hci_chan_del +- smb: client: Fix match_session bug preventing session reuse +- smb3: add support for IAKerb +- i2c: sis630: Fix an error handling path in sis630_probe() +- i2c: ali15x3: Fix an error handling path in ali15x3_probe() +- i2c: ali1535: Fix an error handling path in ali1535_probe() +- cifs: Throw -EOPNOTSUPP error on unsupported reparse point type from parse_reparse_point() +- cifs: Validate content of WSL reparse point buffers +- cifs: Fix integer overflow while processing closetimeo mount option +- cifs: Fix integer overflow while processing actimeo mount option +- scripts: generate_rust_analyzer: add missing macros deps +- ASoC: codecs: wm0010: Fix error handling path in wm0010_spi_probe() +- ASoC: rt722-sdca: add missing readable registers +- rust: init: add missing newline to pr_info! calls +- ASoC: cs42l43: Fix maximum ADC Volume +- drm/gma500: Add NULL check for pci_gfx_root in mid_get_vbt_data() +- rust: error: add missing newline to pr_warn! calls +- ASoC: ops: Consistently treat platform_max as control value +- smb: client: fix regression with guest option +- qlcnic: fix memory leak issues in qlcnic_sriov_common.c +- arm64: mm: Populate vmemmap at the page level if not section aligned +- ASoC: amd: yc: Support mic on another Lenovo ThinkPad E16 Gen 2 model +- clk: samsung: update PLL locktime for PLL142XX used on FSD platform +- ksmbd: prevent connection release during oplock break notification +- ksmbd: fix use-after-free in ksmbd_free_work_struct +- drm/amd/display: Fix slab-use-after-free on hdcp_work +- drm/amd/display: Assign normalized_pix_clk when color depth = 14 +- drm/amd/display: Restore correct backlight brightness after a GPU reset +- drm/amd/display: Disable unneeded hpd interrupts during dm_init +- drm/dp_mst: Fix locking when skipping CSN before topology probing +- drm/atomic: Filter out redundant DPMS calls +- drm/i915/cdclk: Do cdclk post plane programming later +- x86/microcode/AMD: Fix out-of-bounds on systems with CPU-less NUMA nodes +- USB: serial: option: match on interface class for Telit FN990B +- USB: serial: option: fix Telit Cinterion FE990A name +- USB: serial: option: add Telit Cinterion FE990B compositions +- USB: serial: ftdi_sio: add support for Altera USB Blaster 3 +- Input: i8042 - swap old quirk combination with new quirk for more devices +- Input: i8042 - swap old quirk combination with new quirk for several devices +- Input: i8042 - add required quirks for missing old boardnames +- Input: i8042 - swap old quirk combination with new quirk for NHxxRZQ +- Input: xpad - rename QH controller to Legion Go S +- Input: xpad - add support for TECNO Pocket Go +- Input: xpad - add support for ZOTAC Gaming Zone +- Input: xpad - add multiple supported devices +- Input: xpad - add 8BitDo SN30 Pro, Hyperkin X91 and Gamesir G7 SE controllers +- Input: iqs7222 - preserve system status register +- Input: ads7846 - fix gpiod allocation +- io_uring: use unpin_user_pages() where appropriate +- !16620 close_files(): don't bother with xchg() +- io_uring/kbuf: vmap pinned buffer ring +- io_uring: use vmap() for ring mapping +- io_uring: fix corner case forgetting to vunmap +- io_uring: don't attempt to mmap larger than what the user asks for +- io_uring: get rid of remap_pfn_range() for mapping rings/sqes +- mm: add nommu variant of vm_insert_pages() +- block: fix 'kmem_cache of name 'bio-108' already exists' +- net: Handle napi_schedule() calls from non-interrupt +- drm/nouveau: Do not override forced connector status +- mptcp: safety check before fallback +- x86/irq: Define trace events conditionally +- perf/x86/intel: Use better start period for frequency mode +- drm/vkms: Round fixp2int conversion in lerp_u16 +- fuse: don't truncate cached, mutated symlink +- ASoC: tas2764: Set the SDOUT polarity correctly +- ASoC: tas2764: Fix power control mask +- ASoC: tas2770: Fix volume scale +- nvme: only allow entering LIVE from CONNECTING state +- sctp: Fix undefined behavior in left shift operation +- cifs: Treat unhandled directory name surrogate reparse points as mount directory nodes +- apple-nvme: Release power domains when probe fails +- nvmet-rdma: recheck queue state is LIVE in state lock in recv done +- nvme-pci: quirk Acer FA100 for non-uniqueue identifiers +- io-wq: backoff when retrying worker creation +- net: wwan: mhi_wwan_mbim: Silence sequence number glitch errors +- ASoC: SOF: amd: Handle IPC replies before FW_BOOT_COMPLETE +- ASoC: SOF: Intel: hda: add softdep pre to snd-hda-codec-hdmi module +- ASoC: arizona/madera: use fsleep() in up/down DAPM event delays. +- ASoC: rsnd: adjust convert rate limitation +- ASoC: rsnd: don't indicate warning on rsnd_kctrl_accept_runtime() +- ASoC: rsnd: indicate unsupported clock rate +- ALSA: hda/realtek: Limit mic boost on Positivo ARN50 +- ASoC: simple-card-utils.c: add missing dlc->of_node +- selftests/bpf: Fix invalid flag of recv() +- Bluetooth: L2CAP: Fix slab-use-after-free Read in l2cap_send_cmd +- Xen/swiotlb: mark xen_swiotlb_fixup() __init +- thermal/cpufreq_cooling: Remove structure member documentation +- s390/cio: Fix CHPID "configure" attribute caching +- platform/x86: thinkpad_acpi: Support for V9 DYTC platform profiles +- platform/x86: thinkpad_acpi: Fix invalid fan speed on ThinkPad X120e +- sched: Clarify wake_up_q()'s write to task->wake_q.next +- objtool: Ignore dangling jump table entries +- HID: apple: fix up the F6 key on the Omoton KB066 keyboard +- HID: hid-apple: Apple Magic Keyboard a3203 USB-C support +- HID: topre: Fix n-key rollover on Realforce R3S TKL boards +- usb: phy: generic: Use proper helper for property detection +- HID: ignore non-functional sensor in HP 5MP Camera +- HID: intel-ish-hid: Send clock sync message immediately after reset +- HID: intel-ish-hid: fix the length of MNG_SYNC_FW_CLOCK in doorbell +- vboxsf: fix building with GCC 15 +- alpha/elf: Fix misc/setarch test of util-linux by removing 32bit support +- smb: client: fix noisy when tree connecting to DFS interlink targets +- ACPI: resource: IRQ override for Eluktronics MECH-17 +- scsi: ufs: core: Fix error return with query response +- scsi: core: Use GFP_NOIO to avoid circular locking dependency +- platform/x86/intel: pmc: fix ltr decode in pmc_core_ltr_show() +- sched/debug: Provide slice length for fair tasks +- powercap: call put_device() on an error path in powercap_register_control_type() +- hrtimers: Mark is_migration_base() with __always_inline +- nvme-fc: do not ignore connectivity loss during connecting +- nvme-fc: go straight to connecting state when initializing +- net/mlx5e: Prevent bridge link show failure for non-eswitch-allowed devices +- net/mlx5: Bridge, fix the crash caused by LAG state check +- net/mlx5: Lag, Check shared fdb before creating MultiPort E-Switch +- net: openvswitch: remove misbehaving actions length check +- netfilter: nft_exthdr: fix offset with ipv4_find_option() +- net_sched: Prevent creation of classes with TC_H_ROOT +- ipvs: prevent integer overflow in do_ip_vs_get_ctl() +- netfilter: nf_conncount: Fully initialize struct nf_conncount_tuple in insert_tree() +- bonding: fix incorrect MAC address setting to receive NS messages +- net: switchdev: Convert blocking notification chain to a raw one +- eth: bnxt: do not update checksum in bnxt_xdp_build_skb() +- net/mlx5: handle errors in mlx5_chains_create_table() +- Drivers: hv: vmbus: Don't release fb_mmio resource in vmbus_free_mmio() +- netpoll: hold rcu read lock in __netpoll_send_skb() +- net: mctp i2c: Copy headers if cloned +- net: dsa: mv88e6xxx: Verify after ATU Load ops +- net/mlx5: Fill out devlink dev info only for PFs +- Revert "Bluetooth: hci_core: Fix sleeping function called from invalid context" +- Bluetooth: hci_event: Fix enabling passive scanning +- sched: address a potential NULL pointer dereference in the GRED scheduler. +- netfilter: nf_conncount: garbage collection is not skipped when jiffies wrap around +- ice: fix memory leak in aRFS after reset +- netfilter: nft_ct: Use __refcount_inc() for per-CPU nft_ct_pcpu_template. +- pinctrl: bcm281xx: Fix incorrect regmap max_registers value +- fbdev: hyperv_fb: iounmap() the correct memory when removing a device +- tcp: fix forever orphan socket caused by tcp_abort +- tcp: fix races in tcp_abort() +- hrtimer: Use and report correct timerslack values for realtime tasks +- zram: fix NULL pointer in comp_algorithm_show() +- sched/isolation: Prevent boot crash when the boot CPU is nohz_full +- clockevents/drivers/i8253: Fix stop sequence for timer 0 +- !16604 pds_core: remove write-after-free of client_id +- !16434 ksmbd: fix use-after-free in ksmbd_session_rpc_open +- !16585 tracing: Fix oob write in trace_seq_to_buffer() +- mm/migrate: convert add_page_for_migration() from follow_page() to folio_walk +- mm/pagewalk: fix usage of pmd_leaf()/pud_leaf() without present check +- mm/pagewalk: introduce folio_walk_start() + folio_walk_end() +- mm: introduce vma_pgtable_walk_{begin|end}() +- mm/migrate: use folio_likely_mapped_shared() in add_page_for_migration() +- mm: allocate THP on hugezeropage wp-fault +- mm: abstract THP allocation +- !16537 md: fix mddev uaf while iterating all_mddevs list +- !16538 md/raid10: wait barrier before returning discard request with REQ_NOWAIT +- !16494 scsi: st: Fix array overflow in st_setup() +- !16582 module: ensure that kobject_put() is safe for module type kobjects +- !16605 iio: imu: st_lsm6dsx: fix possible lockup in st_lsm6dsx_read_tagged_fifo +- !16606 CVE-2024-58100 +- close_files(): don't bother with xchg() +- SELinux: Add check for the user data passed to kcalloc in hashtab_init +- !16609 migration: update status after RAS error +- !16586 dlm: prevent NPD when writing a positive value to event_done +- !16580 dm-flakey: Fix memory corruption in optional corrupt_bio_byte feature +- bpf: Fix kabi breakage for changes_pkt_data in struct bpf_prog_aux +- bpf: fix null dereference when computing changes_pkt_data of prog w/o subprogs +- bpf: check changes_pkt_data property for extension programs +- iio: imu: st_lsm6dsx: fix possible lockup in st_lsm6dsx_read_tagged_fifo +- pds_core: remove write-after-free of client_id +- !16571 Fix erofs fid overflow +- !16574 cgroup/cpuset-v1: Add missing support for cpuset_v2_mode +- !16550 media: venus: hfi_parser: refactor hfi packet parsing logic +- !16549 soc: samsung: exynos-chipid: Add NULL pointer check in exynos_chipid_probe() +- !16600 ftrace: Fix preemption accounting for stacktrace trigger command +- !16596 powerpc/crash: Fix non-smp kexec preparation +- !16562 platform/x86/intel-uncore-freq: Add efficiency latency control and other fix for 6.6 +- !16593 vxlan: vnifilter: Fix unlocked deletion of default FDB entry +- crypto: ccp: fix The high priority queue changes cmd_id of the queued elements +- finally take no_llseek out +- migration: update status after RAS error +- ftrace: Fix preemption accounting for stacktrace trigger command +- powerpc/crash: Fix non-smp kexec preparation +- !16588 ipc: fix to protect IPCS lookups using RCU +- vxlan: vnifilter: Fix unlocked deletion of default FDB entry +- ipc: fix to protect IPCS lookups using RCU +- dlm: prevent NPD when writing a positive value to event_done +- tracing: Fix compilation warning on arm32 +- tracing: Fix oob write in trace_seq_to_buffer() +- module: ensure that kobject_put() is safe for module type kobjects +- dm-flakey: Fix memory corruption in optional corrupt_bio_byte feature +- !16492 arm64/watchdog: fix watchdog failure in low power scenarios +- PCI/P2PDMA: Add Zhaoxin Zhaoxin bridges to the whitelist +- PCI: Supplement ACS quirk for more Zhaoxin Root Ports +- cgroup/cpuset-v1: Add missing support for cpuset_v2_mode +- seq_buf: Make DECLARE_SEQ_BUF() usable +- seq_buf: Introduce DECLARE_SEQ_BUF and seq_buf_str() +- !16482 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.69-6.6.70 LTS Conflicts Patches +- sw64: disable EFFICIENT_UNALIGNED_ACCESS for C3B +- sw64: optimize unaligned access +- sw64: ftrace: Add direct call trampoline samples support +- sw64: ftrace: Add direct call support +- sw64: ftrace: No need to save original function's temporary registers +- sw64: perf: fix perf test 'import perf' in python failed +- sw64: add support for function error injection +- sw64: add sw64 rethook implementation +- sw64: emit fixed-length instructions for BPF_PSEUDO_FUNC +- sw64: ftrace: Enable HAVE_FUNCTION_GRAPH_RETVAL +- sw64: mm: adjust userspace memory layout +- sw64: cpufreq: fix the frequency returned by cpufreq_driver->get() +- sw64: ioremap: map some IO space with ioremap +- sw64: ioremap: fix an overflow in pci_remap_cfgspace +- erofs: fix file handle encoding for 64-bit NIDs +- exportfs: define FILEID_INO64_GEN* file handle types +- !16211 [OLK-6.6] Fix CVE-2025-22028 +- !16210 [OLK-6.6] Fix CVE-2025-22109 +- !16209 [OLK-6.6] Fix CVE-2025-22127 +- !16207 [OLK-6.6] Fix CVE-2025-22104 +- !16563 Some random fixes and cleanup to shmem +- !16564 tracing: Fix filter string testing +- sw64: ioremap: support generic ioremap for C4 +- nvme-pci: acquire cq_poll_lock in nvme_poll_irqdisable +- mm/shmem: remove unneeded xa_is_value() check in shmem_unuse_swap_entries() +- mm/shmem: fix potential dead loop in shmem_unuse() +- mm: shmem: add missing shmem_unacct_size() in __shmem_file_setup() +- !16536 sched: fair: Use rq_clock_task to account rq->idle_avg +- tracing: Fix filter string testing +- mm: shmem: avoid unpaired folio_unlock() in shmem_swapin_folio() +- mm: mincore: use pte_batch_hint() to batch process large folios +- !16542 perf: Fix hang while freeing sigtrap event +- media: venus: hfi_parser: refactor hfi packet parsing logic +- soc: samsung: exynos-chipid: Add NULL pointer check in exynos_chipid_probe() +- !16313 PCI: Fix reference leak in pci_register_host_bridge() +- !16395 PCI: vmd: Make vmd_dev::cfg_lock a raw_spinlock_t type +- !16246 backport mainline mm bugfix +- !16541 irqchip/gic: remove useless printing in pv sgi +- !16477 CVE-2024-50140 +- KVM: arm64: fix memory leak in TLBI +- !16435 fix a couple of races in MNT_TREE_BENEATH handling by do_move_mount() +- !16372 ksmbd: Fix UAF in __close_file_table_ids +- !16371 ext4: ignore xattrs past end +- perf: Fix hang while freeing sigtrap event +- irqchip/gic: remove useless printing in pv sgi +- md/raid10: wait barrier before returning discard request with REQ_NOWAIT +- md: fix mddev uaf while iterating all_mddevs list +- !16532 memcg: always call cond_resched() after fn() +- sched: fair: Use rq_clock_task to account rq->idle_avg +- memcg: always call cond_resched() after fn() +- !16524 v3 CVE-2024-27079 +- !16527 v2 some fix about ifs +- !16521 drm/amd/display: Fix slab-use-after-free in hdcp +- !16513 fs/resctrl: Restore default settings for all resctrl_res_level +- interference: Fix IFS cannot show in cgroup v1 sub-cgroup +- interference: Fix compilation issue +- !16523 v2 arm64: entry: Fix ACTLR_XCALL_XINT not set build problem +- platform/x86/intel-uncore-freq: Add efficiency latency control to sysfs interface +- platform/x86/intel-uncore-freq: Add support for efficiency latency control +- platform/x86/intel-uncore-freq: Do not present separate package-die domain +- iommu/vt-d: Fix NULL domain on device release +- iommu: Fix kabi breakage for release_domain in struct iommu_ops +- iommu: Add static iommu_ops->release_domain +- !16515 ASoC: qcom: Fix sc7280 lpass potential buffer overflow +- !16483 bugfix for hisilicon accelerator driver +- arm64: entry: Fix ACTLR_XCALL_XINT not set build problem +- drm/amd/display: Fix slab-use-after-free in hdcp +- ASoC: qcom: Fix sc7280 lpass potential buffer overflow +- fs/resctrl: Add missing rdt_last_cmd_clear() after rdtgroup_kn_lock_live() +- fs/resctrl: Restore default settings for all resctrl_res_level +- fs/resctrl: Free mbm_total and mbm_local when fails +- perf/x86/intel: Only check the group flag for X86 leader +- !16510 v5 Support soft domain +- !16509 v2 Support thp policy control per process +- !16479 wifi: ath12k: Fix invalid data access in ath12k_dp_rx_h_undecap_nwifi +- !16478 pds_core: make wait_context part of q_info +- !16476 drm/i915/huc: Fix fence not released on early probe errors +- !16499 padata: do not leak refcount in reorder_work +- config: Configurate CONFIG_SCHED_SOFT_DOMAIN +- sched: fair: Disable numa migrate for soft domian task +- sched: fair: Select idle cpu in soft domain +- sched: Attach task group to soft domain +- sched: topology: Build soft domain for LLC +- mm: tool: add task_thpctl to control thp per process +- mm: add thp process control interface module +- !16489 v5 Cgroup-Based Interference Statistics +- !16453 perf/x86/intel: KVM: Mask PEBS_ENABLE loaded for guest with vCPU's value. +- !16458 v3 arm64: idle: Handle idle state for xcall/xint +- !16496 fbdev: omapfb: Add 'plane' value check +- !16459 fs/jfs: Prevent integer overflow in AG size calculation +- usb: typec: ucsi: Set orientation as none when connector is unplugged +- !16486 v10 High Performance Container Resource View Isolation +- padata: do not leak refcount in reorder_work +- !16461 net: hns3: add set_loopback interface to realtek phy driver +- !16414 drm/nouveau: prime: fix ttm_bo_delayed_delete oops +- !16413 bus: mhi: host: Fix race between unprepare and queue_buf +- !15156 Add IOAPIC support for Hygon family 18h model 10h +- fbdev: omapfb: Add 'plane' value check +- scsi: st: Fix array overflow in st_setup() +- arm64/watchdog: fix watchdog failure in low power scenarios +- !16449 [OLK-6.6]Hygon: Support to dec and enc userspace pages +- !16488 drm/hisilicon/hibmc: add HPD, getting EDID, colorbar features in DP function +- !16431 v5 IPIV-BUGFIX for HIP12 +- drm/hisilicon/hibmc: hibmc-drm bugfix for DP +- drm/hisilicon/hibmc:fix KVM is not displaying when GRUB VGA is on +- drm/hisilicon/hibmc: fix irq_request()'s irq name variable is local +- drm/hisilicon/hibmc: fix HPD interrupts triggering the wrong behavior +- drm/hisilicon/hibmc: Add vga connector detect functions +- drm/hisilicon/hibmc: Add MSI irq getting and requesting for HPD +- drm/hisilicon/hibmc: Enable this hot plug detect of irq feature +- drm/hisilicon/hibmc: Add colorbar-cfg feature and its debugfs file +- drm/hisilicon/hibmc: Getting connector info and EDID by using AUX channel +- drm/hisilicon/hibmc: Refactor the member of drm_aux in struct hibmc_dp +- drm/hisilicon/hibmc: Add dp serdes cfg in dp process +- drm/hisilicon/hibmc: Add dp serdes cfg to adjust serdes rate, voltage and pre-emphasis +- drm/hisilicon/hibmc: Restructuring the header dp_reg.h +- drm/hisilicon/hibmc: select CONFIG_DRM_DISPLAY_DP_HELPER +- drm: Call drm_atomic_helper_shutdown() at shutdown time for misc drivers +- drm/ast: Implement polling for VGA and SIL164 connectors +- drm/sysfs: Register "ddc" symlink later +- openeuler_defconfig: Disable CGROUP_IFS config by default +- interference: Add cgroup v1 support for CGROUP_IFS +- interference: Add irq interference track support +- interference: Add sched wakelat interference track support +- interference: Exclude throttle time from rundelay +- interference: Add sched throttle interference track support +- interference: Add sched sleep interference track support +- interference: Add sched rundelay interference track support +- interference: Add smt interference track support +- interference: Add cgroup-based inteference statistics framework +- cgroup: Export cgroup_addrm_files as internal interface +- cgroup: Export cgroup_init_cftypes as internal interface +- interference: Add CGROUP_IFS_DEFAULT_ENABLED config +- interference: Add CGROUP_IFS config +- !16487 hisi_sas: Misc patches and cleanups +- scsi: hisi_sas: Fix warning detected by sparse +- scsi: hisi_sas: Wait until error handling is complete +- scsi: hisi_sas: Code style cleanup +- scsi: hisi_sas: Call I_T_nexus after soft reset for SATA disk +- scsi: hisi_sas: Use macro instead of magic number +- scsi: hisi_sas: Fix I/O errors caused by hardware port ID changes +- scsi: hisi_sas: Enable force phy when SATA disk directly connected +- scsi: hisi_sas: Fixed failure to issue vendor specific commands +- Revert "Revert "scsi: hisi_sas: Disable SATA disk phy for severe I_T nexus reset failure"" +- Revert "scsi: hisi_sas: Enable force phy when SATA disk directly connected" +- !15150 correct address translation for Hygon family 18h model 4h~10h +- !16445 [OLK-6.6][Bugfix]hugetlb: Fix NULL pointer dereference BUG +- !16484 v5: ucounts: rlimit scalability issues +- !16481 net: tcp: scale connect() under pressure +- config: Turn on CONFIG_BPF_RVI for openeuler_defconfig +- samples/bpf: Add iterator program for cpu_online +- samples/bpf: Add iterator program for cpuinfo_x86 +- bpf: Add bpf_arch_flags kunc for x86 +- bpf: Add cpuinfo_x86 iterator target +- bpf: pidns: Add {bpf,get}_current_level1_reaper() +- bpf: Add bpf_cpumask_weight() kfunc +- cpuset: Add task_effective_cpumask() +- bpf: cpuset: Add bpf_cpuset_from_task() kfunc +- bpf: Add generic_single_iter +- bpf: Add CONFIG_BPF_RVI +- ucount: fix kabi for ucounts +- ucounts: reinplement rlimit with percpu_counter +- ucounts: add input for inc_rlimit_[get_]ucounts_limit +- ucounts: add CONFIG_UCOUNTS_PERCPU_COUNTER +- ucount: use rcuref_t for reference counting +- ucount: use RCU for ucounts lookups +- ucount: replace get_ucounts_or_wrap() with atomic_inc_not_zero() +- ucounts: move kfree() out of critical zone protected by ucounts_lock +- rcu: provide a static initializer for hlist_nulls_head +- !16473 RDMA/hns: Trace support and bugfixes +- !15148 Add hygon_set_guest_pat_wb parameter to enhance performance for non-passthrough application scenarios +- crypto: hisilicon/zip - do not expose hashagg algorithm when uacce mode is 2 +- crypto: hisilicon/trng - use DEFINE_MUTEX() for mutex lock +- crypto: hisilicon - enable error reporting again +- crypto: hisilicon/qm - alloc power of 2 interrupts +- crypto: hisilicon/qm - fix vfs_num error +- uacce: fix unsigned long print format +- uacce: delete some redundant code +- uacce: fix isolate sysfs check condition +- tcp: Fix bind() regression for v6-only wildcard and v4-mapped-v6 non-wildcard addresses. +- inet: call inet6_ehashfn() once from inet6_hash_connect() +- inet: change lport contribution to inet_ehashfn() and inet6_ehashfn() +- tcp: use RCU lookup in __inet_hash_connect() +- tcp: add RCU management to inet_bind_bucket +- tcp: optimize inet_use_bhash2_on_bind() +- tcp: Use bhash2 for v4-mapped-v6 non-wildcard address. +- tcp: use RCU in __inet{6}_check_established() +- usb: typec: ucsi: glink: fix off-by-one in connector_status +- usb: typec: ucsi: glink: be more precise on orientation-aware ports +- usb: typec: ucsi: glink: set orientation aware if supported +- usb: typec: ucsi: glink: move GPIO reading into connector_status callback +- wifi: ath12k: Fix invalid data access in ath12k_dp_rx_h_undecap_nwifi +- pds_core: make wait_context part of q_info +- task_work: make TWA_NMI_CURRENT handling conditional on IRQ_WORK +- task_work: Fix kabi breakage in enum task_work_notify_mode +- sched/core: Disable page allocation in task_tick_mm_cid() +- task_work: Add TWA_NMI_CURRENT as an additional notify mode. +- drm/i915/huc: Fix fence not released on early probe errors +- drm/virtio: implement virtio_gpu_shutdown +- virtgpu: don't reset on shutdown +- virtio: break and reset virtio devices on device_shutdown() +- wifi: ath12k: Optimize the mac80211 hw data access +- !16422 v3 fuse: Backport Anolis Features and Performance patches +- RDMA/hns: Optimize HW performance by limiting ACK request frequency +- RDMA/hns: Fix scc_param failed logic judgments +- RDMA/hns: Fix congestion control algorithm parameter range +- RDMA/hns: Fix address information leakage of DCA memory +- RDMA/hns: Inappropriate format characters cleanup +- RDMA/hns: Add trace for CMDQ dumping +- RDMA/hns: Include hnae3.h in hns_roce_hw_v2.h +- RDMA/hns: Add trace for MR/MTR attribute dumping +- RDMA/hns: Add trace for AEQE dumping +- RDMA/hns: Add trace for WQE dumping +- RDMA/hns: Add trace for flush CQE +- !16448 [OLK-6.6]riscv: fix kprobe crash due to broken patch_text() API +- drm/phytium: Fix some Bugs in Phytium Display Engine +- LoongArch: sync kvm head file from upstream +- LoongArch: fix migrate issue +- LoongArch: Save and restore CSR.CNTC for hibernation +- !16447 Intel: Intel_idle add GNRD and CWF platform support for 6.6 +- !16441 ext4: better scalability for ext4 block allocation +- !16456 v5 xfs: support log grant track bytes +- !16346 yt6801: Add Motorcomm yt6801 PCIe driver +- !16028 [OLK-6.6][eNFS]add nfs feature to support multipath +- !16398 cachefiles: Parse the "secctx" immediately +- net: libwx: fix Tx L4 checksum +- net: hns3: add set_loopback interface to realtek phy driver +- !16363 [OLK-6.6]:update patches for sw64 architecture +- riscv: livepatch.c adapt patch_text() changes +- !15153 Get the correct instance id calculation for Hygon family 18h model 6h~10h +- fs/jfs: Prevent integer overflow in AG size calculation +- arm64: entry: Support hardware xcall and xint +- arm64/Kconfig: Add turbo features submenuconfig +- xfs: fix incorrect tail lsn tracking when AIL is empty +- xfs: revert AIL TASK_KILLABLE threshold +- xfs: skip flushing log items during push +- xfs: grant heads track byte counts, not LSNs +- xfs: pass the full grant head to accounting functions +- xfs: track log space pinned by the AIL +- xfs: collapse xlog_state_set_callback in caller +- xfs: l_last_sync_lsn is really AIL state +- xfs: ensure log tail is always up to date +- xfs: background AIL push should target physical space +- xfs: AIL doesn't need manual pushing +- xfs: move and rename xfs_trans_committed_bulk +- Revert "xfs: update the last_sync_lsn with ctx start lsn" +- !16404 FIX CVE-2024-58096 +- !16432 wifi: ath11k: fix RCU stall while reaping monitor destination ring +- perf/x86/intel: KVM: Mask PEBS_ENABLE loaded for guest with vCPU's value. +- !16433 Fix CVE-2025-21972 +- x86/mm: support to dec and enc userspace pages +- !16399 CVE-2025-37925 +- !16291 Fix CVE-2025-37773 +- !16317 Fix CVE-2025-37782 +- !16378 vdpa:Adding compilation macro to decouple the vdpa and iommufd modules +- riscv: Pass patch_text() the length in bytes +- intel_idle: add Clearwater Forest SoC support +- intel_idle: add Granite Rapids Xeon D support +- !16041 [OLK-6.6] 支持用户态通过hygon_psp_resource2设备节点mmap PSP设备的mmio内存 +- mm/hugetlb: fix kernel NULL pointer dereference when replacing free hugetlb folios +- ext4: move mb_last_[group|start] to ext4_inode_info +- ext4: add ext4_try_lock_group() to skip busy groups +- !16366 Backport 6.6.55-6.6.60 LTS +- !16389 mm: remove unnecessary calls to lru_add_drain +- !16388 cgroup: Eliminate the need for cgroup_mutex in proc_cgroup_show() +- mm/huge_memory: fix dereferencing invalid pmd migration entry +- selftests/mm: compaction_test: support platform with huge mount of memory +- vdpa:Adding compilation macro to decouple the vdpa and iommufd modules +- EDAC/amd64: Get intlv_num_dies from F0x60 for Hygon family 18h model 6h +- EDAC/amd64: Use u16 for some umc variables for Hygon family 18h model 4h +- EDAC/amd64: Fix the calculation of cs id for Hygon family 18h model 4h +- EDAC/amd64: Rename the address translation function for hygon family 18h model 4h +- crypto: ccp: Support userspace mmap psp mmio +- !16100 kvm:Optimize the processing performance of multiple eventfd and irqfd +- !16427 [OLK-6.6] 移除vpsp运行时对CSV功能的依赖 +- !16428 RDMA/core: Silence oversized kvmalloc() warning +- fix a couple of races in MNT_TREE_BENEATH handling by do_move_mount() +- !16408 drm/amdgpu: handle amdgpu_cgs_create_device() errors in amd_powerplay_create() +- ksmbd: fix use-after-free in ksmbd_session_rpc_open +- wifi: ath11k: fix RCU stall while reaping monitor destination ring +- !16423 arm64/mpam: Update QoS partition default value +- net: mctp: unshare packets when reassembling +- arm64/kabi: use KABI_EXTEND to skip KABI check +- kvm: hisi: print error for IPIV +- KVM: arm64: using kvm_vgic_global_state for ipiv +- kvm: hisi: Don't allow to change mpidr in IPIv +- kvm: hisi: make sure vcpu_id and vcpu_idx have same value in IPIv +- KVM: arm64: fix live migration bug of IPIv +- irqchip/gic: Add HiSilicon PV SGI support +- KVM: arm64: Implement PV_SGI related calls +- KVM: arm64: Document PV-sgi interface +- KVM: arm64: Introduce ipiv enable ioctl +- arm64/config: add config to control whether enable IPIV feature +- !15657 [OLK-6.6] perf/x86/uncore: Add DF PMU support for Hygon family 18h model 8h +- RDMA/core: Silence oversized kvmalloc() warning +- !16421 mm: fix VMA heap bounds checking +- crypto: ccp: support psp generic ringbuffer +- crypto: ccp: Remove vpsp's dependency on sev module +- objtool, spi: amd: Fix out-of-bounds stack access in amd_set_spi_freq() +- !15903 bcache: call force_wake_up_gc() if necessary in check_should_bypass() +- !16394 CVE-2025-37970 +- arm64/mpam: Update QoS partition default value +- x86/resctrl: Add max_bw to struct resctrl_membw +- x86/resctrl: Remove hard-coded memory bandwidth limit +- arm64/mpam: Set 1 as the minimum setting value for MBA +- arm64/mpam: Fix num_rmids information +- fs/resctrl: Fix the crash caused by mounting resctrl but not support RDT +- fuse: remove an unnecessary if statement +- fuse: use fuse_range_is_writeback() instead of iterating pages +- fuse: remove unneeded lock which protecting update of congestion_threshold +- fuse: clear FR_PENDING if abort is detected when sending request +- fuse: cleanup request queuing towards virtiofs +- fuse: add fast path for fuse_range_is_writeback +- fuse: enable dynamic configuration of fuse max pages limit (FUSE_MAX_MAX_PAGES) +- fuse: Add initial support for fs-verity +- anolis: fuse: introduce write alignment +- anolis: fuse: separate bg_queue for write and other requests +- fuse: check aborted connection before adding requests to pending list for resending +- fuse: clear FR_SENT when re-adding requests into pending list +- fuse: set FR_PENDING atomically in fuse_resend() +- fuse: Use the high bit of request ID for indicating resend requests +- fuse: Introduce a new notification type for resend pending requests +- fuse: add support for explicit export disabling +- config: enable FS_STACK at correct position +- mm: fix VMA heap bounds checking +- drm/nouveau: prime: fix ttm_bo_delayed_delete oops +- bus: mhi: host: Fix race between unprepare and queue_buf +- add enfs feature +- !16392 【olk 6.6】backport two bugfixes for hibmcge driver +- drm/amdgpu: handle amdgpu_cgs_create_device() errors in amd_powerplay_create() +- !16401 v7 Add FUSE Passthrough Support +- wifi: ath11k: add srng->lock for ath11k_hal_srng_* in monitor mode +- fuse: config: enable FUSE_PASSTHROUGH and FS_STACK +- fs: use do_splice_direct() for nfsd/ksmbd server-side-copy +- fs: move file_start_write() into direct_splice_actor() +- fs: fork splice_file_range() from do_splice_direct() +- backing-file: convert to using fops->splice_write +- fuse: update inode size after extending passthrough write +- fs: pass offset and result to backing_file end_write() callback +- fuse: disable the combination of passthrough and writeback cache +- fs: fix __sb_write_started() kerneldoc formatting +- remap_range: merge do_clone_file_range() into vfs_clone_file_range() +- ovl: relax WARN_ON in ovl_verify_area() +- ovl: fix dentry reference leak after changes to underlying layers +- fuse: use exclusive lock when FUSE_I_CACHE_IO_MODE is set +- fuse: respect FOPEN_KEEP_CACHE on opendir +- fuse: verify zero padding in fuse_backing_map +- fuse: fix parallel dio write on file open in passthrough mode +- fuse: fix wrong ff->iomode state changes from parallel dio write +- fuse: implement passthrough for mmap +- fuse: implement splice read/write passthrough +- fuse: implement read/write passthrough +- fuse: implement open in passthrough mode +- fuse: prepare for opening file in passthrough mode +- fuse: implement ioctls to manage backing files +- fuse: introduce FUSE_PASSTHROUGH capability +- fuse: factor out helper for FUSE_DEV_IOC_CLONE +- fs: factor out backing_file_mmap() helper +- fs: factor out backing_file_splice_{read,write}() helpers +- fs: factor out backing_file_{read,write}_iter() helpers +- fs: prepare for stackable filesystems backing file helpers +- fs: create {sb,file}_write_not_started() helpers +- fs: create file_write_started() helper +- fs: create __sb_write_started() helper +- fs: move kiocb_start_write() into vfs_iocb_iter_write() +- fs: move permission hook out of do_iter_read() +- fs: move permission hook out of do_iter_write() +- fs: move file_start_write() into vfs_iter_write() +- coda: change locking order in coda_file_write_iter() +- btrfs: move file_start_write() to after permission hook +- remap_range: move file_start_write() to after permission hook +- remap_range: move permission hooks out of do_clone_file_range() +- splice: remove permission hook from iter_file_splice_write() +- splice: move permission hook out of splice_file_to_pipe() +- splice: move permission hook out of splice_direct_to_actor() +- splice: remove permission hook from do_splice_direct() +- ovl: add permission hooks outside of do_splice_direct() +- ovl: do not encode lower fh with upper sb_writers held +- ovl: do not open/llseek lower file with upper sb_writers held +- ovl: reorder ovl_want_write() after ovl_inode_lock() +- ovl: split ovl_want_write() into two helpers +- ovl: add helper ovl_file_modified() +- fs: store real path instead of fake path in backing file f_path +- fs: create helper file_user_path() for user displayed mapped file path +- fs: get mnt_writers count for an open backing file's real path +- ovl: protect copying of realinode attributes to ovl inode +- ovl: punt write aio completion to workqueue +- ovl: propagate IOCB_APPEND flag on writes to realfile +- ovl: use simpler function to convert iocb to rw flags +- fuse: allow parallel dio writes with FUSE_DIRECT_IO_ALLOW_MMAP +- fuse: introduce inode io modes +- fuse: prepare for failing open response +- fuse: break up fuse_open_common() +- fuse: allocate ff->release_args only if release is needed +- fuse: factor out helper fuse_truncate_update_attr() +- fuse: add fuse_dio_lock/unlock helper functions +- fuse: create helper function if DIO write needs exclusive lock +- fs: export mnt_{get,put}_write_access() to modules +- fs: rename __mnt_{want,drop}_write*() helpers +- jfs: reject on-disk inodes of an unsupported type +- cachefiles: Parse the "secctx" immediately +- PCI: vmd: Make vmd_dev::cfg_lock a raw_spinlock_t type +- iio: imu: st_lsm6dsx: fix possible lockup in st_lsm6dsx_read_fifo +- net: hibmcge: fix wrong ndo.open() after reset fail issue. +- net: hibmcge: fix incorrect statistics update issue +- sw64: adapt to the upstream changes in early_init_dt_(scan/verify) +- sw64: gpu: replace __sw_64__ with CONFIG_SW64 +- sw64: kprobe on ftrace: fix bug in restoring regs +- sw64: iommu: do real map on iova in 32bit mmio space +- mm: remove unnecessary calls to lru_add_drain +- cgroup: Eliminate the need for cgroup_mutex in proc_cgroup_show() +- !16374 LTS Conflicts Backport +- sw64: qspinlock: add CNA support for sw64 +- sw64: qspinlock: add PARAVIRT SPINLOCK support for sw64 +- sw64: gpu: memset_io and memcpy_toio/fromio for iomem on AMD swsmu +- Revert "sw64: gpu: memset_io and memcpy_toio/fromio for iomem on AMD swsmu" +- sw64: move syscall handling to sys_sw64.c +- sw64: improve sw64_rrk +- sw64: fix printing issue when using kprobe on ftrace +- sw64: add powercap driver +- sw64: clocksource: fix the issue of clockevents not being notified +- sw64: modify the interface for firmware to pass the running mode +- sw64: fix shared_cpu_map of L3 Cache +- sw64: fix SWVM_IRQS undeclared when enabling HIBERNATION +- sw64: kvm: optimize the compatibility of guest interrupt +- sw64: kvm: optimize the implementation of guest interrupt +- sw64: fix some compile issues for cpuautoplug.c +- sw64: cpufreq: fix for emulator that do not support cpufreq +- sw64: cpufreq: refactor cpufreq related code +- sw64: iommu: work around iova range check after resv_region +- sw64: gpu: fix SIMD IO access issues +- sw64: msi: fix guest msi interrupt handling +- sw64: msi: modify msi migration strategy +- sw64: msi: clear the residual vector_irq information of cpu +- sw64: ftrace: add notrace attribute to memset&memcpy +- sw64: ftrace: fix bug in restoring regs +- sw64: cpufreq: remove some meaningless code +- sw64: cpufreq: move cpufreq notifier to its user +- sw64: cpufreq: rename sw64_cpufreq.c to sunway-cpufreq.c +- sw64: cpufreq: remove debugfs code +- sw64: fix an error in BPF exception handling +- sw64: modify CPU autoplug initialization process +- sw64: iommu: print more info for interrupts +- sw64: msi: fix irq affinity setting during migration +- sw64: use CONFIG_HZ_PERIODIC by default +- sw64: fix specific performance counter error +- sw64: enable all HW PMUs for max sampling period events +- sw64: update exclusive counter handling +- sw64: fix perf event L1I cache write support +- sw64: locking/atomic: make atomic*_{cmp,}xchg optional +- sw64: sound: fix compile error when CONFIG_SND_DUMMY=y +- sw64: kexec: add support for crashkernel=size[KMG] +- sw64: bpf: add ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE to Kconfig +- sw64: ftrace: save&restore more stack frame in ftrace_caller +- sw64: enable -fsw-unalign-byte for new subarchs +- sw64: reduce jump instructions in entUna +- sw64: simplify GP handling for entUna +- sw64: fix secondary_cpu_start for CORE3B +- sw64: reimplement cmpxchg and xchg +- sw64: change condition of some cflags +- sw64: put cpu_set_node() in init text +- sw64: make sw64_(read/write)_csr always inline +- sw64: compile divide.S for c3b only +- sw64: vdso: add support for time namespaces +- sw64: rename PT_REGS_SP to PT_REGS_R30 +- sw64: modify parameters of the exception functions +- sw64: kvm: modify parameters of the exception functions +- sw64/vdso: Support mremap() for vDSO +- sw64/kvm: fix a bug for starting CORE3 VM +- sw64: ftrace: fix atomicity bugs in ftrace +- sw64: fix copy_oldmem_page according to upstream +- sw64: fix nmi_(enter/exit) error in CORE3B +- sw64: add a 1ms delay before CPU offline +- sw64: topology: discard topology information from PPTT +- sw64: iommu: implement resv_region APIs +- sw64: pci: fix secondary bus reset issue +- sw64: sound: avoid pagefault in pcm driver's memset +- sw64: flush TLB after modifying ptbr_sys +- sw64: fix an error when running no main() program +- sw64: cache: fix cache ID is not unique when ACPI disabled +- sw64: ftrace: implement ftrace_modify_call +- sw64: kvm: fix incorrect vm_flags in vmem_mmap() +- sw64: cpu: give preference to CPU information from SMBIOS +- sw64: irqchip: use sunway as vendor prefix in device tree +- sw64: irqchip: use pr_fmt instead of prefix string macro +- sw64: numa: decrease CONFIG_NODES_SHIFT to 3 +- sw64: acpi: suppress meaningless error log when acpi=off +- sw64: emulator: refactor shutdown and restart functions +- perf jitdump: Add SW64 support +- sw64: emulator: add shutdown and restart functions +- sw64: emulator: fix the multi-core boot issue +- sw64: fix bad address error for vfio +- sw64: acpi: add support for ACPI S3 +- sw64: iommu: restore iommu_map/unmap callbacks +- sw64: iommu: work around lazy translation initialization +- sw64: iommu: adjust def_domain_type +- sw64: iommu: fix iommu_v1 capability +- sw64: fix iommu commandline parameter +- sw64: iommu: remove unnecessary locking in map/unmap_pages +- sw64: fix no CONFIG_DYNAMIC_FTRACE_WITH_REGS error +- sw64: fix kprobe function +- sw64: perf: fix specific performance events can be sampled +- sw64: bpf: optimize JIT for load imm +- sw64: bpf: fix BPF_CALL JIT for multi-function programs +- sw64: bpf: optimize some instructions +- sw64: bpf: fix XADD32 and XADD64 +- sw64: kexec: fix a bug in menuconfig +- sw64: kexec: fix a compilation error +- sw64: iommu: mark hose->iommu_enable as true before probing +- sw64: iommu: fix work around of iommu interrupt remapping capability +- sw64: iommu: fix def_domain_type callback +- sw64: iommu: introduce PCI device check before iommu probing +- !16085 init/Kconfig: Add SMP to the dependencies of QOS_SCHED +- !16311 kernel: Remove signal hacks for vhost_tasks +- !16365 Add CFTYPE_NOT_ON_ROOT for qos_level +- !16364 gfp: include __GFP_NOWARN in GFP_NOWAIT +- eventfd:Introduce the eventfd batch processing interface +- !16369 ftrace: Add cond_resched() to ftrace_graph_set_hash() +- x86/ptrace: Fix kabi breakage in struct pt_regs +- x86/ptrace: Add FRED additional information to the pt_regs structure +- x86/ptrace: Cleanup the definition of the pt_regs structure +- ksmbd: Fix UAF in __close_file_table_ids +- ext4: ignore xattrs past end +- ftrace: Add cond_resched() to ftrace_graph_set_hash() +- !16367 scsi: sg: fix refcount underflow in +- scsi: sg: Enable runtime power management +- Revert "scsi: sg: Enable runtime power management" +- net: Fix kabi breakage in struct netlink_dump_control +- net: Fix kabi breakage in struct netlink_callback +- rtnetlink: add RTNL_FLAG_DUMP_UNLOCKED flag +- rtnetlink: change nlk->cb_mutex role +- Add CFTYPE_NOT_ON_ROOT for qos_level +- !16349 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.82-6.6.83 LTS Patches +- docs/core-api: memory-allocation: GFP_NOWAIT doesn't need __GFP_NOWARN +- gfp: include __GFP_NOWARN in GFP_NOWAIT +- !16359 net: hns3: some bugfixes for hns3 driver +- !16350 [OLK-6.6]Hygon: Address boot failures of CSV1/CSV2 VMs on Hygon 2g and C86-3g CPUs with firmware build ID below 1878 and 1810. +- !16281 【OLK-6.6】新增海光psp/ccp device id以及优化海光可信功能内核驱动 +- !16344 [OLK-6.6] backport important bugfix for nvme driver from upstream linux-6.6.y +- net: hns3: disable interrupt when ptp init failed +- net: hns3: avoid unnecessary checking when unmap buffer +- net: hns3: fix a use of uninitialized variable problem +- kbuild: userprogs: use correct lld when linking through clang +- pfifo_tail_enqueue: Drop new packet when sch->limit == 0 +- spi-mxs: Fix chipselect glitch +- x86/boot: Sanitize boot params before parsing command line +- x86/boot: Rename conflicting 'boot_params' pointer to 'boot_params_ptr' +- ima: Reset IMA_NONACTION_RULE_FLAGS after post_setattr +- x86/mm: Don't disable PCID when INVLPG has been fixed by microcode +- drm/i915/dsi: Use TRANS_DDI_FUNC_CTL's own port width macro +- ALSA: hda: realtek: fix incorrect IS_REACHABLE() usage +- mm: hugetlb: Add huge page size param to huge_ptep_get_and_clear() +- iio: adc: at91-sama5d2_adc: fix sama7g5 realbits value +- iio: dac: ad3552r: clear reset status flag +- iio: filter: admv8818: Force initialization of SDO +- eeprom: digsy_mtc: Make GPIO lookup table match the device +- bus: mhi: host: pci_generic: Use pci_try_reset_function() to avoid deadlock +- drivers: core: fix device leak in __fw_devlink_relax_cycles() +- intel_th: pci: Add Panther Lake-P/U support +- intel_th: pci: Add Panther Lake-H support +- intel_th: pci: Add Arrow Lake support +- mei: me: add panther lake P DID +- cdx: Fix possible UAF error in driver_override_show() +- KVM: x86: Explicitly zero EAX and EBX when PERFMON_V2 isn't supported by KVM +- KVM: SVM: Suppress DEBUGCTL.BTF on AMD +- KVM: SVM: Drop DEBUGCTL[5:2] from guest's effective value +- usb: xhci: Enable the TRB overfetch quirk on VIA VL805 +- xhci: pci: Fix indentation in the PCI device ID definitions +- usb: gadget: Check bmAttributes only if configuration is valid +- usb: gadget: Fix setting self-powered state on suspend +- usb: gadget: Set self-powered based on MaxPower and bmAttributes +- usb: typec: tcpci_rt1711h: Unmask alert interrupts to fix functionality +- usb: typec: ucsi: increase timeout for PPM reset operations +- usb: dwc3: gadget: Prevent irq storm when TH re-executes +- usb: dwc3: Set SUSPENDENABLE soon after phy init +- usb: gadget: u_ether: Set is_suspend flag if remote wakeup fails +- usb: renesas_usbhs: Flush the notify_hotplug_work +- usb: typec: ucsi: Fix NULL pointer access +- usb: quirks: Add DELAY_INIT and NO_LPM for Prolific Mass Storage Card Reader +- usb: hub: lack of clearing xHC resources +- usb: renesas_usbhs: Use devm_usb_get_phy() +- usb: renesas_usbhs: Call clk_put() +- gpio: rcar: Fix missing of_node_put() call +- net: dsa: mt7530: Fix traffic flooding for MMIO devices +- sched/fair: Fix potential memory corruption in child_cfs_rq_on_list +- ublk: set_params: properly check if parameters can be applied +- net-timestamp: support TCP GSO case for a few missing flags +- exfat: fix soft lockup in exfat_clear_bitmap +- x86/sgx: Fix size overflows in sgx_encl_create() +- vlan: enforce underlying device type +- ppp: Fix KMSAN uninit-value warning with bpf +- net: ipa: Enable checksum for IPA_ENDPOINT_AP_MODEM_{RX,TX} for v4.7 +- net: ipa: Fix QSB data for v4.7 +- net: ipa: Fix v4.7 resource group names +- be2net: fix sleeping while atomic bugs in be_ndo_bridge_getlink +- drm/sched: Fix preprocessor guard +- hwmon: fix a NULL vs IS_ERR_OR_NULL() check in xgene_hwmon_probe() +- llc: do not use skb_get() before dev_queue_xmit() +- ALSA: usx2y: validate nrpacks module parameter on probe +- tracing: probe-events: Remove unused MAX_ARG_BUF_LEN macro +- hwmon: (ad7314) Validate leading zero bits and return error +- hwmon: (ntc_thermistor) Fix the ncpXXxh103 sensor table +- hwmon: (pmbus) Initialise page count in pmbus_identify() +- caif_virtio: fix wrong pointer check in cfv_probe() +- net: gso: fix ownership in __udp_gso_segment +- bluetooth: btusb: Initialize .owner field of force_poll_sync_fops +- HID: google: fix unused variable warning under !CONFIG_ACPI +- wifi: iwlwifi: limit printed string from FW file +- mm: don't skip arch_sync_kernel_mappings() in error paths +- mm/page_alloc: fix uninitialized variable +- block: fix conversion of GPT partition name to 7-bit +- s390/traps: Fix test_monitor_call() inline assembly +- dma: kmsan: export kmsan_handle_dma() for modules +- rapidio: fix an API misues when rio_add_net() fails +- wifi: nl80211: reject cooked mode if it is set along with other flags +- Bluetooth: Add check for mgmt_alloc_skb() in mgmt_device_connected() +- Bluetooth: Add check for mgmt_alloc_skb() in mgmt_remote_name() +- mptcp: fix 'scheduling while atomic' in mptcp_pm_nl_append_new_local_addr +- x86/cpu: Properly parse CPUID leaf 0x2 TLB descriptor 0x63 +- x86/cpu: Validate CPUID leaf 0x2 EDX output +- x86/cacheinfo: Validate CPUID leaf 0x2 EDX output +- platform/x86: thinkpad_acpi: Add battery quirk for ThinkPad X131e +- drm/radeon: Fix rs400_gpu_init for ATI mobility radeon Xpress 200M +- hwmon: (peci/dimmtemp) Do not provide fake thresholds data +- ALSA: hda/realtek: update ALC222 depop optimize +- ALSA: hda/realtek - add supported Mic Mute LED for Lenovo platform +- ALSA: hda: intel: Add Dell ALC3271 to power_save denylist +- ALSA: seq: Avoid module auto-load handling at event delivery +- gpio: rcar: Use raw_spinlock to protect register access +- ksmbd: fix bug on trap in smb2_lock +- ksmbd: fix use-after-free in smb2_lock +- ksmbd: fix type confusion via race condition when using ipc_msg_send_request +- LoongArch: Set max_pfn with the PFN of the last page +- LoongArch: Use polling play_dead() when resuming from hibernation +- LoongArch: Convert unreachable() to BUG() +- tracing: tprobe-events: Fix a memory leak when tprobe with $retval +- !16325 perf/core: Fix WARN_ON(!ctx) in __free_event() for partial init +- !16326 mfd: ene-kb3930: Fix a potential NULL pointer dereference +- crypto: ccp: Fix for potential memory Out-of-Bounds access due to insufficient array size +- !16324 LoongArch: add PCA953X/2K3000 GMAC/IGC netcard/NR_CPUS=2048/SCHED_MC support +- !16352 [OLK-6.6] Fix sxe compile errors without CONFIG_PM_SLEEP +- Linkdata：net：fix sxe compile errors 'sxe_resume' defined but not used +- !16280 Add Loongson PWM controller support +- !16279 crypto: loongson: add wst se chip support +- riscv: cacheinfo: Use of_property_present() for non-boolean properties +- riscv: Prevent a bad reference count on CPU nodes +- riscv: cacheinfo: initialize cacheinfo's level and type from ACPI PPTT +- riscv: cacheinfo: remove the useless input parameter (node) of ci_leaf_init() +- NFS: O_DIRECT writes must check and adjust the file length +- x86/speculation: Add __update_spec_ctrl() helper +- net: enetc: VFs do not support HWTSTAMP_TX_ONESTEP_SYNC +- net: enetc: Replace ifdef with IS_ENABLED +- net: enetc: Remove setting of RX software timestamp +- drm/amdgpu: disable BAR resize on Dell G5 SE +- drm/amdgpu: Check extended configuration space register when system uses large bar +- smb: client: fix chmod(2) regression with ATTR_READONLY +- drm/i915/ddi: Fix HDMI port width programming in DDI_BUF_CTL +- drm/i915/xe2lpd: Move D2D enable/disable +- efi: Don't map the entire mokvar table to determine its size +- x86/amd_nb: Use rdmsr_safe() in amd_get_mmconfig_range() +- !16342 v5 Memory access profiler(SPE) driven NUMA balancing and damon +- !16302 IMA: support virtcca extend assigned register +- !15384 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.63-6.6.64 LTS Conflicts Patches +- !16229 drivers/iommu: Enhance the robustness of the iommu module +- KVM: SVM: CSV is unsupported if both the firmware with a build ID < 1878 and SME is not in use +- KVM: SVM: The ASID range available for CSV2 guests vary across different firmware +- yt6801: Add Motorcomm yt6801 PCIe driver +- !16328 drm/amd/display: prevent hang on link training fail +- !16310 ocfs2: validate l_tree_depth to avoid out-of-bounds access +- IMA: support virtcca extend assigned register +- nvme-tcp: fix premature queue removal and I/O failover +- config: Enable memory sampling based pmu for numa balance and damon by default +- arm-spe: Export boost SPE sampling info via tracefs tracepoint +- arm-spe: Add kernel cmdline option to enable SPE boost +- arm-spe: Boost SPE add TLB hot page and remote access tracking +- mm/damon/vaddr: Add demotion interface for migrating cold pages to target nodemask +- mm/damon/vaddr: Extend mem_sampling sysctl to support DAMON +- mm/damon/vaddr: Support hardware-assisted memory access sampling +- mm/numa: Add tracepoints for access sampling and NUMA page migration +- mm/mem_sampling: Add sysctl control for NUMA balancing integration +- mm/numa: Enable mem_sampling-based access tracking for NUMA balancing +- mm/numa: Use mem_sampling framework for NUMA balancing +- mm/mem_sampling:: Add proc and cmdline interface to control sampling enable +- sched: Enable per-process mem_sampling from sched switch path +- mm/mem_sampling: Add sched switch hook to control sampling state +- mm/mem_sampling: Add eBPF interface for memory access tracing +- mm_monitor/mm_spe: Introduce arm_spe_user to abstract SPE usage +- mm_monitor/mm_spe: Add PMU based memory sampling abstract layer +- mm_monitor/mm_spe: Init per-CPU buffers and SPE state +- mm_monitor/mm_spe: Introduce standalone SPE profiling framework +- !16323 bpf: Fix WARN() in get_bpf_raw_tp_regs +- !16286 arm64: Add support for FEAT_{LS64, LS64_V}. +- !12573 [OLK-6.6] Fix rnpvf compile errors without CONFIG_MXGBEVF_FIX_VF_QUEUE +- !15434 Backport PRM update and bugfixes up to v6.14. +- !16278 drm: LoongArch: fix rx550/gfx6/gfx7/gfx8 error +- drm/amd/display: prevent hang on link training fail +- mfd: ene-kb3930: Fix a potential NULL pointer dereference +- perf/core: Fix WARN_ON(!ctx) in __free_event() for partial init +- LoongArch: Prevent cond_resched() occurring within kernel-fpu +- LoongArch: 2K3000 GMAC support. +- LoongArch: Add SCHED_MC (Multi-core scheduler) support +- LoongArch: Enable GPIO_PCA953X driver +- LoongArch: Enable IGC driver +- LoongArch: CONFIG_NR_CPUS expanded to 2048 +- bpf: Fix WARN() in get_bpf_raw_tp_regs +- !16298 v2 CVE-2024-58098 +- !16218 bpf: Fix kmemleak warning for percpu hashmap +- !16217 bpf: Fix deadlock between rcu_tasks_trace and event_mutex. +- PCI: Fix reference leak in pci_register_host_bridge() +- KVM: arm64: Enable FEAT_{LS64, LS64_V} in the supported guest +- Workaround the issue when compile with CONFIG_FUNCTION_ALIGNMENT_64B +- arm64: Add support for FEAT_{LS64, LS64_V} +- arm64: Provide basic EL2 setup for FEAT_{LS64, LS64_V} usage at EL0/1 +- !16300 【olk 6.6】 add two bugfixes for ptp_hisi driver +- binfmt_elf: Wire up AT_HWCAP3 at AT_HWCAP4 +- uapi/auxvec: Define AT_HWCAP3 and AT_HWCAP4 aux vector, entries +- arm64: Support AT_HWCAP3 +- kernel: Remove signal hacks for vhost_tasks +- ocfs2: validate l_tree_depth to avoid out-of-bounds access +- hfs/hfsplus: fix slab-out-of-bounds in hfs_bnode_read_key +- ptp: hisi: the print of element in ptp need protected by lock +- ptp: hisi: fix list not delete problem +- !15157 Add support edac for hygon family 18h model 10h +- !15155 Add LS and IF mce types for Hygon family 18h model 7h +- bpf: Fix kabi breakage in struct bpf_subprog_info +- selftests/bpf: test for changing packet data from global functions +- bpf: track changes_pkt_data property for global functions +- bpf: refactor bpf_helper_changes_pkt_data to use helper number +- bpf: add find_containing_subprog() utility function +- crypto: ccp: add new pci device id and modify psp_do_cmd() interface to not rely on sev device +- !16271 kvm: x86: fix infinite loop in kvm_guest_time_update when tsc is 0 +- !16284 Sync from 6.6 for the sam kabi +- bcache: call force_wake_up_gc() if necessary in check_should_bypass() +- !15291 [Intel-SIG]dmaengine: idxd: Add a new DSA device ID for Granite Rapids-D platform +- !16277 Revert DP HPD module to fix the problem +- tracing: Return from tracing_buffers_read() if the file has been closed +- trace: Fix kabi breakage in struct trace_iterator +- ring-buffer: Make wake once of ring_buffer_wait() more robust +- tracing/ring-buffer: Fix wait_on_pipe() race +- !16192 drivers/yunsilicon: fix query/modify pkt_dst_info return error +- !16180 PINCTRL:ENABLE_CONFIG_PINCTRL for resolve Intel notebook touchpad issue; fw, net: wireless: Add RTW88 driver and compressed firmware support +- !16231 LoongArch: Return NULL from huge_pte_offset() for invalid PMD +- !16256 CVE-2025-21916 +- !16257 CVE-2025-21950 +- Revert "drm/sysfs: Register "ddc" symlink later" +- Revert "drm/ast: Implement polling for VGA and SIL164 connectors" +- Revert "drm: Call drm_atomic_helper_shutdown() at shutdown time for misc drivers" +- Revert "drm/hisilicon/hibmc: select CONFIG_DRM_DISPLAY_DP_HELPER" +- Revert "drm/hisilicon/hibmc: Restructuring the header dp_reg.h" +- Revert "drm/hisilicon/hibmc: Add dp serdes cfg to adjust serdes rate, voltage and pre-emphasis" +- Revert "drm/hisilicon/hibmc: Add dp serdes cfg in dp process" +- Revert "drm/hisilicon/hibmc: Refactor the member of drm_aux in struct hibmc_dp" +- Revert "drm/hisilicon/hibmc: Getting connector info and EDID by using AUX channel" +- Revert "drm/hisilicon/hibmc: Add colorbar-cfg feature and its debugfs file" +- Revert "drm/hisilicon/hibmc: Enable this hot plug detect of irq feature" +- Revert "drm/hisilicon/hibmc: Add MSI irq getting and requesting for HPD" +- Revert "drm/hisilicon/hibmc: Add vga connector detect functions" +- Revert "drm/hisilicon/hibmc: fix HPD interrupts triggering the wrong behavior" +- Revert "drm/hisilicon/hibmc: fix irq_request()'s irq name variable is local" +- pwm: Add Loongson pwm driver support +- dt-bindings: pwm: Add Loongson PWM controller +- crypto: loongson: add wst se chip support +- drm/amdgpu: Fix RX550 pcie order dislocation. +- drm/amdgpu: make duplicated EOP packet for GFX6 have real content +- drm/amdgpu: make duplicated EOP packet for GFX7/8 have real content +- drm/radeon: repeat the same EOP packet for EOP workaround on CIK +- drm/amdgpu: Fix pcie order dislocation +- !16039 [OLK-6.6] 优化海光密钥管理虚拟化场景的性能 +- !16233 usb: chipidea: ci_hdrc_imx: fix usbmisc handling +- virtiofs: add filesystem context source name check +- kvm: x86: fix infinite loop in kvm_guest_time_update when tsc is 0 +- !16208 Fix CVE-2025-37860 +- !16189 jfs: Fix uninit-value access of imap allocated in the diMount() function +- drivers: virt: acrn: hsm: Use kzalloc to avoid info leak in pmcmd_ioctl +- usb: atm: cxacru: fix a flaw in existing endpoint checks +- !16183 xhci: Limit time spent with xHC interrupts disabled during bus resume +- !16193 net: libwx: handle page_pool_dev_alloc_pages error +- !16224 backlight: led_bl: Hold led_access lock when calling led_sysfs_disable() +- !16198 Fix CVE-2025-37830 +- sfc: fix NULL dereferences in ef100_process_design_param() +- !16174 Add gpio-hisi softdepend +- pds_core: handle unsupported PDS_CORE_CMD_FW_CONTROL result +- usb: chipidea: ci_hdrc_imx: fix usbmisc handling +- LoongArch: Return NULL from huge_pte_offset() for invalid PMD +- backlight: led_bl: Hold led_access lock when calling led_sysfs_disable() +- media: vimc: skip .s_stream() for stopped entities +- media: subdev: Add v4l2_subdev_is_streaming() +- media: subdev: Improve v4l2_subdev_enable/disable_streams_fallback +- media: subdev: Fix use of sd->enabled_streams in call_s_stream() +- drivers/iommu: Enhance the robustness of the iommu module +- driver/vfio: disable vfio pci intx +- drivers/iommu: Define the alias of the iommu device +- loongarch/configs: enable vfio mdev config +- drivers/iommu: Add the "iommu disable" probe +- !16196 net/hinic3: Synchronize new NIC features and bug fixes +- f2fs: fix potential deadloop in prepare_compress_overwrite() +- bpf: Fix kmemleak warning for percpu hashmap +- bpf: Fix deadlock between rcu_tasks_trace and event_mutex. +- ax25: Remove broken autobind +- ibmvnic: Use kernel helpers for hex dumps +- !16165 vmxnet3: Fix malformed packet sizing in vmxnet3_process_xdp +- !12616 [OLK-6.6] fix bug：compiler error on loongarch64 for mucse rnpm driver +- !16186 【olk 6.6】net: hns3: backport some bugfixes +- !15158 Add support for Hygon family 18h model 8h +- !14328 Fixed multiple vfio devices not working properly +- !16081 Revert "LoongArch: Set hugetlb mmap base address aligned with pmd size" +- !16024 [OLK-6.6][bugfix]Hygon: Remove HYDCU fixup header driver from kernel source code +- !16026 [OLK-6.6]Hygon: In a CSV3 VM, the hypercall should notify real page enc/dec status +- !16025 [OLK-6.6][bugfix]Hygon：Avoid concurrent pinning shared pages at both KVM side and vfio side when passthrough device to CSV3 VM +- !16020 [OLK-6.6]Backport some mm related commits from Linux upstream +- !16019 [OLK-6.6][bugfix]Hygon: Return 0 at the beginning of csv_guest_hygon_coco_extension() for non-CSV3 VMs +- cpufreq: scmi: Fix null-ptr-deref in scmi_cpufreq_get_rate() +- net/hinic3: Synchronize new NIC features and bug fixes +- drivers/yunsilicon: fix query/modify pkt_dst_info return error +- net: libwx: handle page_pool_dev_alloc_pages error +- jfs: Fix uninit-value access of imap allocated in the diMount() function +- xhci: Limit time spent with xHC interrupts disabled during bus resume +- Add gpio-hisi softdepend +- net: hns3: defer calling ptp_clock_register() +- net: hns3: fixed debugfs tm_qset size +- net: hns3: fix an interrupt residual problem +- net: hns3: store rx VLAN tag offload state for VF +- KVM: x86/svm: Add hygon_set_guest_pat_wb parameter for non-passthrough application scenarios +- !16087 CVE-2025-21979 +- vmxnet3: Fix malformed packet sizing in vmxnet3_process_xdp +- !16102 V3:mm: shmem: skip swapcache for swapin of synchronous swap device +- !16144 [openEuler-24.03-LTS-SP2] drivers: update Yunsilicon drivers to 2412GA +- !16101 scsi: qla1280: Fix kernel oops when debug level > 2 +- !16103 md/raid1,raid10: don't ignore IO flags +- !16055 mailbox: pcc: Always clear the platform ack interrupt first +- !16143 Revert "tmpfs: fault in smaller chunks if large folio allocation not allowed" +- !16108 mm: zswap: handle incorrect attempts to load large folios +- !16075 mpage: fix softlockup in mpage_readahead() +- !16079 v2 tracing: Verify event formats that have "%*p.." +- Revert "tmpfs: fault in smaller chunks if large folio allocation not allowed" +- !16117 PCI: brcmstb: Fix error path after a call to regulator_bulk_get() +- !16116 spufs: fix gang directory lifetimes +- drivers: update Yunsilicon drivers to version rel_2412_std_card +- mailbox: pcc: Always clear the platform ack interrupt first +- mailbox: pcc: Fix the possible race in updation of chan_in_use flag +- !16099 fs/resctrl: Create l2 cache monitors +- PCI: brcmstb: Fix error path after a call to regulator_bulk_get() +- spufs: fix gang directory lifetimes +- !15719 vdpa/iommufd: Support taking over vDPA devices' iommu groups through the iommufd API +- !14169 [OLK-6.6] Fix rnp errors dmesg show in hygon +- !14174 [OLK-6.6] Fix os crash when cat some sys interface when net down +- !14100 [OLK-6.6] mucse: rnpm ocp nic support hotplug and show sub-zero temperature +- !16080 PCI/ASPM: Fix link state exit during switch upstream function removal +- !16074 x86/mm: Fix flush_tlb_range() when used for zapping normal PMDs +- !16073 acpi: nfit: fix narrowing conversion in acpi_nfit_ctl +- !15908 soc: qcom: pdr: Fix the potential deadlock +- !16094 ksmbd: fix session use-after-free in multichannel connection +- mm: zswap: handle incorrect attempts to load large folios +- md/raid1,raid10: don't ignore IO flags +- !16086 media: streamzap: fix race between device disconnection and urb callback +- mm: shmem: skip swapcache for swapin of synchronous swap device +- mm: swap: remove 'synchronous' argument to swap_read_folio() +- vhost/vdpa: Add support to bind and attach iommufd +- scsi: qla1280: Fix kernel oops when debug level > 2 +- !15897 Two backport bugfix patches +- !15948 mm/huge_memory: drop beyond-EOF folios with the right number of refs +- arm64/mpam: Refuse to enter powerdown state after L2 msc updated +- arm64/mpam: Refuse cpu offline when L2 msc is enabled +- fs/resctrl: Add l2 mount option to enable L2 msc +- arm64: mpam: Restore the expected MPAM sysregs on cpuhp +- arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs +- fs/resctrl: Create l2 cache monitors +- x86/resctrl: Add a handling path of default label in get_arch_mbm_state() +- ksmbd: fix session use-after-free in multichannel connection +- !16034 swap-in support large folio +- !15994 ksmbd: fix use-after-free in ksmbd_sessions_deregister() +- !16066 rds: sysctl: rds_tcp_{rcv,snd}buf: avoid using current->nsproxy +- !16062 arm64/mpam: Expanding MPAM's QoS capability +- !16053 jfs: add check read-only before truncation in jfs_truncate_nolock() +- !16037 ext4: fix the logic to determine whether buffered IO can use iomap +- wifi: cfg80211: init wiphy_work before allocating rfkill fails +- wifi: cfg80211: cancel wiphy_work before freeing wiphy +- media: streamzap: fix race between device disconnection and urb callback +- !15986 RDMA/hns: Fix the compatibility between the kernel mode and user mode +- init/Kconfig: Add SMP to the dependencies of QOS_SCHED +- Revert "LoongArch: Set hugetlb mmap base address aligned with pmd size" +- PCI/ASPM: Fix link state exit during switch upstream function removal +- tracing: Verify event formats that have "%*p.." +- mpage: fix softlockup in mpage_readahead() +- x86/mm: Fix flush_tlb_range() when used for zapping normal PMDs +- acpi: nfit: fix narrowing conversion in acpi_nfit_ctl +- !16045 irqchip/mbigen: add check before deference 'mgn_chip' +- rds: sysctl: rds_tcp_{rcv,snd}buf: avoid using current->nsproxy +- arm64/mpam: Add limit feature +- arm64/mpam: Add PRIO feature +- arm64/mpam: Add mbw_min and cmin features +- arm64/mpam: Add CMAX feature +- fs/resctrl: As a pre-patch for expanding MPAM's QoS capability +- perf/x86/uncore: Add DF PMU support for Hygon family 18h model 8h +- soc: hisilicon: kunpeng_hccs: Fix incorrect string assembly +- jfs: add check read-only before truncation in jfs_truncate_nolock() +- !15887 LoongArch: sync internal repository 20250414 +- !15937 md/raid1: fix memleak and double free in +- irqchip/mbigen: add check before deference 'mgn_chip' +- crypto: ccp: Support vpsp ringbuffer overcommit +- ext4: fix the logic to determine whether buffered IO can use iomap +- mm: add per-order mTHP swap-in fallback/fallback_charge counters +- mm: add per-order mTHP swpin counters +- mm: fix PSWPIN counter for large folios swap-in +- mm: remove unused stub for can_swapin_thp() +- mm: zswap: fix zswap_never_enabled() for CONFIG_ZSWAP==N +- mm: support large folios swap-in for sync io devices +- mm: add nr argument in mem_cgroup_swapin_uncharge_swap() helper to support large folios +- mm: zswap: add zswap_never_enabled() +- mm: convert swap_cluster_readahead and swap_vma_readahead to return a folio +- mm: return a folio from read_swap_cache_async() +- mm: remove page_swap_info() +- mm: convert swap_readpage() to swap_read_folio() +- mm: convert swap_page_sector() to swap_folio_sector() +- mm: pass a folio to swap_readpage_bdev_async() +- mm: pass a folio to swap_readpage_bdev_sync() +- mm: pass a folio to swap_readpage_fs() +- mm: pass a folio to swap_writepage_bdev_async() +- mm: pass a folio to swap_writepage_bdev_sync() +- mm: pass a folio to swap_writepage_fs() +- mm: pass a folio to __swap_writepage() +- mm: return the folio from __read_swap_cache_async() +- crypto: ccp: Use a workqueue to clean up the vpsp ringbuffer +- crypto: ccp: move vpsp-related functions to vpsp.c +- !15931 CVE-2025-22113 +- !16023 CVE-2025-22121 +- x86/kvm: Don't hypercall to enc all usable RAM on CSV3 platform +- KVM: SVM: CSV: fix CSV3 launch failures because of concurrent longterm pin +- Revert "drm/hygon: Add support to passthrough Hygon DCU to virtual machine" +- Revert "x86/config: Set CONFIG_HYDCU_FIXUP_HEADER=y by default" +- ext4: Restore the deleted check paths of the xattr. +- ext4: fix out-of-bound read in ext4_xattr_inode_dec_ref_all() +- ext4: introduce ITAIL helper +- !15918 [OLK-6.6] 从上游linux-6.6.y回合nvme驱动重要bugfix +- !15589 [OLK-6.6] Ethernet: Supports Linkdata ethernet Controllers +- !15962 LoongArch: BPF: Don't override subprog's return value +- !16007 compile error: implicit declaration of +- mm/hugetlb: wait for hugetlb folios to be freed +- mm: replace free hugepage folios after migration +- mm/cma: using per-CMA locks to improve concurrent allocation performance +- KVM: SEV: Use long-term pin when registering encrypted memory regions +- Revert "KVM: SEV: Pin SEV guest memory out of CMA area" +- KVM: SVM: CSV: Return 0 at the beginning of csv_guest_hygon_coco_extension() for non-CSV3 VMs +- kvm: hisi_virt: Fix compilation error due to missing ACPI configuration +- !15981 LoongArch: BPF: Fix off-by-one error in build_prologue() +- !15935 HID: appleir: Fix potential NULL dereference at raw event handle +- !15988 fs/resctrl: Fix configuration to wrong control group when CDP is enabled +- ksmbd: fix use-after-free in ksmbd_sessions_deregister() +- !15970 erofs/cachefiles: Change the unmark inuse sequence in erofs ondemand mode +- !15987 v12 Add support for IPIv +- fs/resctrl: Fix configuration to wrong control group when CDP is enabled +- arm64/mpam: Add debugging information about CDP monitor value +- arm64/mpam: Fix allocated cache size information +- fs/resctrl: Fix kmemleak caused by closid_init() +- arm64/mpam: Correct the judgment condition of the CMAX feature +- arm64/mpam: Set the cpbm width of msc class with the minimum +- fs/resctrl: Determine whether the MBM monitors require overflow checking +- arm64/mpam: Optimize CSU/MBWU monitor multiplexing +- arm64/mpam: fix impossible condition in resctrl_arch_rmid_read() +- arm64/mpam: fix impossible condition in get_cpumask_from_cache_id() +- arm64/mpam: fix memleak in resctrl_arch_mon_ctx_alloc_no_wait() +- kabi: Use KABI_EXTEND to perform kabi repair for IPIV +- kvm: hisi_virt: Probe and configure IPIV capacity on HIP12 +- kvm: arm64: Add interface KVM_CAP_ARM_IPIV_MODE +- kvm: hisi_virt: Register ipiv exception interrupt +- irqchip: gicv3-its: Set base address of vm table and targe ITS when vpe schedule and deschedule +- kvm: arm64: avoid sending multi-SGIs in IPIV +- kvm: hisi_virt: Allocate VM table and save vpeid in it +- RDMA/hns: Fix the compatibility between the kernel mode and user mode +- !15980 ntb_hw_switchtec: Fix shift-out-of-bounds in switchtec_ntb_mw_set_trans +- !15979 thermal: int340x: Add NULL check for adev +- !15921 Support SMT control on arm64. +- !15975 perf/x86: Fix open counting event error +- LoongArch: BPF: Fix off-by-one error in build_prologue() +- ntb_hw_switchtec: Fix shift-out-of-bounds in switchtec_ntb_mw_set_trans +- thermal: int340x: Add NULL check for adev +- !15968 v3 Backport mainline patches to avoid crash caused by rsize being 0 +- !15960 iscsi_ibft: Fix UBSAN shift-out-of-bounds warning in ibft_attr_show_nic() +- !15974 RDMA/bnxt_re: Fix the page details for the srq created by kernel consumers +- !15952 ksmbd: validate zero num_subauth before sub_auth is accessed +- !15950 drm/hyperv: Fix address space leak when Hyper-V DRM device is removed +- perf/x86: Fix open counting event error +- RDMA/bnxt_re: Fix the page details for the srq created by kernel consumers +- erofs/cachefiles: Change the unmark inuse sequence in erofs ondemand mode +- smb: client: Update IO sizes after reconnection +- smb: client: Store original IO parameters and prevent zero IO sizes +- smb:client: smb: client: Add reverse mapping from tcon to superblocks +- Revert "cifs: Prevent NULL pointer dereference caused by cifs_sb->rsize is 0" +- !15958 cpufreq: governor: Fix negative 'idle_time' handling in dbs_update() +- !15916 LoongArch: Increase ARCH_DMA_MINALIGN up to 16 +- !15961 drm/mediatek: dp: drm_err => dev_err in HPD path to avoid NULL ptr +- !15945 crypto: hisilicon - some bugfix +- LoongArch: BPF: Don't override subprog's return value +- drm/mediatek: dp: drm_err => dev_err in HPD path to avoid NULL ptr +- iscsi_ibft: Fix UBSAN shift-out-of-bounds warning in ibft_attr_show_nic() +- !15929 arm_mpam: Add SMMU support for MPAM +- cpufreq: governor: Fix negative 'idle_time' handling in dbs_update() +- !15793 CVE-2025-21892 +- !15795 CVE-2025-21941 +- ksmbd: validate zero num_subauth before sub_auth is accessed +- drm/hyperv: Fix address space leak when Hyper-V DRM device is removed +- mm/huge_memory: drop beyond-EOF folios with the right number of refs +- !15871 v2 CVE-2024-57795 +- !15906 [OLK-6.6] drm/hisilicon/hibmc: some bugfixes for hibmc-drm driver +- !15834 uacce: fix for the numa distance calculation +- crypto: hisilicon/qm - disable error report before flr +- hisi_acc_vfio_pci: obtain the mailbox configuration at one time +- crypto: hisilicon/qm - obtain the mailbox configuration at one time +- crypto: hisilicon/qm - fix the pf2vf timeout when global reset +- crypto: hisilicon/qm - modify interrupt processing resource application +- crypto: hisilicon/qm - mask axi error before memory init +- crypto: hisilicon/qm - invalidate queues in use +- crypto: hisilicon/sec2 - fix memory use-after-free issue +- crypto: hisilicon/sec2 - fix for gcm spec check +- crypto: hisilicon/hpre - fix dma unmap sequence +- !15936 ASoC: imx-card: Add NULL check in imx_card_probe() +- !15922 mm: shmem: fix potential data corruption during shmem swapin +- !15924 mm/vmscan: drop checking if _deferred_list is empty before using TTU_SYNC +- !15913 block: fix resource leak in blk_register_queue() error path +- md/raid1: fix memory leak in raid1_run() if no active rdev +- md/raid1: don't free conf on raid0_run failure +- ASoC: imx-card: Add NULL check in imx_card_probe() +- !15923 RDMA/hns :Add some mainline patches to OLK-6.6 +- HID: appleir: Fix potential NULL dereference at raw event handle +- ext4: avoid journaling sb update on error if journal is destroying +- ext4: define ext4_journal_destroy wrapper +- iommu: Fix kabi broken of struct iommu_ops +- fs/resctrl: Move iommu_groups back when their associated RDT group is deleted +- fs/resctrl: Fix the iommu_group parsing process +- iommu/arm-smmu-v3: Check pointer valid before dereferencing it +- arm_mpam: Select CONFIG_RESCTRL_IOMMU on the ARM64 by default +- ACPI/MPAM: Parse the rest of the ACPI table +- fs/resctrl: Add support for assigning iommu_groups to resctrl groups +- arm_mpam: resctrl: Add iommu helpers to get/set the partid and pmg +- kobject: Add kset_get_next_obj() to allow a kset to be walked +- iommu: Add helper to retrieve iommu kset +- iommu: Add helpers to retrieve iommu_groups by id or kobject +- iommu: Add helpers to get and set the QoS state +- iommu/arm-smmu-v3: Add mpam helpers to query and set state +- iommu/arm-smmu-v3: Issue a batch of commands to the same cmdq +- iommu/arm-smmu-v3: Register SMMU capabilities with MPAM +- mm: fix filemap_get_folios_contig returning batches of identical folios +- mm: shmem: fix potential data corruption during shmem swapin +- mm/vmscan: drop checking if _deferred_list is empty before using TTU_SYNC +- RDMA/hns: Fix wrong maximum DMA segment size +- Revert "RDMA/hns: fix iommu_map_sg() failed when MR bigger than 4G" +- RDMA/hns: initialize db in update_srq_db() +- RDMA/hns: Remove unused parameters +- config: enable CONFIG_HOTPLUG_SMT for arm64 +- arm64: Kconfig: Enable HOTPLUG_SMT +- arm64: topology: Support SMT control on ACPI based system +- arch_topology: Support SMT control for OF based system +- cpu/SMT: Provide a default topology_is_primary_thread() +- Revert "arm64: Kconfig: Enable HOTPLUG_SMT" +- nvmet-tcp: Fix a possible sporadic response drops in weakly ordered arch +- nvme-tcp: fix possible UAF in nvme_tcp_poll +- !15904 v2 PCI: AER: fix deadlock in do_recovery +- !15910 v2 soc cache: Add support for HiSilicon L3 cache +- LoongArch: Increase ARCH_DMA_MINALIGN up to 16 +- block: fix resource leak in blk_register_queue() error path +- drm/hisilicon/hibmc: fix irq_request()'s irq name variable is local +- drm/hisilicon/hibmc: fix HPD interrupts triggering the wrong behavior +- !15909 coresight: tmc: Don't change the buffer size if it's in use +- soc cache: Modify default config to compile HiSilicon SoC cache driver +- soc cache: Support cache maintenance for HiSilicon SoC Hydra Home Agent +- soc cache: Add framework driver for HiSilicon SoC cache +- coresight: tmc: Don't change the buffer size if it's in use +- soc: qcom: pdr: Fix the potential deadlock +- PCI: AER: fix deadlock in do_recovery +- !15863 Some mainline patches merged into olk-6.6: drivers/hwmon +- !15812 rapidio: add check for rio_add_net() in rio_scan_alloc_net() +- !15821 LoongArch: Set hugetlb mmap base address aligned with pmd size +- !15882 Updates for HiSilicon L3C PMU driver +- drivers/pci: Enable pci bridge acs capability +- driver/iommu: Set iommu driver buildin kernel +- LoongArch: Revert "LoongArch: Fix cpu hotplug issue" +- LoongArch: KVM: Repair the restart pressure test virtual machine hang dead problem +- LoongArch:config: enable pci host controller fdt driver +- selftests: mincore: fix tmpfs mincore test failure +- !15886 Add HPD, getting EDID, colorbar features in DP function +- !15851 migration: adapt to new migration configuration +- LoongArch: Export some signal functions +- LoongArch: Make some signal and ptrace functions non-static +- LoongArch: Handle fp, lsx, lasx and lbt assembly symbols +- LoongArch: Preserve firmware configuration if ACPI requires. +- LoongArch: configs: enable some configs +- LoongArch: configs: set CONFIG_UNWINDER_ORC=y +- LoongArch: configs: Disable CONFIG_RT_GROUP_SCHED to prevent cgroup2 issues +- objtool: Fix C jump table annotations for Clang +- drm/hisilicon/hibmc: Add vga connector detect functions +- drm/hisilicon/hibmc: Add MSI irq getting and requesting for HPD +- drm/hisilicon/hibmc: Enable this hot plug detect of irq feature +- drm/hisilicon/hibmc: Add colorbar-cfg feature and its debugfs file +- drm/hisilicon/hibmc: Getting connector info and EDID by using AUX channel +- drm/hisilicon/hibmc: Refactor the member of drm_aux in struct hibmc_dp +- drm/hisilicon/hibmc: Add dp serdes cfg in dp process +- drm/hisilicon/hibmc: Add dp serdes cfg to adjust serdes rate, voltage and pre-emphasis +- drm/hisilicon/hibmc: Restructuring the header dp_reg.h +- drm/hisilicon/hibmc: select CONFIG_DRM_DISPLAY_DP_HELPER +- drm: Call drm_atomic_helper_shutdown() at shutdown time for misc drivers +- drm/ast: Implement polling for VGA and SIL164 connectors +- drm/sysfs: Register "ddc" symlink later +- !15612 Some trivial cleanups for shmem +- drivers/perf: hisi: Add support for L3C PMU v3 +- drivers/perf: hisi: Refactor the event configuration of L3C PMU +- drivers/perf: hisi: Extend the field of tt_core +- drivers/perf: hisi: Extract the event filter check of L3C PMU +- drivers/perf: hisi: Simplify the probe process of each L3C PMU version +- drivers/perf: hisi: Export hisi_uncore_pmu_isr() +- drivers/perf: hisi: Relax the event ID check in the framework +- !15872 【olk 6.6】backport some bugfixes for hibmcge driver +- !15852 Fix CVE-2025-22013 +- !15845 regulator: check that dummy regulator has been probed before using it +- !15854 perf arm-spe: Add support for SPE Data Source packet on HiSilicon HIP12 +- !15838 General updates of HiSilicon PMU drivers +- hwmon: (acpi_power_meter) Replace the deprecated hwmon_device_register +- hwmon: Fix the missing of 'average' word in hwmon_power_attr_templates +- hwmon: (acpi_power_meter) Ensure IPMI space handler is ready on Dell systems +- ACPI: IPMI: Add helper to wait for when SMI is selected +- RDMA/rxe: Fix the failure of ibv_query_device() and ibv_query_device_ex() tests +- RDMA/rxe: Remove the direct link to net_device +- net: hibmcge: fix multiple phy_stop() issue +- net: hibmcge: fix not restore rx pause mac addr after reset issue +- net: hibmcge: fix the incorrect np_link fail state issue. +- net: hibmcge: fix wrong mtu log issue +- net: hibmcge: fix the share of irq statistics among different network ports issue +- net: hibmcge: fix incorrect multicast filtering issue +- net: hibmcge: fix incorrect pause frame statistics issue +- Ethernet: Linkdata: Supports Linkdata ethernet Controllers +- migration: adapt to new migration configuration +- migration: qm updates BAR configuration +- migration: update BAR space size +- hisi_acc_vfio_pci: update function return values +- hisi_acc_vfio_pci: bugfix live migration function without VF device driver +- hisi_acc_vfio_pci: bugfix the problem of uninstalling driver +- hisi_acc_vfio_pci: bugfix cache write-back issue +- hisi_acc_vfio_pci: add eq and aeq interruption restore +- hisi_acc_vfio_pci: fix XQE dma address error +- !15839 ext4: Some zeroing fixes +- perf arm-spe: Add support for SPE Data Source packet on HiSilicon HIP12 +- arm64/fpsimd: Remove unused declaration fpsimd_kvm_prepare() +- KVM: arm64: Unconditionally save+flush host FPSIMD/SVE/SME state +- !15321 printk: Fix signed integer overflow when defining LOG_BUF_LEN_MAX +- regulator: check that dummy regulator has been probed before using it +- !15826 nvme-tcp: fix potential memory corruption in nvme_tcp_recv_pdu() +- !15754 v3 Support the FEAT_HDBSS introduced in Armv9.5 +- jbd2: fix off-by-one while erasing journal +- iomap: do not interrupt IOMAP_ZERO +- ext4: fix potential memory exposure issues during truncate in iomap mode. +- ext4: do not always order data when partial zeroing out a block +- mm: zero range of eof folio exposed by inode size extension +- mm: convert pagecache_isize_extended to use a folio +- ext4: goto right label 'out_mmap_sem' in ext4_setattr() +- drivers/perf: hisi: Add support for HiSilicon MN PMU driver +- drivers/perf: hisi: Add support for HiSilicon NoC PMU +- drivers/perf: hisi: Support PMUs with no interrupt +- drivers/perf: hisi: Relax the event number check of v2 PMUs +- drivers/perf: hisi: Add support for HiSilicon SLLC v3 PMU driver +- drivers/perf: hisi: Use ACPI driver_data to retrieve SLLC PMU information +- drivers/perf: hisi: Add support for HiSilicon DDRC v3 PMU driver +- drivers/perf: hisi: Simplify the probe process for each DDRC version +- drivers/perf: hisi: Delete redundant blank line of DDRC PMU +- drivers/perf: hisi: Fix incorrect variable name "hha_pmu" in DDRC PMU driver +- drivers/perf: hisi: Export associated CPUs of each PMU through sysfs +- drivers/perf: hisi: Provide a generic implementation of cpumask/identifier +- drivers/perf: hisi: Add a common function to retrieve topology from firmware +- drivers/perf: hisi: Extract topology information to a separate structure +- drivers/perf: hisi: Refactor the detection of associated CPUs +- drivers/perf: hisi: Migrate to one online CPU if no associated one online +- drivers/perf: hisi: Don't update the associated_cpus on CPU offline +- drivers/perf: hisi: Define a symbol namespace for HiSilicon Uncore PMUs +- !15827 perf stat: Enable iostat mode for HiSilicon PCIe PMU +- !15072 hisilicon: ACC Live Migration DFX Replacement Mainline Solution +- x86: openeuler_defconfig add CONFIG_VFIO_DEBUGFS=y +- arm64: openeuler_defconfig add CONFIG_VFIO_DEBUGFS=y +- PINCTRL:ENABLE_CONFIG_PINCTRL_AMD +- perf stat: Enable iostat mode for HiSilicon PCIe PMU +- LoongArch: KVM: Add interrupt checking with Loongson AVEC +- LoongArch: KVM: Reload guest CSR registers after S4 +- anolis: LoongArch: KVM: add virt extioi cpu encode support +- anolis: LoongArch: KVM: Fixed VM migration failure after ptw was enabled +- anolis: LoongArch: KVM: Add reset function for irqchip +- anolis: LoongArch: LSVZ: Clear LLBCTL if secondary mmu mapping is changed +- anolis: driver/iommu: Fixed multiple vfio devices not working properly +- anolis: LoongArch: KVM: enable ptw for kvm +- anolis: LoongArch: fix compile error when enable CONFIG_PARAVIRT +- vfio: fix kabi breakage due to struct vfio_device and enum vfio_device_mig_state +- Documentation: add debugfs description for vfio +- Documentation: add debugfs description for hisi migration +- MAINTAINERS: Add vfio debugfs interface doc link +- nvme-tcp: fix potential memory corruption in nvme_tcp_recv_pdu() +- MAINTAINERS: add Baolin as shmem reviewer +- mm: shmem: factor out the within_size logic into a new helper +- mm: shmem: change the return value of shmem_find_swap_entries() +- mm: shmem: remove duplicate error validation +- mm: shmem: remove 'fadvise()' comments +- mm: shmem: drop the unused macro +- docs: tmpfs: drop 'fadvise()' from the documentation +- !15611 Minimize xa_node allocation during xarry split +- docs: tmpfs: update the large folios policy for tmpfs and shmem +- mm: shmem: add a kernel command line to change the default huge policy for tmpfs +- hisi_acc_vfio_pci: register debugfs for hisilicon migration driver +- hisi_acc_vfio_pci: create subfunction for data reading +- hisi_acc_vfio_pci: extract public functions for container_of +- hisi_acc_vfio_pci: Remove the deferred_reset logic +- vfio/migration: Add debugfs to live migration driver +- Revert "vfio/migration: added map length page alignment" +- Revert "vfio/migration: bugfix some driver code" +- Revert "vfio/migration: add eq and aeq interruption restore" +- Revert "vfio/migration: bugfix cache write-back issue" +- Revert "vfio/migration: remove unused local variable" +- Revert "hisi-acc-vfio-pci:add DFX for acc migration driver" +- Revert "hisi_acc_vfio_pci: add exception error handling" +- Revert "migration: fix reference counting exception issue" +- Revert "migration: modify dfx error type without VM driver" +- !15636 Kunpeng Accelerator Enables New Features +- LoongArch: Set hugetlb mmap base address aligned with pmd size +- crypto: hisilicon/sec2 - fix for sec spec check +- crypto: hisilicon/sec2 - fix for aead authsize alignment +- crypto: hisilicon/sec2 - fix for aead auth key length +- crypto: hisilicon/hpre - adapt ECDH for high-performance cores +- crypto: hisilicon/qm - support new function communication +- Revert "crypto: hisilicon/qm - fix the pf2vf timeout when global reset" +- Revert "crypto: hisilicon/qm - obtain the mailbox configuration at one time" +- crypto: hisilicon/sec2 - fix for aead invalid authsize +- crypto: hisilicon/sec2 - fix for aead icv error +- Revert "crypto: hisilicon/sec2: fix memory use-after-free issue" +- Revert "crypto: hisilicon/sec2 - fix for aead icv error" +- Revert "crypto: hisilicon/sec2 - fix for aead invalid authsize" +- crypto: hisilicon/zip - support new error report +- Revert "crypto: hisilicon/qm - check device status before sending mailbox" +- Revert "hisi_acc_vfio_pci: obtain the mailbox configuration at one time" +- Revert "crypto: hisilicon/qm - disable error report before flr" +- crypto: hisilicon/zip - add data aggregation feature +- !15783 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.80-6.6.81 LTS Patches +- !15650 PCI/DPC: Ignore Surprise Down error on hot removal +- !15813 fix CVE-2025-21883 +- ice: Fix deinitializing VF in error path +- ice: add E830 HW VF mailbox message limit support +- ice: Add E830 device IDs, MAC type and registers +- !15800 proc: fix UAF in proc_get_inode() +- !15799 ksmbd: fix incorrect validation for num_aces field of smb_acl +- !15798 ksmbd: fix out-of-bounds in parse_sec_desc() +- !15749 CVE-2025-21963 && CVE-2025-21964 +- !15680 scsi: ufs: core: Fix use-after free in init error and remove paths +- !15681 kernel: be more careful about dup_mmap() failures and uprobe registering +- rapidio: add check for rio_add_net() in rio_scan_alloc_net() +- !15772 tracing/osnoise: Fix possible recursive locking for cpus_read_lock() +- !15789 slimbus: messaging: Free transaction ID in delayed interrupt scenario +- !15788 wifi: cfg80211: regulatory: improve invalid hints checking +- !15787 i2c: npcm: disable interrupt enable bit before devm_request_irq +- !15786 ASoC: SOF: ipc4-topology: Harden loops for looking up ALH copiers +- firmware: cs_dsp: Ensure cs_dsp_load[_coeff]() returns 0 on success +- scsi: ufs: core: Cancel RTC work during ufshcd_remove() +- scsi: ufs: core: Start the RTC update work later +- scsi: ufs: core: Fix another deadlock during RTC update +- x86/microcode/AMD: Fix a -Wsometimes-uninitialized clang false positive +- scsi: ufs: core: Fix deadlock during RTC update +- x86/microcode/intel: Remove unnecessary cache writeback and invalidation +- arm64: dts: rockchip: Disable DMA for uart5 on px30-ringneck +- intel_idle: Handle older CPUs, which stop the TSC in deeper C states, correctly +- gve: set xdp redirect target only when it is available +- amdgpu/pm/legacy: fix suspend/resume issues +- riscv: signal: fix signal frame size +- riscv/futex: sign extend compare value in atomic cmpxchg +- rseq/selftests: Fix riscv rseq_offset_deref_addv inline asm +- sched/core: Prevent rescheduling when interrupts are disabled +- rcuref: Plug slowpath race in rcuref_put() +- vmlinux.lds: Ensure that const vars with relocations are mapped R/O +- mptcp: reset when MPTCP opts are dropped after join +- mptcp: always handle address removal under msk socket lock +- phy: exynos5-usbdrd: fix MPLL_MULTIPLIER and SSC_REFCLKSEL masks in refclk +- phy: tegra: xusb: reset VBUS & ID OVERRIDE +- net: enetc: fix the off-by-one issue in enetc_map_tx_tso_buffs() +- net: enetc: correct the xdp_tx statistics +- net: enetc: update UDP checksum when updating originTimestamp field +- net: enetc: keep track of correct Tx BD count in enetc_map_tx_tso_buffs() +- net: enetc: fix the off-by-one issue in enetc_map_tx_buffs() +- usbnet: gl620a: fix endpoint checking in genelink_bind() +- i2c: npcm: disable interrupt enable bit before devm_request_irq +- drm/amd/display: Fix HPD after gpu reset +- drm/amd/display: Disable PSR-SU on eDP panels +- perf/core: Fix low freq setting via IOC_PERIOD +- perf/x86: Fix low freqency setting issue +- ALSA: usb-audio: Re-add sample rate quirk for Pioneer DJM-900NXS2 +- riscv: KVM: Fix SBI TIME error generation +- riscv: KVM: Fix SBI IPI error generation +- riscv: KVM: Fix hart suspend status check +- RISCV: KVM: Introduce mp_state_lock to avoid lock inversion +- phy: rockchip: naneng-combphy: compatible reset with old DT +- x86/CPU: Fix warm boot hang regression on AMD SC1100 SoC systems +- io_uring/net: save msg_control for compat +- net: ti: icss-iep: Reject perout generation request +- net: ti: icss-iep: Remove spinlock-based synchronization +- include: net: add static inline dst_dev_overhead() to dst.h +- net/mlx5: IRQ, Fix null string in debug print +- net: mvpp2: cls: Fixed Non IP flow, with vlan tag flow defination. +- net: Clear old fragment checksum value in napi_reuse_skb +- ice: Fix deinitializing VF in error path +- ice: add E830 HW VF mailbox message limit support +- ice: Add E830 device IDs, MAC type and registers +- firmware: cs_dsp: Remove async regmap writes +- ipvs: Always clear ipvs_property flag in skb_scrub_packet() +- ASoC: es8328: fix route from DAC to output +- net: set the minimum for net_hotdata.netdev_budget_usecs +- net: loopback: Avoid sending IP packets without an Ethernet header +- afs: Fix the server_list to unuse a displaced server rather than putting it +- afs: Make it possible to find the volumes that are using a server +- rxrpc: rxperf: Fix missing decoding of terminal magic cookie +- Bluetooth: L2CAP: Fix L2CAP_ECRED_CONN_RSP response +- ALSA: usb-audio: Avoid dropping MIDI events at closing multiple ports +- sunrpc: suppress warnings for unused procfs functions +- RDMA/mlx5: Fix bind QP error cleanup flow +- scsi: core: Clear driver private data when retrying request +- SUNRPC: Handle -ETIMEDOUT return from tlshd +- SUNRPC: Prevent looping due to rpc_signal_task() races +- SUNRPC: convert RPC_TASK_* constants to enum +- scsi: ufs: core: Fix ufshcd_is_ufs_dev_busy() and ufshcd_eh_timed_out() +- scsi: ufs: core: Prepare to introduce a new clock_gating lock +- scsi: ufs: core: Introduce ufshcd_has_pending_tasks() +- scsi: ufs: core: Add UFS RTC support +- scsi: ufs: core: Add ufshcd_is_ufs_dev_busy() +- RDMA/mana_ib: Allocate PAGE aligned doorbell index +- IB/mlx5: Set and get correct qp_num for a DCT QP +- !15807 RDMA/mlx5: Fix a WARN during dereg_mr for DM type +- RDMA/mlx5: Fix a WARN during dereg_mr for DM type +- !15338 partitions: mac: fix handling of bogus partition table +- PCI/DPC: Ignore Surprise Down error on hot removal +- !15771 bpf: Add tracepoints with null-able arguments +- !15775 v2 CVE-2024-57857 +- !15790 fix CVE-2025-21976 +- proc: fix UAF in proc_get_inode() +- ksmbd: fix incorrect validation for num_aces field of smb_acl +- ksmbd: fix out-of-bounds in parse_sec_desc() +- !15785 v2 HID: intel-ish-hid: Fix use-after-free issue in ishtp_hid_remove() +- drm/amd/display: Fix null check for pipe_ctx->plane_state in resource_build_scaling_params +- RDMA/mlx5: Fix the recovery flow of the UMR QP +- fbdev: hyperv_fb: Allow graceful removal of framebuffer +- fbdev: hyperv_fb: Simplify hvfb_putmem +- fbdev: Introduce devm_register_framebuffer() +- !15757 Fix kmemleak false positive +- slimbus: messaging: Free transaction ID in delayed interrupt scenario +- wifi: cfg80211: regulatory: improve invalid hints checking +- i2c: npcm: disable interrupt enable bit before devm_request_irq +- ASoC: SOF: ipc4-topology: Harden loops for looking up ALH copiers +- ASoC: SOF: topology: Parse DAI type token for dspless mode +- ASoC: SOF: topology: dynamically allocate and store DAI widget->private +- !15396 fix CVE-2024-57999 +- !15762 scsi: ufs: core: bsg: Fix crash when arpmb command fails +- !15410 CVE-2025-21715 +- !15408 CVE-2025-21744 +- HID: intel-ish-hid: Fix use-after-free issue in ishtp_hid_remove() +- !15729 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.79-6.6.80 LTS Patches +- mtd: rawnand: cadence: fix unchecked dereference +- RDMA/siw: Remove direct link to net_device +- RDMA: Export ib_device_get_netdev() +- !15743 uprobes: Reject the shared zeropage in uprobe_write_opcode() +- ibmvnic: Inspect header requirements before using scrq direct +- ibmvnic: Perform tx CSO during send scrq direct +- x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit +- nilfs2: handle errors that nilfs_prepare_chunk() may return +- nilfs2: eliminate staggered calls to kunmap in nilfs_rename +- nilfs2: move page release outside of nilfs_delete_entry and nilfs_set_link +- arm64: dts: rockchip: change eth phy mode to rgmii-id for orangepi r1 plus lts +- md: Fix md_seq_ops() regressions +- ftrace: Do not add duplicate entries in subops manager ops +- ftrace: Correct preemption accounting for function tracing. +- EDAC/qcom: Correct interrupt enable register configuration +- mtd: rawnand: cadence: fix incorrect device in dma_unmap_single +- mtd: rawnand: cadence: use dma_map_resource for sdma address +- mtd: rawnand: cadence: fix error code in cadence_nand_init() +- mm,madvise,hugetlb: check for 0-length range after end address adjustment +- ASoC: SOF: pcm: Clear the susbstream pointer to NULL on close +- ALSA: hda/conexant: Add quirk for HP ProBook 450 G4 mute LED +- ALSA: hda: Add error check for snd_ctl_rename_id() in snd_hda_create_dig_out_ctls() +- ASoC: fsl_micfil: Enable default case in micfil_set_quality() +- ASoC: SOF: stream-ipc: Check for cstream nullity in sof_ipc_msg_data() +- nfp: bpf: Add check for nfp_app_ctrl_msg_alloc() +- lib/iov_iter: fix import_iovec_ubuf iovec management +- soc: loongson: loongson2_guts: Add check for devm_kstrdup() +- drop_monitor: fix incorrect initialization order +- tee: optee: Fix supplicant wait loop +- drm/i915/dp: Fix error handling during 128b/132b link training +- drm/i915: Make sure all planes in use by the joiner have their crtc included +- drm/msm/dpu: Disable dither in phys encoder cleanup +- arm64: dts: mediatek: mt8183: Disable DSI display output by default +- drm/nouveau/pmu: Fix gp10b firmware guard +- bpf: skip non exist keys in generic_map_lookup_batch +- nvme/ioctl: add missing space in err message +- drm/msm: Avoid rounding up to one jiffy +- nouveau/svm: fix missing folio unlock + put after make_device_exclusive_range() +- power: supply: da9150-fg: fix potential overflow +- bpf: Fix deadlock when freeing cgroup storage +- bpf: Disable non stream socket for strparser +- bpf, test_run: Fix use-after-free issue in eth_skb_pkt_type() +- drm/msm/gem: prevent integer overflow in msm_ioctl_gem_submit() +- drm/msm/gem: Demote userspace errors to DRM_UT_DRIVER +- drm/tidss: Fix race condition while handling interrupt registers +- drm/tidss: Add simple K2G manual reset +- !15768 gpio: aggregator: protect driver attr handlers against module unload +- !15755 x86/dumpstack: Fix inaccurate unwinding from exception stacks due to misplaced assignment +- tracing/osnoise: Fix possible recursive locking for cpus_read_lock() +- !15739 ftrace: Avoid potential division by zero in function_stat_show() +- !15748 tracing: Fix bad hist from corrupting named_triggers list +- !15750 perf/core: Fix pmus_lock vs. pmus_srcu ordering +- bpf: Add tracepoints with null-able arguments +- gpio: aggregator: protect driver attr handlers against module unload +- !15697 fs/dcache: fix bad unlock balance in shrink_dentry_list() +- scsi: ufs: core: bsg: Fix crash when arpmb command fails +- efi/arm64: Fix kmemleak false positive in arm64_efi_rt_init() +- x86/dumpstack: Fix inaccurate unwinding from exception stacks due to misplaced assignment +- arm64/kabi: use KABI_EXTEND to skip KABI check +- arm64/config: add config to control whether enable HDBSS feature +- arm64/kvm: support to handle the HDBSSF event +- arm64/kvm: using ioctl to enable/disable the HDBSS feature +- arm64/kvm: support set the DBM attr during memory abort +- arm64/sysreg: add HDBSS related register information +- perf/core: Fix pmus_lock vs. pmus_srcu ordering +- cifs: Fix integer overflow while processing acdirmax mount option +- cifs: Fix integer overflow while processing acregmax mount option +- tracing: Fix bad hist from corrupting named_triggers list +- !15613 CVE-2024-57952 +- !15724 CVE-2024-52560 +- !15277 drm/amdgpu: bail out when failed to load fw in psp_init_cap_microcode() +- uprobes: Reject the shared zeropage in uprobe_write_opcode() +- ftrace: Avoid potential division by zero in function_stat_show() +- !15641 block: don't revert iter for -EIOCBQUEUED +- !15515 media: uvcvideo: Fix crash during unbind if gpio unit is in use +- !15731 Some mainline patches merged into drivers/hwmon +- hwmon: (acpi_power_meter) Fix the fake power alarm reporting +- hwmon: (acpi_power_meter) Fix a check for the return value of read_domain_devices(). +- hwmon: (acpi_power_meter) Fix update the power trip points on failure +- hwmon: (acpi_power_meter) Fix uninitialized variables +- hwmon: (acpi_power_meter) Fix fail to load module on platform without _PMD method +- net: axienet: Set mac_managed_pm +- arp: switch to dev_getbyhwaddr() in arp_req_set_public() +- net: Add non-RCU dev_getbyhwaddr() helper +- flow_dissector: Fix port range key handling in BPF conversion +- flow_dissector: Fix handling of mixed port and port-range keys +- geneve: Suppress list corruption splat in geneve_destroy_tunnels(). +- tcp: adjust rcvq_space after updating scaling ratio +- vsock/bpf: Warn on socket without transport +- sockmap, vsock: For connectible sockets allow only connected +- ibmvnic: Don't reference skb after sending to VIOS +- ibmvnic: Add stat for tx direct vs tx batched +- ibmvnic: Introduce send sub-crq direct +- ibmvnic: Return error code on TX scrq flush fail +- ALSA: seq: Drop UMP events when no UMP-conversion is set +- net/sched: cls_api: fix error handling causing NULL dereference +- ALSA: hda/cirrus: Correct the full scale volume set logic +- geneve: Fix use-after-free in geneve_find_dev(). +- powerpc/code-patching: Fix KASAN hit by not flagging text patching area as VM_ALLOC +- ALSA: hda/realtek: Fixup ALC225 depop procedure +- powerpc/64s: Rewrite __real_pte() and __rpte_to_hidx() as static inline +- powerpc/64s/mm: Move __real_pte stubs into hash-4k.h +- ASoC: rockchip: i2s-tdm: fix shift config for SND_SOC_DAIFMT_DSP_[AB] +- USB: gadget: f_midi: f_midi_complete to call queue_work +- usb: gadget: core: flush gadget workqueue after device removal +- USB: gadget: core: create sysfs link between udc and gadget +- nvmem: imx-ocotp-ele: fix MAC address byte order +- nvmem: Move and rename ->fixup_cell_info() +- nvmem: Simplify the ->add_cells() hook +- nvmem: Create a header for internal sharing +- media: uvcvideo: Remove dangling pointers +- media: uvcvideo: Only save async fh if success +- media: uvcvideo: Refactor iterators +- soc: mediatek: mtk-devapc: Fix leaking IO map on driver remove +- soc/mediatek: mtk-devapc: Convert to platform remove callback returning void +- arm64: dts: qcom: sm8550: Fix ADSP memory base and length +- arm64: dts: qcom: sm8550: add missing qcom,non-secure-domain property +- arm64: dts: qcom: sm8550: Add dma-coherent property +- arm64: dts: qcom: sm8450: Fix ADSP memory base and length +- arm64: dts: qcom: sm8450: add missing qcom,non-secure-domain property +- scsi: core: Do not retry I/Os during depopulation +- scsi: core: Handle depopulation and restoration in progress +- firmware: qcom: scm: Fix missing read barrier in qcom_scm_is_available() +- ASoC: renesas: rz-ssi: Add a check for negative sample_space +- Input: synaptics - fix crash when enabling pass-through port +- Input: serio - define serio_pause_rx guard to pause and resume serio ports +- Bluetooth: qca: Fix poor RF performance for WCN6855 +- Bluetooth: qca: Update firmware-name to support board specific nvm +- Bluetooth: qca: Support downloading board id specific NVM for WCN7850 +- cpufreq: fix using cpufreq-dt as module +- cpufreq: dt-platdev: add missing MODULE_DESCRIPTION() macro +- memcg: fix soft lockup in the OOM process +- mm: update mark_victim tracepoints fields +- md/md-bitmap: Synchronize bitmap_get_stats() with bitmap lifetime +- md/md-bitmap: add 'sync_size' into struct md_bitmap_stats +- md/md-cluster: fix spares warnings for __le64 +- md/md-bitmap: replace md_bitmap_status() with a new helper md_bitmap_get_stats() +- !15711 Some bug fix patches for RDMA/hns to olk-6.6 +- !15705 tracing: Fix use-after-free in print_graph_function_flags during tracer switching +- !15704 perf/core: Add RCU read lock protection to perf_iterate_ctx() +- !15672 ovl: fix UAF in ovl_dentry_update_reval by moving dput() in ovl_link_up +- !15621 misc: fastrpc: Fix copy buffer page size +- !15620 s390/ism: add release function for struct device +- !15622 Fix UAF in acct(2) +- !15488 CVE-2025-21816 +- !15706 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.78-6.6.79 LTS Patches +- !14287 [OLK-6.6] SCSI: Support Linkdata HBA/RAID Controllers +- fs/ntfs3: Unify inode corruption marking with _ntfs_bad_inode() +- fs/ntfs3: Mark inode as bad as soon as error detected in mi_enum_attr() +- kbuild: hdrcheck: fix cross build with clang +- HID: hid-steam: Fix use-after-free when detaching device +- Revert "vfio/platform: check the bounds of read/write syscalls" +- x86/i8253: Disable PIT timer 0 when not in use +- vsock: Orphan socket after transport release +- vsock: Keep the binding until socket destruction +- io_uring/kbuf: reallocate buf lists on upgrade +- HID: hid-steam: Don't use cancel_delayed_work_sync in IRQ context +- alpha: replace hardcoded stack offsets with autogenerated ones +- mm: gup: fix infinite loop within __get_longterm_locked +- arm64: Filter out SVE hwcaps when FEAT_SVE isn't implemented +- md/md-bitmap: move bitmap_{start, end}write to md upper layer +- md: add a new callback pers->bitmap_sector() +- md/md-bitmap: remove the last parameter for bimtap_ops->endwrite() +- md/md-bitmap: factor behind write counters out from bitmap_{start/end}write() +- selftests: rtnetlink: update netdevsim ipsec output format +- netdevsim: print human readable IP address +- x86/static-call: Remove early_boot_irqs_disabled check to fix Xen PVH dom0 +- drm/v3d: Stop active perfmon if it is being destroyed +- drm/rcar-du: dsi: Fix PHY lock bit check +- drm/tidss: Clear the interrupt status for interrupts being disabled +- drm/tidss: Fix issue in irq handling causing irq-flood issue +- ipv6: mcast: add RCU protection to mld_newpack() +- ipv6: mcast: extend RCU protection in igmp6_send() +- ndisc: extend RCU protection in ndisc_send_skb() +- openvswitch: use RCU protection in ovs_vport_cmd_fill_info() +- arp: use RCU protection in arp_xmit() +- neighbour: use RCU protection in __neigh_notify() +- neighbour: delete redundant judgment statements +- ndisc: use RCU protection in ndisc_alloc_skb() +- HID: hid-steam: Move hidraw input (un)registering to work +- HID: hid-steam: Make sure rumble work is canceled on removal +- HID: hid-steam: Add Deck IMU support +- HID: hid-steam: Fix cleanup in probe() +- HID: hid-steam: remove pointless error message +- HID: hid-steam: Add gamepad-only mode switched to by holding options +- HID: hid-steam: Update list of identifiers from SDL +- HID: hid-steam: Clean up locking +- HID: hid-steam: Disable watchdog instead of using a heartbeat +- HID: hid-steam: Avoid overwriting smoothing parameter +- ipv6: icmp: convert to dev_net_rcu() +- ipv6: use RCU protection in ip6_default_advmss() +- flow_dissector: use RCU protection to fetch dev_net() +- ipv4: icmp: convert to dev_net_rcu() +- ipv4: use RCU protection in __ip_rt_update_pmtu() +- net: ipv4: Cache pmtu for all packet paths if multipath enabled +- ipv4: use RCU protection in inet_select_addr() +- ipv4: use RCU protection in rt_is_expired() +- ipv4: use RCU protection in ipv4_default_advmss() +- net: add dev_net_rcu() helper +- net: treat possible_net_t net pointer as an RCU one and add read_pnet_rcu() +- ipv4: add RCU protection to ip4_dst_hoplimit() +- btrfs: fix hole expansion when writing at an offset beyond EOF +- mlxsw: Add return value check for mlxsw_sp_port_get_stats_raw() +- igc: Set buffer type for empty frames in igc_init_empty_frame +- mmc: mtk-sd: Fix register settings for hs400(es) mode +- arm64: Handle .ARM.attributes section in linker scripts +- regmap-irq: Add missing kfree() +- regulator: qcom_smd: Add l2, l5 sub-node to mp5496 regulator +- partitions: mac: fix handling of bogus partition table +- gpio: stmpe: Check return value of stmpe_reg_read in stmpe_gpio_irq_sync_unlock +- gpiolib: acpi: Add a quirk for Acer Nitro ANV14 +- alpha: align stack for page fault and user unaligned trap handlers +- serial: 8250: Fix fifo underflow on flush +- efi: Avoid cold plugged memory for placing the kernel +- kbuild: userprogs: fix bitsize and target detection on clang +- wifi: ath12k: fix handling of 6 GHz rules +- alpha: make stack 16-byte aligned (most cases) +- can: etas_es58x: fix potential NULL pointer dereference on udev->serial +- can: j1939: j1939_sk_send_loop(): fix unable to send messages with data length zero +- can: c_can: fix unbalanced runtime PM disable in error path +- can: ctucanfd: handle skb allocation failure +- USB: serial: option: drop MeiG Smart defines +- USB: serial: option: fix Telit Cinterion FN990A name +- USB: serial: option: add Telit Cinterion FN990B compositions +- USB: serial: option: add MeiG Smart SLM828 +- usb: cdc-acm: Fix handling of oversized fragments +- USB: cdc-acm: Fill in Renesas R-Car D3 USB Download mode quirk +- usb: gadget: f_midi: fix MIDI Streaming descriptor lengths +- USB: Add USB_QUIRK_NO_LPM quirk for sony xperia xz1 smartphone +- USB: quirks: add USB_QUIRK_NO_LPM quirk for Teclast dist +- usb: core: fix pipe creation for get_bMaxPacketSize0 +- USB: pci-quirks: Fix HCCPARAMS register error for LS7A EHCI +- usb: dwc2: gadget: remove of_node reference upon udc_stop +- usb: gadget: udc: renesas_usb3: Fix compiler warning +- usb: roles: set switch registered flag early on +- usb: dwc3: Fix timeout issue during controller enter/exit from halt state +- usb: gadget: f_midi: Fixing wMaxPacketSize exceeded issue during MIDI bind retries +- perf/x86/intel: Ensure LBRs are disabled when a CPU is starting +- KVM: nSVM: Enter guest mode before initializing nested NPT MMU +- KVM: x86: Reject Hyper-V's SEND_IPI hypercalls if local APIC isn't in-kernel +- drm/amdgpu: avoid buffer overflow attach in smu_sys_set_pp_table() +- batman-adv: Drop unmanaged ELP metric worker +- batman-adv: Ignore neighbor throughput metrics in error case +- batman-adv: fix panic during interface removal +- ASoC: Intel: bytcr_rt5640: Add DMI quirk for Vexia Edu Atla 10 tablet 5V +- ACPI: x86: Add skip i2c clients quirk for Vexia EDU ATLA 10 tablet 5V +- selftests: gpio: gpio-sim: Fix missing chip disablements +- Grab mm lock before grabbing pt lock +- vfio/pci: Enable iowrite64 and ioread64 for vfio pci +- rtla/timerlat_top: Abort event processing on second signal +- rtla/timerlat_hist: Abort event processing on second signal +- scsi: ufs: bsg: Set bsg_queue to NULL after removal +- PCI: switchtec: Add Microchip PCI100X device IDs +- PCI/DPC: Quirk PIO log size for Intel Raptor Lake-P +- media: uvcvideo: Add Kurokesu C1 PRO camera +- media: uvcvideo: Add new quirk definition for the Sonix Technology Co. 292a camera +- media: uvcvideo: Implement dual stream quirk to fix loss of usb packets +- media: i2c: ds90ub953: Add error handling for i2c reads/writes +- media: i2c: ds90ub913: Add error handling to ub913_hw_init() +- media: cxd2841er: fix 64-bit division on gcc-9 +- soc/tegra: fuse: Update Tegra234 nvmem keepout list +- fbdev: omap: use threaded IRQ for LCD DMA +- RDMA/efa: Reset device on probe failure +- tools: fix annoying "mkdir -p ..." logs when building tools in parallel +- RDMA/hns: Fix remove debugfs after device has been unregistered +- RDMA/hns: Reorder uctx deallocation +- RDMA/hns: Fix a meaningless loop in free_buffer_pages_proc() +- RDMA/hns: Fix DCA error path in alloc_wqe_buf() +- RDMA/hns: Fix mismatched kzalloc vs kvfree +- RDMA/hns: Fix delay-destruction mechanism not processing kernel db +- RDMA/hns: Fix delayed destruction of db not taking effect +- RDMA/hns: Move mtr_node into the mtr struct +- RDMA/hns: Change mtr member to pointer in hns QP/CQ/MR/SRQ/EQ struct +- x86/xen: allow larger contiguous memory regions in PV guests +- xen/swiotlb: relax alignment requirements +- drm/amdgpu: bail out when failed to load fw in psp_init_cap_microcode() +- gpio: bcm-kona: Add missing newline to dev_err format string +- gpio: bcm-kona: Make sure GPIO bits are unlocked when requesting IRQ +- gpio: bcm-kona: Fix GPIO lock/unlock for banks above bank 0 +- drm/i915/selftests: avoid using uninitialized context +- cgroup: Remove steal time from usage_usec +- arm64: cacheinfo: Avoid out-of-bounds write to cacheinfo array +- team: better TEAM_OPTION_TYPE_STRING validation +- LoongArch: Fix idle VS timer enqueue +- vxlan: check vxlan_vnigroup_init() return value +- vrf: use RCU protection in l3mdev_l3_out() +- ndisc: ndisc_send_redirect() must use dev_get_by_index_rcu() +- ax25: Fix refcount leak caused by setting SO_BINDTODEVICE sockopt +- spi: sn-f-ospi: Fix division by zero +- HID: multitouch: Add NULL check in mt_input_configured +- pinctrl: cy8c95x0: Respect IRQ trigger settings from firmware +- tracing: Fix use-after-free in print_graph_function_flags during tracer switching +- perf/core: Add RCU read lock protection to perf_iterate_ctx() +- !15686 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.76-6.6.78 LTS Patches +- fs/dcache: fix bad unlock balance in shrink_dentry_list() +- !15645 nilfs2: do not force clear folio if buffer is referenced +- char: misc: deallocate static minor in error path +- ALSA: hda/realtek: Fix built-in mic assignment on ASUS VivoBook X515UA +- ALSA: hda/realtek: Fix wrong mic setup for ASUS VivoBook 15 +- ALSA: hda/realtek: Add quirks for ASUS ROG 2023 models +- ALSA: hda/realtek: Fix microphone regression on ASUS N705UD +- Revert "of: reserved-memory: Fix using wrong number of cells to get property 'alignment'" +- rtla/timerlat_top: Set OSNOISE_WORKLOAD for kernel threads +- rtla/timerlat_hist: Set OSNOISE_WORKLOAD for kernel threads +- Revert "rtla/timerlat_hist: Set OSNOISE_WORKLOAD for kernel threads" +- Revert "rtla/timerlat_top: Set OSNOISE_WORKLOAD for kernel threads" +- Revert "KVM: PPC: e500: Mark "struct page" dirty in kvmppc_e500_shadow_map()" +- Revert "KVM: PPC: e500: Mark "struct page" pfn accessed before dropping mmu_lock" +- Revert "KVM: PPC: e500: Use __kvm_faultin_pfn() to handle page faults" +- Revert "KVM: e500: always restore irqs" +- KVM: x86: Re-split x2APIC ICR into ICR+ICR2 for AMD (x2AVIC) +- KVM: x86: Make x2APIC ID 100% readonly +- tty: xilinx_uartps: split sysrq handling +- x86/mm/ident_map: Use gbpages only where full GB page should be mapped. +- mptcp: prevent excessive coalescing on receive +- btrfs: avoid monopolizing a core when activating a swap file +- Revert "btrfs: avoid monopolizing a core when activating a swap file" +- ocfs2: check dir i_size in ocfs2_find_entry +- net/ncsi: use dev_set_mac_address() for Get MC MAC Address handling +- spi: atmel-qspi: Memory barriers after memory-mapped I/O +- spi: atmel-quadspi: Create `atmel_qspi_ops` to support newer SoC families +- MIPS: ftrace: Declare ftrace_get_parent_ra_addr() as static +- rtc: zynqmp: Fix optional clock name property +- pinctrl: samsung: fix fwnode refcount cleanup if platform_get_irq_optional() fails +- rtla/timerlat_top: Stop timerlat tracer on signal +- rtla/timerlat_hist: Stop timerlat tracer on signal +- rtla: Add trace_instance_stop +- rtla/timerlat_top: Set OSNOISE_WORKLOAD for kernel threads +- rtla/timerlat_hist: Set OSNOISE_WORKLOAD for kernel threads +- rtla/osnoise: Distinguish missing workload option +- scripts/gdb: fix aarch64 userspace detection in get_current_task +- maple_tree: simplify split calculation +- net: phy: c45-tjaxx: add delay between MDIO write and read in soft_reset +- net/ncsi: wait for the last response to Deselect Package before configuring channel +- misc: fastrpc: Fix copy buffer page size +- misc: fastrpc: Fix registered buffer page address +- misc: fastrpc: Deregister device nodes properly in error scenarios +- misc: misc_minor_alloc to use ida for all dynamic/misc dynamic minors +- mtd: onenand: Fix uninitialized retlen in do_otp_read() +- irqchip/apple-aic: Only handle PMC interrupt as FIQ when configured so +- i3c: master: Fix missing 'ret' assignment in set_speed() +- NFC: nci: Add bounds checking in nci_hci_create_pipe() +- mailbox: tegra-hsp: Clear mailbox before using message +- ocfs2: fix incorrect CPU endianness conversion causing mount failure +- pnfs/flexfiles: retry getting layout segment for reads +- selftests: mptcp: connect: -f: no reconnect +- vfio/platform: check the bounds of read/write syscalls +- io_uring/rw: commit provided buffer state on async +- io_uring: fix io_req_prep_async with provided buffers +- io_uring/net: don't retry connect operation on EPOLLERR +- io_uring: fix multishots with selected buffers +- nvmem: imx-ocotp-ele: set word length to 1 +- nvmem: imx-ocotp-ele: fix reading from non zero offset +- nvmem: imx-ocotp-ele: simplify read beyond device check +- nvmem: core: improve range check for nvmem_cell_write() +- nvmem: qcom-spmi-sdam: Set size in struct nvmem_config +- crypto: qce - unregister previously registered algos in error path +- crypto: qce - fix goto jump in error path +- media: uvcvideo: Remove redundant NULL assignment +- media: uvcvideo: Support partial control reads +- media: uvcvideo: Fix event flags in uvc_ctrl_send_events +- media: uvcvideo: Fix crash during unbind if gpio unit is in use +- media: i2c: ds90ub960: Fix logging SP & EQ status only for UB9702 +- media: i2c: ds90ub960: Fix UB9702 VC map +- media: i2c: ds90ub960: Fix use of non-existing registers on UB9702 +- media: i2c: ds90ub9x3: Fix extra fwnode_handle_put() +- media: ccs: Fix cleanup order in ccs_probe() +- media: ccs: Fix CCS static data parsing for large block sizes +- media: ov5640: fix get_light_freq on auto +- media: imx296: Add standby delay during probe +- media: mc: fix endpoint iteration +- media: mmp: Bring back registration of the device +- soc: qcom: smem_state: fix missing of_node_put in error path +- soc: mediatek: mtk-devapc: Fix leaking IO map on error paths +- iio: light: as73211: fix channel handling in only-color triggered buffer +- media: ccs: Clean up parsed CCS static data on parse failure +- kfence: skip __GFP_THISNODE allocations on NUMA systems +- rv: Reset per-task monitors also for idle tasks +- tpm: Change to kvalloc() in eventlog/acpi.c +- ACPI: PRM: Remove unnecessary strict handler address checks +- xfs: Add error handling for xfs_reflink_cancel_cow_range +- xfs: Propagate errors from xfs_reflink_cancel_cow_range in xfs_dax_write_iomap_end +- pwm: microchip-core: fix incorrect comparison with max period +- arm64: tegra: Disable Tegra234 sce-fabric node +- arm64: tegra: Fix typo in Tegra234 dce-fabric compatible +- crypto: qce - fix priority to be less than ARMv8 CE +- arm64: dts: qcom: sm8550: correct MDSS interconnects +- arm64: dts: qcom: sm8550: Fix MPSS memory length +- arm64: dts: qcom: sm8550: Fix CDSP memory length +- arm64: dts: qcom: sm8450: Fix MPSS memory length +- arm64: dts: qcom: sm8450: Fix CDSP memory length +- arm64: dts: qcom: sm8350: Fix MPSS memory length +- arm64: dts: qcom: sm8350: Fix CDSP memory base and length +- arm64: dts: qcom: sm8350: Fix ADSP memory base and length +- arm64: dts: qcom: sm6375: Fix MPSS memory base and length +- arm64: dts: qcom: sm6375: Fix CDSP memory base and length +- arm64: dts: qcom: sm6375: Fix ADSP memory length +- arm64: dts: qcom: sm6350: Fix uart1 interconnect path +- arm64: dts: qcom: sm6350: Fix MPSS memory length +- arm64: dts: qcom: sm6350: Fix ADSP memory length +- arm64: dts: qcom: sm6115: Fix ADSP memory base and length +- arm64: dts: qcom: sm6115: Fix CDSP memory length +- arm64: dts: qcom: sm6115: Fix MPSS memory length +- ARM: dts: ti/omap: gta04: fix pm issues caused by spi module +- ARM: dts: dra7: Add bus_dma_limit for l4 cfg bus +- ubi: Add a check for ubi_num +- x86/boot: Use '-std=gnu11' to fix build with GCC 15 +- rust: init: use explicit ABI to clean warning in future compilers +- kbuild: Move -Wenum-enum-conversion to W=2 +- scsi: storvsc: Set correct data length for sending SCSI command without payload +- scsi: ufs: qcom: Fix crypto key eviction +- scsi: qla2xxx: Move FCE Trace buffer allocation to user control +- scsi: st: Don't set pos_unknown just after device recognition +- nvme-pci: Add TUXEDO IBP Gen9 to Samsung sleep quirk +- nvme-pci: Add TUXEDO InfinityFlex to Samsung sleep quirk +- PCI: endpoint: Finish virtual EP removal in pci_epf_remove_vepf() +- PCI: Avoid putting some root ports into D3 on TUXEDO Sirius Gen1 +- arm64: tegra: Fix Tegra234 PCIe interrupt-map +- ALSA: hda: Fix headset detection failure due to unstable sort +- ALSA: hda/realtek: Enable headset mic on Positivo C6400 +- Revert "media: uvcvideo: Require entities to have a non-zero unique ID" +- block: don't revert iter for -EIOCBQUEUED +- mips/math-emu: fix emulation of the prefx instruction +- dm-crypt: track tag_offset in convert_context +- dm-crypt: don't update io->sector after kcryptd_crypt_write_io_submit() +- powerpc/pseries/eeh: Fix get PE state translation +- LoongArch: Extend the maximum number of watchpoints +- MIPS: Loongson64: remove ROM Size unit in boardinfo +- serial: sh-sci: Do not probe the serial port if its slot in sci_ports[] is in use +- serial: sh-sci: Drop __initdata macro for port_cfg +- soc: qcom: socinfo: Avoid out of bounds read of serial number +- ASoC: acp: Support microphone from Lenovo Go S +- usbnet: ipheth: document scope of NCM implementation +- usbnet: ipheth: fix DPE OoB read +- usbnet: ipheth: break up NCM header size computation +- usbnet: ipheth: refactor NCM datagram loop +- usbnet: ipheth: check that DPE points past NCM header +- usbnet: ipheth: use static NDP16 location in URB +- usb: gadget: f_tcm: Don't prepare BOT write request twice +- usb: gadget: f_tcm: ep_autoconfig with fullspeed endpoint +- usb: gadget: f_tcm: Decrement command ref count on cleanup +- usb: gadget: f_tcm: Translate error to sense +- wifi: mt76: mt7915: add module param to select 5 GHz or 6 GHz on MT7916 +- wifi: rtw88: sdio: Fix disconnection after beacon loss +- wifi: mt76: mt7921u: Add VID/PID for TP-Link TXE50UH +- wifi: brcmfmac: fix NULL pointer dereference in brcmf_txfinalize() +- wifi: rtlwifi: rtl8821ae: Fix media status report +- HID: hid-sensor-hub: don't use stale platform-data on remove +- of: reserved-memory: Fix using wrong number of cells to get property 'alignment' +- of: Fix of_find_node_opts_by_path() handling of alias+path+options +- of: Correct child specifier used as input of the 2nd nexus node +- scsi: ufs: core: Fix the HIGH/LOW_TEMP Bit Definitions +- perf bench: Fix undefined behavior in cmpworker() +- efi: libstub: Use '-std=gnu11' to fix build with GCC 15 +- clk: mediatek: mt2701-mm: add missing dummy clk +- clk: mediatek: mt2701-img: add missing dummy clk +- clk: mediatek: mt2701-bdp: add missing dummy clk +- clk: mediatek: mt2701-aud: fix conversion to mtk_clk_simple_probe +- clk: mediatek: mt2701-vdec: fix conversion to mtk_clk_simple_probe +- clk: qcom: clk-rpmh: prevent integer overflow in recalc_rate +- clk: qcom: gcc-mdm9607: Fix cmd_rcgr offset for blsp1_uart6 rcg +- clk: qcom: gcc-sm6350: Add missing parent_map for two clocks +- clk: qcom: gcc-sm8550: Do not turn off PCIe GDSCs during gdsc_disable() +- clk: qcom: clk-alpha-pll: fix alpha mode configuration +- media: i2c: ds90ub960: Fix UB9702 refclk register access +- clk: sunxi-ng: a100: enable MMC clock reparenting +- Bluetooth: L2CAP: accept zero as a special value for MTU auto-selection +- Bluetooth: L2CAP: handle NULL sock pointer in l2cap_sock_alloc +- drm/i915: Drop 64bpp YUV formats from ICL+ SDR planes +- drm/komeda: Add check for komeda_get_layer_fourcc_list() +- drm/i915: Fix page cleanup on DMA remap failure +- drm/i915/guc: Debug print LRC state entries only if the context is pinned +- Revert "drm/amd/display: Use HW lock mgr for PSR1" +- drm/amdkfd: only flush the validate MES contex +- drm/amd/pm: Mark MM activity as unsupported +- ksmbd: fix integer overflows on 32 bit systems +- KVM: s390: vsie: fix some corner-cases when grabbing vsie pages +- KVM: Explicitly verify target vCPU is online in kvm_get_vcpu() +- arm64: dts: rockchip: increase gmac rx_delay on rk3399-puma +- drm/rockchip: cdn-dp: Use drm_connector_helper_hpd_irq_event() +- KVM: arm64: timer: Always evaluate the need for a soft timer +- fs/proc: do_task_stat: Fix ESP not readable during coredump +- m68k: vga: Fix I/O defines +- s390/futex: Fix FUTEX_OP_ANDN implementation +- smb: client: change lease epoch type from unsigned int to __u16 +- smb: client: fix order of arguments of tracepoints +- drm/modeset: Handle tiled displays in pan_display_atomic. +- ALSA: hda/realtek: Enable Mute LED on HP Laptop 14s-fq1xxx +- leds: lp8860: Write full EEPROM, not only half of it +- cpufreq: s3c64xx: Fix compilation warning +- rxrpc: Fix call state set to not include the SERVER_SECURING state +- net: sched: Fix truncation of offloaded action statistics +- tun: revert fix group permission check +- netem: Update sch->q.qlen before qdisc_tree_reduce_backlog() +- ACPI: property: Fix return value for nval == 0 in acpi_data_prop_read() +- x86/xen: add FRAME_END to xen_hypercall_hvm() +- x86/xen: fix xen_hypercall_hvm() to not clobber %rbx +- net: rose: lock the socket in rose_bind() +- net: atlantic: fix warning during hot unplug +- gpio: pca953x: Improve interrupt support +- rxrpc: Fix the rxrpc_connection attend queue handling +- udp: gso: do not drop small packets when PMTU reduces +- tg3: Disable tg3 PCIe AER on system reboot +- vmxnet3: Fix tx queue race condition with XDP +- ice: Add check for devm_kzalloc() +- net: bcmgenet: Correct overlaying of PHY and MAC Wake-on-LAN +- nvme-fc: use ctrl state getter +- ice: put Rx buffers after being done with current frame +- gpu: drm_dp_cec: fix broken CEC adapter properties check +- firmware: iscsi_ibft: fix ISCSI_IBFT Kconfig entry +- net/ncsi: fix locking in Get MAC Address handling +- net/ncsi: Add NC-SI 1.2 Get MC MAC Address command +- KVM: e500: always restore irqs +- KVM: PPC: e500: Use __kvm_faultin_pfn() to handle page faults +- KVM: PPC: e500: Mark "struct page" pfn accessed before dropping mmu_lock +- KVM: PPC: e500: Mark "struct page" dirty in kvmppc_e500_shadow_map() +- platform/x86: acer-wmi: Ignore AC events +- Input: allocate keycode for phone linking +- ASoC: amd: Add ACPI dependency to fix build error +- ASoC: soc-pcm: don't use soc_pcm_ret() on .prepare callback +- platform/x86: int3472: Check for adev == NULL +- iommu/arm-smmu-v3: Clean up more on probe failure +- iommu/arm-smmu-qcom: add sdm670 adreno iommu compatible +- x86/kexec: Allocate PGD for x86_64 transition page tables separately +- selftests/net/ipsec: Fix Null pointer dereference in rtattr_pack() +- tipc: re-order conditions in tipc_crypto_key_rcv() +- mmc: sdhci-msm: Correctly set the load for the regulator +- net: wwan: iosm: Fix hibernation by re-binding the driver around it +- Bluetooth: MGMT: Fix slab-use-after-free Read in mgmt_remove_adv_monitor_sync +- APEI: GHES: Have GHES honor the panic= setting +- i2c: Force ELAN06FA touchpad I2C bus freq to 100KHz +- wifi: iwlwifi: avoid memory leak +- net/mlx5: use do_aux_work for PHC overflow checks +- HID: Wacom: Add PCI Wacom device support +- clk: qcom: Make GCC_8150 depend on QCOM_GDSC +- mfd: lpc_ich: Add another Gemini Lake ISA bridge PCI device-id +- tomoyo: don't emit warning in tomoyo_write_control() +- wifi: brcmsmac: add gain range check to wlc_phy_iqcal_gainparams_nphy() +- mmc: core: Respect quirk_max_rate for non-UHS SDIO card +- tun: fix group permission check +- wifi: rtw89: add crystal_cap check to avoid setting as overflow value +- safesetid: check size of policy writes +- drm/bridge: it6505: fix HDCP CTS KSV list wait timer +- drm/bridge: it6505: fix HDCP CTS compare V matching +- drm/bridge: it6505: fix HDCP encryption when R0 ready +- drm/bridge: it6505: fix HDCP Bstatus check +- drm/bridge: it6505: Change definition MAX_HDCP_DOWN_STREAM_COUNT +- drm/amd/display: Fix Mode Cutoff in DSC Passthrough to DP2.1 Monitor +- printk: Fix signed integer overflow when defining LOG_BUF_LEN_MAX +- drm/virtio: New fence for every plane update +- x86/amd_nb: Restrict init function to AMD-based systems +- lockdep: Fix upper limit for LOCKDEP_*_BITS configs +- sched: Don't try to catch up excess steal time. +- btrfs: convert BUG_ON in btrfs_reloc_cow_block() to proper error handling +- btrfs: fix data race when accessing the inode's disk_i_size at btrfs_drop_extents() +- s390/stackleak: Use exrl instead of ex in __stackleak_poison() +- arm64/mm: Ensure adequate HUGE_MAX_HSTATE +- btrfs: fix use-after-free when attempting to join an aborted transaction +- btrfs: fix assertion failure when splitting ordered extent after transaction abort +- Revert "hostfs: convert hostfs to use the new mount API" +- Revert "hostfs: fix string handling in __dentry_name()" +- Revert "hostfs: Add const qualifier to host_root in hostfs_fill_super()" +- Revert "hostfs: fix the host directory parse when mounting." +- kernel: be more careful about dup_mmap() failures and uprobe registering +- scsi: ufs: core: Fix use-after free in init error and remove paths +- !15660 v2 add support DVMBM in HIP12 +- ovl: fix UAF in ovl_dentry_update_reval by moving dput() in ovl_link_up +- KVM: arm64: Add new HiSi CPU type to support DVMBM +- kvm: hisi_virt: Update TLBI broadcast feature for hip12 +- kvm: hisi_virt: fix kernel panic when enable DVMBM in nVHE +- uacce: fix for the numa distance calculation +- uacce: fixes the mutex repeatedly locking error +- !15593 io_uring: prevent opcode speculation +- !15644 ata: libata-sff: Ensure that we cannot write outside the allocated buffer +- nilfs2: do not force clear folio if buffer is referenced +- ata: libata-sff: Ensure that we cannot write outside the allocated buffer +- block: don't revert iter for -EIOCBQUEUED +- !15581 smb: client: Add check for next_buffer in receive_encrypted_standard() +- acct: block access to kernel internal filesystems +- acct: perform last write from workqueue +- misc: fastrpc: Fix copy buffer page size +- s390/ism: add release function for struct device +- mm/shmem: use xas_try_split() in shmem_split_large_entry() +- mm/filemap: use xas_try_split() in __filemap_add_folio() +- xarray: add xas_try_split() to split a multi-index entry +- KVM: arm64: vgic-its: Add stronger type-checking to the ITS entry sizes +- mptcp: fix possible integer overflow in mptcp_reset_tout_timer +- ext4: fix race in buffer_head read fault injection +- ext4: remove array of buffer_heads from mext_page_mkuptodate() +- ext4: pipeline buffer reads in mext_page_mkuptodate() +- Revert "ext4: Fix race in buffer_head read fault injection" +- scsi: sg: Enable runtime power management +- gfs2: Remove and replace gfs2_glock_queue_work +- gfs2: Fix unlinked inode cleanup +- gfs2: Replace gfs2_glock_queue_put with gfs2_glock_put_async +- gfs2: No longer use 'extern' in function declarations +- KVM: arm64: vgic-its: Add a data length check in vgic_its_save_* +- libfs: Fix duplicate directory entry in offset_dir_lookup +- libfs: Use d_children list to iterate simple_offset directories +- libfs: Replace simple_offset end-of-directory detection +- Revert "libfs: fix infinite directory reads for offset dir" +- Revert "libfs: Add simple_offset_empty()" +- libfs: Return ENOSPC when the directory offset range is exhausted +- SCSI: Linkdata: Supports Linkdata HBA/RAID Controllers +- mm: shmem: add large folio support for tmpfs +- mm: shmem: change shmem_huge_global_enabled() to return huge order bitmap +- mm: factor out the order calculation into a new helper +- Xarray: distinguish large entries correctly in xas_split_alloc() +- io_uring: prevent opcode speculation +- smb: client: Add check for next_buffer in receive_encrypted_standard() +- media: uvcvideo: Fix crash during unbind if gpio unit is in use +- hrtimer: Fix kabi broken of struct hrtimer_cpu_base +- hrtimers: Force migrate away hrtimers queued after CPUHP_AP_HRTIMERS_DYING +- ACPI: PRM: Remove unnecessary strict handler address checks +- ACPI: PRM: Add PRM handler direct call support +- ACPI: PRM: Annotate struct prm_module_info with __counted_by +- net: davicom: fix UAF in dm9000_drv_remove +- wifi: brcmfmac: fix NULL pointer dereference in brcmf_txfinalize() +- powerpc/pseries/iommu: IOMMU incorrectly marks MMIO range in DDW +- KVM: arm64: vgic-its: Clear ITE when DISCARD frees an ITE +- KVM: arm64: vgic-its: Clear DTE when MAPD unmaps a device +- partitions: mac: fix handling of bogus partition table +- printk: Fix signed integer overflow when defining LOG_BUF_LEN_MAX +- dmaengine: idxd: Add a new DSA device ID for Granite Rapids-D platform +- drm/amdgpu: bail out when failed to load fw in psp_init_cap_microcode() +- EDAC/mce_amd: Add LS and IF mce types for Hygon family 18h model 7h +- EDAC/amd64: Add support for Hygon family 18h model 8h +- x86/amd_nb: Add support for Hygon family 18h model 8h +- EDAC/amd64: Refactor the calculation of instance id for Hygon family 18h model 10h +- EDAC/amd64: Calculate instance id for hygon family 18h model 7h +- EDAC/amd64: Get correct memory type for Hygon family 18h model 10h +- EDAC/amd64: Check if umc channel is enabled for Hygon family 18h model 10h +- x86/amd_nb: Add helper function to identify Hygon family 18h model 10h +- iommu/hygon: Add support for Hygon family 18h model 10h IOAPIC +- EDAC/amd64: Fix the calculation of instance id for Hygon family 18h model 6h +- irqchip/gic-v3-its: Init reserved rsv_devid_pools use pci bus info +- irqchip/gic-v3-its: Move build_devid_pools from its to acpi iort init +- irqchip/gic-v3-its: Add ACPI_IORT as VIRT_PLAT_DEV's dependency +- acpi/iort: Add func to get used deviceid bitmap +- RNP: NET: Fix rnp errors dmesg show in hygon +- net: mucse: Fix os crash when cat some sys interface when net down +- mucse: rnpm ocp nic support hotplug and show sub-zero temperature +- mucse: rnpm compiler error on loongarch64 +- RNPVF: NET: Fix rnpvf compile errors without CONFIG_MXGBEVF_FIX_VF_QUEUE + +* Tue Mar 25 2025 ZhangPeng - 6.6.0-84.0.0.89 +- !15625 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.75-6.6.76 LTS Patches +- HID: hid-thrustmaster: fix stack-out-of-bounds read in usb_check_int_endpoints() +- Revert "drivers/card_reader/rtsx_usb: Restore interrupt based detection" +- selftests/landlock: Fix error message +- LoongArch: Change 8 to 14 for LOONGARCH_MAX_{BRP,WRP} +- s390: Add '-std=gnu11' to decompressor and purgatory CFLAGS +- btrfs: output the reason for open_ctree() failure +- media: imx-jpeg: Fix potential error pointer dereference in detach_pm() +- staging: media: max96712: fix kernel oops when removing module +- usb: gadget: f_tcm: Don't free command immediately +- media: uvcvideo: Fix double free in error path +- remoteproc: core: Fix ida_free call while not allocated +- mptcp: handle fastopen disconnect correctly +- mptcp: consolidate suboption status +- usb: typec: tcpci: Prevent Sink disconnection before vPpsShutdown in SPR PPS +- usb: typec: tcpm: set SRC_SEND_CAPABILITIES timeout to PD_T_SENDER_RESPONSE +- usb: dwc3: core: Defer the probe until USB power supply ready +- usb: dwc3-am62: Fix an OF node leak in phy_syscon_pll_refclk() +- usb: gadget: f_tcm: Fix Get/SetInterface return value +- drivers/card_reader/rtsx_usb: Restore interrupt based detection +- net: usb: rtl8150: enable basic endpoint checking +- ALSA: usb-audio: Add delay quirk for iBasso DC07 Pro +- ktest.pl: Check kernelrelease return in get_version +- selftests/rseq: Fix handling of glibc without rseq support +- netfilter: nf_tables: reject mismatching sum of field_len with set key length +- kbuild: switch from lz4c to lz4 for compression +- Revert "SUNRPC: Reduce thread wake-up rate when receiving large RPC messages" +- f2fs: Introduce linear search for dentries +- hexagon: Fix unbalanced spinlock in die() +- hexagon: fix using plain integer as NULL pointer warning in cmpxchg +- kconfig: fix memory leak in sym_warn_unmet_dep() +- kconfig: WERROR unmet symbol dependency +- kconfig: deduplicate code in conf_read_simple() +- kconfig: remove unused code for S_DEF_AUTO in conf_read_simple() +- kconfig: require a space after '#' for valid input +- kconfig: fix file name in warnings when loading KCONFIG_DEFCONFIG_LIST +- cifs: Fix getting and setting SACLs over SMB1 +- cifs: Validate EAs for WSL reparse points +- io_uring/uring_cmd: use cached cmd_op in io_uring_cmd_sock() +- ASoC: rockchip: i2s_tdm: Re-add the set_sysclk callback +- RISC-V: Mark riscv_v_init() as __init +- hostfs: fix the host directory parse when mounting. +- hostfs: Add const qualifier to host_root in hostfs_fill_super() +- hostfs: fix string handling in __dentry_name() +- hostfs: convert hostfs to use the new mount API +- genksyms: fix memory leak when the same symbol is read from *.symref file +- genksyms: fix memory leak when the same symbol is added from source +- net: hsr: fix fill_frame_info() regression vs VLAN packets +- net: sh_eth: Fix missing rtnl lock in suspend/resume path +- net: xdp: Disallow attaching device-bound programs in generic mode +- tcp: correct handling of extreme memory squeeze +- bgmac: reduce max frame size to support just MTU 1500 +- vsock: Allow retrying on connect() failure +- Bluetooth: btnxpuart: Fix glitches seen in dual A2DP streaming +- perf trace: Fix runtime error of index out of bounds +- net: stmmac: Limit FIFO size by hardware capability +- net: stmmac: Limit the number of MTL queues to hardware capability +- ptp: Properly handle compat ioctls +- net: davicom: fix UAF in dm9000_drv_remove +- vxlan: Fix uninit-value in vxlan_vnifilter_dump() +- net: netdevsim: try to close UDP port harness races +- net: rose: fix timer races against user threads +- iavf: allow changing VLAN state without calling PF +- PM: hibernate: Add error handling for syscore_suspend() +- net: fec: implement TSO descriptor cleanup +- gpio: mxc: remove dead code after switch to DT-only +- regulator: core: Add missing newline character +- ubifs: skip dumping tnc tree when zroot is null +- xfrm: delete intermediate secpath entry in packet offload mode +- dmaengine: ti: edma: fix OF node reference leaks in edma_driver +- xfrm: replay: Fix the update of replay_esn->oseq_hi for GSO +- tools/bootconfig: Fix the wrong format specifier +- NFSv4.2: mark OFFLOAD_CANCEL MOVEABLE +- NFSv4.2: fix COPY_NOTIFY xdr buf size calculation +- serial: 8250: Adjust the timeout for FIFO mode +- driver core: class: Fix wild pointer dereferences in API class_dev_iter_next() +- module: Extend the preempt disabled section in dereference_symbol_descriptor(). +- nilfs2: protect access to buffers with no active references +- nilfs2: convert nilfs_lookup_dirty_data_buffers to use folio_create_empty_buffers +- buffer: make folio_create_empty_buffers() return a buffer_head +- ocfs2: mark dquot as inactive if failed to start trans while releasing dquot +- scsi: ufs: bsg: Delete bsg_dev when setting up bsg fails +- scsi: mpt3sas: Set ioc->manu_pg11.EEDPTagMode directly to 1 +- PCI: endpoint: pci-epf-test: Fix check for DMA MEMCPY test +- PCI: endpoint: pci-epf-test: Set dma_chan_rx pointer to NULL on error +- PCI: imx6: Skip controller_id generation logic for i.MX7D +- PCI: imx6: Simplify clock handling by using clk_bulk*() function +- media: dvb-usb-v2: af9035: fix ISO C90 compilation error on af9035_i2c_master_xfer +- staging: media: imx: fix OF node leak in imx_media_add_of_subdevs() +- watchdog: rti_wdt: Fix an OF node leak in rti_wdt_probe() +- media: nxp: imx8-isi: fix v4l2-compliance test errors +- mtd: hyperbus: hbmc-am654: fix an OF node reference leak +- mtd: hyperbus: hbmc-am654: Convert to platform remove callback returning void +- mtd: rawnand: brcmnand: fix status read of brcmnand_waitfunc +- media: uvcvideo: Propagate buf->error to userspace +- media: camif-core: Add check for clk_enable() +- media: mipi-csis: Add check for clk_enable() +- media: i2c: ov9282: Correct the exposure offset +- media: i2c: imx412: Add missing newline to prints +- media: i2c: imx290: Register 0x3011 varies between imx327 and imx290 +- media: marvell: Add check for clk_enable() +- PCI: endpoint: Destroy the EPC device in devm_pci_epc_destroy() +- media: lmedm04: Handle errors for lme2510_int_read +- media: rc: iguanair: handle timeouts +- spi: omap2-mcspi: Correctly handle devm_clk_get_optional() errors +- RDMA/rxe: Fix the warning "__rxe_cleanup+0x12c/0x170 [rdma_rxe]" +- efi: sysfb_efi: fix W=1 warnings when EFI is not set +- of: reserved-memory: Do not make kmemleak ignore freed address +- RDMA/mlx5: Fix indirect mkey ODP page count +- i3c: dw: Fix use-after-free in dw_i3c_master driver due to race condition +- i3c: dw: Add hot-join support. +- arm64: tegra: Fix DMA ID for SPI2 +- fbdev: omapfb: Fix an OF node leak in dss_of_port_get_parent_device() +- ARM: dts: mediatek: mt7623: fix IR nodename +- arm64: dts: qcom: sm8250: Fix interrupt types of camss interrupts +- arm64: dts: qcom: sdm845: Fix interrupt types of camss interrupts +- arm64: dts: mediatek: add per-SoC compatibles for keypad nodes +- dts: arm64: mediatek: mt8195: Remove MT8183 compatible for OVL +- arm64: dts: qcom: sc8280xp: Fix up remoteproc register space sizes +- arm64: dts: qcom: sm8150-microsoft-surface-duo: fix typos in da7280 properties +- arm64: dts: qcom: sc7180: fix psci power domain node names +- arm64: dts: qcom: sc7180: change labels to lower-case +- arm64: dts: qcom: Add SM7125 device tree +- arm64: dts: qcom: sc7180-trogdor-pompom: rename 5v-choke thermal zone +- arm64: dts: qcom: sc7180-*: Remove thermal zone polling delays +- arm64: dts: qcom: sc7180-trogdor-quackingstick: add missing avee-supply +- arm64: dts: qcom: sdm845-db845c-navigation-mezzanine: remove disabled ov7251 camera +- arm64: dts: qcom: sdm845-db845c-navigation-mezzanine: Convert mezzanine riser to dtso +- ARM: omap1: Fix up the Retu IRQ on Nokia 770 +- RDMA/bnxt_re: Fix to drop reference to the mmap entry in case of error +- arm64: dts: allwinner: a64: explicitly assign clock parent for TCON0 +- arm64: dts: ti: k3-am62a: Remove duplicate GICR reg +- arm64: dts: ti: k3-am62: Remove duplicate GICR reg +- ARM: dts: microchip: sama5d27_wlsom1_ek: Add no-1-8-v property to sdmmc0 node +- ARM: dts: microchip: sama5d27_wlsom1_ek: Remove mmc-ddr-3_3v property from sdmmc0 node +- arm64: dts: qcom: sm8450: correct sleep clock frequency +- arm64: dts: qcom: sm8350: correct sleep clock frequency +- arm64: dts: qcom: sm8250: correct sleep clock frequency +- arm64: dts: qcom: sm6375: correct sleep clock frequency +- arm64: dts: qcom: sm6125: correct sleep clock frequency +- arm64: dts: qcom: sm4450: correct sleep clock frequency +- arm64: dts: qcom: sdx75: correct sleep clock frequency +- arm64: dts: qcom: sc7280: correct sleep clock frequency +- arm64: dts: qcom: qrb4210-rb2: correct sleep clock frequency +- arm64: dts: qcom: q[dr]u1000: correct sleep clock frequency +- arm64: dts: qcom: qcs404: correct sleep clock frequency +- arm64: dts: qcom: msm8994: correct sleep clock frequency +- arm64: dts: qcom: msm8939: correct sleep clock frequency +- arm64: dts: qcom: msm8916: correct sleep clock frequency +- arm64: dts: qcom: sm7225-fairphone-fp4: Drop extra qcom,msm-id value +- arm64: dts: qcom: msm8994: Describe USB interrupts +- arm64: dts: qcom: msm8996: Fix up USB3 interrupts +- arm64: dts: qcom: sa8775p: Update sleep_clk frequency +- arm64: dts: qcom: move common parts for sa8775p-ride variants into a .dtsi +- arm64: dts: qcom: sa8775p-ride: enable pmm8654au_0_pon_resin +- arm64: dts: qcom: sa8775p-ride: Describe sgmii_phy1 irq +- arm64: dts: qcom: sa8775p-ride: Describe sgmii_phy0 irq +- arm64: dts: qcom: msm8996-xiaomi-gemini: Fix LP5562 LED1 reg property +- arm64: dts: mediatek: mt8183-kukui-jacuzzi: Drop pp3300_panel voltage settings +- memory: tegra20-emc: fix an OF node reference bug in tegra_emc_find_node_by_ram_code() +- ARM: dts: stm32: Swap USART3 and UART8 alias on STM32MP15xx DHCOM SoM +- ARM: dts: stm32: Deduplicate serial aliases and chosen node for STM32MP15xx DHCOM SoM +- arm64: dts: mediatek: mt8195: Remove suspend-breaking reset from pcie1 +- RDMA/srp: Fix error handling in srp_add_port +- arm64: dts: mediatek: mt8183: willow: Support second source touchscreen +- arm64: dts: mediatek: mt8183: kenzo: Support second source touchscreen +- RDMA/rxe: Fix mismatched max_msg_sz +- RDMA/rxe: Improve newline in printing messages +- arm: dts: socfpga: use reset-name "stmmaceth-ocp" instead of "ahb" +- ARM: dts: aspeed: yosemite4: correct the compatible string for max31790 +- ARM: dts: aspeed: yosemite4: Add required properties for IOE on fan boards +- ARM: dts: aspeed: yosemite4: correct the compatible string of adm1272 +- arm64: dts: mediatek: mt8173-evb: Fix MT6397 PMIC sub-node names +- arm64: dts: mediatek: mt8173-elm: Fix MT6397 PMIC sub-node names +- arm64: dts: mediatek: mt8195-demo: Drop regulator-compatible property +- arm64: dts: mediatek: mt8195-cherry: Drop regulator-compatible property +- arm64: dts: mediatek: mt8192-asurada: Drop regulator-compatible property +- arm64: dts: mediatek: mt8173-elm: Drop regulator-compatible property +- arm64: dts: mediatek: mt8173-evb: Drop regulator-compatible property +- rdma/cxgb4: Prevent potential integer overflow on 32bit +- RDMA/mlx4: Avoid false error about access to uninitialized gids array +- ARM: dts: stm32: Fix IPCC EXTI declaration on stm32mp151 +- arm64: dts: mediatek: mt8516: reserve 192 KiB for TF-A +- arm64: dts: mediatek: mt8516: add i2c clock-div property +- arm64: dts: mediatek: mt8516: fix wdt irq type +- arm64: dts: mediatek: mt8516: fix GICv2 range +- arm64: dts: mt8183: set DMIC one-wire mode on Damu +- arm64: dts: mediatek: mt8186: Move wakeup to MTU3 to get working suspend +- ARM: at91: pm: change BU Power Switch to automatic mode +- soc: atmel: fix device_node release in atmel_soc_device_init() +- cifs: Use cifs_autodisable_serverino() for disabling CIFS_MOUNT_SERVER_INUM in readdir.c +- smb: client: fix oops due to unset link speed +- padata: avoid UAF for reorder_work +- padata: add pd get/put refcnt helper +- padata: fix UAF in padata_reorder +- perf lock: Fix parse_lock_type which only retrieve one lock flag +- ALSA: hda/realtek - Fixed headphone distorted sound on Acer Aspire A115-31 laptop +- bpf: tcp: Mark bpf_load_hdr_opt() arg2 as read-write +- pinctrl: amd: Take suspend type into consideration which pins are non-wake +- pinctrl: stm32: Add check for clk_enable() +- perf report: Fix misleading help message about --demangle +- ASoC: Intel: avs: Fix theoretical infinite loop +- ASoC: Intel: avs: Do not readq() u32 registers +- ASoC: Intel: avs: Abstract IPC handling +- ASoC: Intel: avs: Prefix SKL/APL-specific members +- perf namespaces: Fixup the nsinfo__in_pidns() return type, its bool +- perf namespaces: Introduce nsinfo__set_in_pidns() +- perf machine: Don't ignore _etext when not a text symbol +- perf top: Don't complain about lack of vmlinux when not resolving some kernel samples +- padata: fix sysfs store callback check +- ALSA: seq: Make dependency on UMP clearer +- ALSA: seq: remove redundant 'tristate' for SND_SEQ_UMP_CLIENT +- crypto: ixp4xx - fix OF node reference leaks in init_ixp_crypto() +- ktest.pl: Remove unused declarations in run_bisect_test function +- perf expr: Initialize is_test value in expr__ctx_new() +- ASoC: renesas: rz-ssi: Use only the proper amount of dividers +- perf bpf: Fix two memory leakages when calling perf_env__insert_bpf_prog_info() +- perf header: Fix one memory leakage in process_bpf_prog_info() +- perf header: Fix one memory leakage in process_bpf_btf() +- crypto: caam - use JobR's space to access page 0 regs +- selftests/bpf: Fix fill_link_info selftest on powerpc +- ASoC: sun4i-spdif: Add clock multiplier settings +- tools/testing/selftests/bpf/test_tc_tunnel.sh: Fix wait for server bind +- libbpf: don't adjust USDT semaphore address if .stapsdt.base addr is missing +- net/rose: prevent integer overflows in rose_setsockopt() +- tcp_cubic: fix incorrect HyStart round start detection +- net: ethernet: ti: am65-cpsw: fix freeing IRQ in am65_cpsw_nuss_remove_tx_chns() +- netfilter: nft_flow_offload: update tcp state flags under lock +- netfilter: nf_tables: fix set size with rbtree backend +- netfilter: nft_set_rbtree: prefer sync gc to async worker +- netfilter: nft_set_rbtree: rename gc deactivate+erase function +- netfilter: nf_tables: de-constify set commit ops function argument +- net: sched: Disallow replacing of child qdisc from one parent to another +- net: avoid race between device unregistration and ethnl ops +- octeon_ep: remove firmware stats fetch in ndo_get_stats64 +- net/mlxfw: Drop hard coded max FW flash image size +- net: let net.core.dev_weight always be non-zero +- pwm: stm32: Add check for clk_enable() +- clk: analogbits: Fix incorrect calculation of vco rate delta +- inet: ipmr: fix data-races +- wifi: cfg80211: adjust allocation of colocated AP data +- wifi: cfg80211: Handle specific BSSID in 6GHz scanning +- selftests: harness: fix printing of mismatch values in __EXPECT() +- selftests: timers: clocksource-switch: Adapt progress to kselftest framework +- wifi: mt76: mt7996: fix ldpc setting +- wifi: mt76: mt7996: fix incorrect indexing of MIB FW event +- wifi: mt76: mt7996: fix HE Phy capability +- wifi: mt76: mt7996: fix the capability of reception of EHT MU PPDU +- wifi: mt76: mt7996: add max mpdu len capability +- wifi: mt76: mt7996: fix register mapping +- wifi: mt76: mt7915: fix register mapping +- wifi: mt76: mt7915: fix omac index assignment after hardware reset +- wifi: mt76: mt7915: improve hardware restart reliability +- wifi: mt76: connac: move mt7615_mcu_del_wtbl_all to connac +- wifi: mt76: mt7915: firmware restart on devices with a second pcie link +- wifi: mt76: mt7996: fix rx filter setting for bfee functionality +- wifi: mt76: mt7915: fix overflows seen when writing limit attributes +- wifi: mt76: mt7921: fix using incorrect group cipher after disconnection. +- wifi: mt76: mt76u_vendor_request: Do not print error messages when -EPROTO +- landlock: Handle weird files +- net/smc: fix data error when recvmsg with MSG_PEEK flag +- clk: ralink: mtmips: remove duplicated 'xtal' clock for Ralink SoC RT3883 +- wifi: mac80211: don't flush non-uploaded STAs +- wifi: mac80211: Fix common size calculation for ML element +- wifi: mac80211: fix tid removal during mesh forwarding +- wifi: mac80211: prohibit deactivating all links +- wifi: mt76: mt7915: Fix mesh scan on MT7916 DBDC +- wifi: wlcore: fix unbalanced pm_runtime calls +- samples/landlock: Fix possible NULL dereference in parse_path() +- mfd: syscon: Fix race in device_node_get_regmap() +- mfd: syscon: Use scoped variables with memory allocators to simplify error paths +- mfd: syscon: Add of_syscon_register_regmap() API +- mfd: syscon: Remove extern from function prototypes +- leds: cht-wcove: Use devm_led_classdev_register() to avoid memory leak +- HID: fix generic desktop D-Pad controls +- HID: hid-thrustmaster: Fix warning in thrustmaster_probe by adding endpoint check +- clk: qcom: gcc-sdm845: Do not use shared clk_ops for QUPs +- wifi: ath12k: fix tx power, max reg power update to firmware +- ipmi: ssif_bmc: Fix new request loss when bmc ready for a response +- OPP: OF: Fix an OF node leak in _opp_add_static_v2() +- ax25: rcu protect dev->ax25_ptr +- regulator: of: Implement the unwind path of of_regulator_match() +- clk: sunxi-ng: a64: stop force-selecting PLL-MIPI as TCON0 parent +- clk: sunxi-ng: a64: drop redundant CLK_PLL_VIDEO0_2X and CLK_PLL_MIPI +- dt-bindings: clock: sunxi: Export PLL_VIDEO_2X and PLL_MIPI +- team: prevent adding a device which is already a team device lower +- clk: imx8mp: Fix clkout1/2 support +- cpufreq: qcom: Implement clk_ops::determine_rate() for qcom_cpufreq* clocks +- cpufreq: qcom: Fix qcom_cpufreq_hw_recalc_rate() to query LUT if LMh IRQ is not available +- gpio: pca953x: log an error when failing to get the reset GPIO +- gpio: pca953x: Fully convert to device managed resources +- gpio: pca953x: Drop unused fields in struct pca953x_platform_data +- cpufreq: schedutil: Fix superfluous updates caused by need_freq_update +- pwm: stm32-lp: Add check for clk_enable() +- inetpeer: do not get a refcount in inet_getpeer() +- inetpeer: update inetpeer timestamp in inet_getpeer() +- inetpeer: remove create argument of inet_getpeer() +- inetpeer: remove create argument of inet_getpeer_v[46]() +- leds: netxbig: Fix an OF node reference leak in netxbig_leds_get_of_pdata() +- dt-bindings: mfd: bd71815: Fix rsense and typos +- cpupower: fix TSC MHz calculation +- ACPI: fan: cleanup resources in the error path of .probe() +- wifi: brcmfmac: add missing header include for brcmf_dbg +- regulator: dt-bindings: mt6315: Drop regulator-compatible property +- HID: multitouch: fix support for Goodix PID 0x01e9 +- wifi: rtlwifi: pci: wait for firmware loading before releasing memory +- wifi: rtlwifi: fix memory leaks and invalid access at probe error path +- wifi: rtlwifi: destroy workqueue at rtl_deinit_core +- wifi: rtlwifi: remove unused check_buddy_priv +- dt-bindings: leds: class-multicolor: Fix path to color definitions +- clk: fix an OF node reference leak in of_clk_get_parent_name() +- of: remove internal arguments from of_property_for_each_u32() +- clk: si5351: allow PLLs to be adjusted without reset +- serial: sc16is7xx: use device_property APIs when configuring irda mode +- dt-bindings: mmc: controller: clarify the address-cells description +- rxrpc: Fix handling of received connection abort +- spi: zynq-qspi: Add check for clk_enable() +- net_sched: sch_sfq: don't allow 1 packet limit +- net_sched: sch_sfq: handle bigger packets +- net_sched: sch_sfq: annotate data-races around q->perturb_period +- wifi: rtlwifi: usb: fix workqueue leak when probe fails +- wifi: rtlwifi: fix init_sw_vars leak when probe fails +- wifi: rtlwifi: wait for firmware loading before releasing memory +- wifi: rtlwifi: rtl8192se: rise completion of firmware loading as last step +- wifi: rtlwifi: do not complete firmware loading needlessly +- wifi: ath11k: Fix unexpected return buffer manager error for WCN6750/WCN6855 +- ipmi: ipmb: Add check devm_kasprintf() returned value +- drm/amdgpu: tear down ttm range manager for doorbell in amdgpu_ttm_fini() +- drm/bridge: it6505: Change definition of AUX_FIFO_MAX_SIZE +- drm/msm: Check return value of of_dma_configure() +- drm/msm/dpu: link DSPP_2/_3 blocks on SM8550 +- drm/msm/dpu: link DSPP_2/_3 blocks on SM8350 +- drm/msm/dpu: link DSPP_2/_3 blocks on SM8250 +- drm/msm/dpu: link DSPP_2/_3 blocks on SC8180X +- drm/msm/dpu: link DSPP_2/_3 blocks on SM8150 +- OPP: fix dev_pm_opp_find_bw_*() when bandwidth table not initialized +- drm/amdgpu/vcn: reset fw_shared under SRIOV +- drm/rockchip: vop2: include rockchip_drm_drv.h +- drm/rockchip: move output interface related definition to rockchip_drm_drv.h +- drm/rockchip: vop2: Check linear format for Cluster windows on rk3566/8 +- drm/rockchip: vop2: Fix the windows switch between different layers +- drm/rockchip: vop2: set bg dly and prescan dly at vop2_post_config +- drm/rockchip: vop2: Set YUV/RGB overlay mode +- drm/rockchip: vop2: Fix the mixer alpha setup for layer 0 +- drm/rockchip: vop2: Fix cluster windows alpha ctrl regsiters offset +- drm/amdgpu: Fix potential NULL pointer dereference in atomctrl_get_smc_sclk_range_table +- drm/amd/pm: Fix an error handling path in vega10_enable_se_edc_force_stall_config() +- HID: core: Fix assumption that Resolution Multipliers must be in Logical Collections +- drm/etnaviv: Fix page property being used for non writecombine buffers +- drm/msm/dp: set safe_to_exit_level before printing it +- x86/topology: Use x86_sched_itmt_flags for PKG domain unconditionally +- x86/cpu: Enable SD_ASYM_PACKING for PKG domain on AMD +- sched/topology: Rename 'DIE' domain to 'PKG' +- afs: Fix the fallback handling for the YFS.RemoveFile2 RPC call +- select: Fix unbalanced user_access_end() +- partitions: ldm: remove the initial kernel-doc notation +- selftests/powerpc: Fix argument order to timer_sub() +- pstore/blk: trivial typo fixes +- block: retry call probe after request_module in blk_request_module +- fs: fix proc_handler for sysctl_nr_open +- afs: Fix cleanup of immediately failed async calls +- afs: Fix directory format encoding struct +- afs: Fix EEXIST error returned from afs_rmdir() to be ENOTEMPTY +- dlm: fix srcu_read_lock() return type to int +- powerpc/book3s64/hugetlb: Fix disabling hugetlb when fadump is active +- !15477 PCI: rcar-ep: Fix incorrect variable used when calling devm_request_mem_region() +- PCI: rcar-ep: Fix incorrect variable used when calling devm_request_mem_region() +- !15472 PCI: dwc: ep: Prevent changing BAR size/flags in pci_epc_set_bar() +- PCI: dwc: ep: Prevent changing BAR size/flags in pci_epc_set_bar() +- !15473 iommufd/iova_bitmap: Fix shift-out-of-bounds in iova_bitmap_offset_to_index() +- iommufd/iova_bitmap: Fix shift-out-of-bounds in iova_bitmap_offset_to_index() +- !15474 OPP: add index check to assert to avoid buffer overflow in _read_freq() +- OPP: add index check to assert to avoid buffer overflow in _read_freq() +- !15475 wifi: wcn36xx: fix channel survey memory allocation size +- wifi: wcn36xx: fix channel survey memory allocation size +- !15476 LoongArch: Fix warnings during S3 suspend +- LoongArch: Fix warnings during S3 suspend +- !15464 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.72-6.6.75 LTS Patches +- drm/v3d: Assign job pointer to NULL before signaling the fence +- Input: xpad - add support for wooting two he (arm) +- Input: xpad - add support for Nacon Evol-X Xbox One Controller +- Input: xpad - improve name of 8BitDo controller 2dc8:3106 +- Input: xpad - add QH Electronics VID/PID +- Input: xpad - add unofficial Xbox 360 wireless receiver clone +- Input: atkbd - map F23 key to support default copilot shortcut +- Input: xpad - add support for Nacon Pro Compact +- ALSA: usb-audio: Add delay quirk for USB Audio Device +- Revert "usb: gadget: u_serial: Disable ep before setting port to null to fix the crash caused by port being null" +- scsi: storvsc: Ratelimit warning logs to prevent VM denial of service +- vfio/platform: check the bounds of read/write syscalls +- cachestat: fix page cache statistics permission checking +- Revert "HID: multitouch: Add support for lenovo Y9000P Touchpad" +- net: sched: fix ets qdisc OOB Indexing +- smb: client: handle lack of EA support in smb2_query_path_info() +- libfs: Add simple_offset_rename() API +- ASoC: samsung: Add missing depends on I2C +- hwmon: (drivetemp) Set scsi command timeout to 10s +- irqchip/sunxi-nmi: Add missing SKIP_WAKE flag +- of/unittest: Add test that of_address_to_resource() fails on non-translatable address +- drm/amd/display: Use HW lock mgr for PSR1 +- scsi: iscsi: Fix redundant response for ISCSI_UEVENT_GET_HOST_STATS request +- seccomp: Stub for !CONFIG_SECCOMP +- ASoC: samsung: Add missing selects for MFD_WM8994 +- ASoC: wm8994: Add depends on MFD core +- x86/xen: fix SLS mitigation in xen_hypercall_iret() +- nfsd: add list_head nf_gc to struct nfsd_file +- Revert "drm/amdgpu: rework resume handling for display (v2)" +- fs: relax assertions on failure to encode file handles +- ovl: support encoding fid from inode with no alias +- ovl: pass realinode to ovl_encode_real_fh() instead of realdentry +- ocfs2: fix deadlock in ocfs2_get_system_file_inode +- iio: imu: inv_icm42600: fix spi burst write not supported +- Revert "PCI: Use preserve_config in place of pci_flags" +- drm/amdgpu: always sync the GFX pipe on ctx switch +- drm/i915/fb: Relax clear color alignment to 64 bytes +- hrtimers: Handle CPU state correctly on hotplug +- irqchip: Plug a OF node reference leak in platform_irqchip_probe() +- filemap: avoid truncating 64-bit offset to 32 bits +- nouveau/fence: handle cross device fences properly +- vsock: prevent null-ptr-deref in vsock_*[has_data|has_space] +- vsock: reset socket state when de-assigning the transport +- vsock/virtio: cancel close work in the destructor +- vsock/virtio: discard packets if the transport changes +- net: ethernet: xgbe: re-add aneg to supported features in PHY quirks +- selftests: mptcp: avoid spurious errors on disconnect +- mptcp: fix spurious wake-up on under memory pressure +- mptcp: be sure to send ack when mptcp-level window re-opens +- i2c: atr: Fix client detach +- zram: fix potential UAF of zram table +- ALSA: hda/realtek: Add support for Ayaneo System using CS35L41 HDA +- x86/asm: Make serialize() always_inline +- nvmet: propagate npwg topology +- RDMA/bnxt_re: Fix to export port num to ib_query_qp +- poll_wait: add mb() to fix theoretical race between waitqueue_active() and .poll() +- ACPI: resource: acpi_dev_irq_override(): Check DMI match last +- smb: client: fix double free of TCP_Server_Info::hostname +- i2c: rcar: fix NACK handling when being a target +- i2c: mux: demux-pinctrl: check initial mux selection, too +- Revert "mtd: spi-nor: core: replace dummy buswidth from addr to data" +- hwmon: (tmp513) Fix division of negative numbers +- soc: ti: pruss: Fix pruss APIs +- drm/v3d: Ensure job pointer is set to NULL after job completion +- drm/vmwgfx: Add new keep_resv BO param +- net/mlx5e: Always start IPsec sequence number from 1 +- net/mlx5e: Rely on reqid in IPsec tunnel mode +- net/mlx5e: Fix inversion dependency warning while enabling IPsec tunnel +- net/mlx5: Clear port select structure when fail to create +- net/mlx5: Fix RDMA TX steering prio +- net: fec: handle page_pool_dev_alloc_pages error +- net: xilinx: axienet: Fix IRQ coalescing packet count overflow +- nfp: bpf: prevent integer overflow in nfp_bpf_event_output() +- pktgen: Avoid out-of-bounds access in get_imix_entries +- openvswitch: fix lockup on tx to unregistering netdev with carrier +- bpf: Fix bpf_sk_select_reuseport() memory leak +- net: ethernet: ti: cpsw_ale: Fix cpsw_ale_get_field() +- Revert "ovl: do not encode lower fh with upper sb_writers held" +- Revert "ovl: pass realinode to ovl_encode_real_fh() instead of realdentry" +- Revert "ovl: support encoding fid from inode with no alias" +- !15608 perf/core: Order the PMU list to fix warning about unordered pmu_ctx_list +- perf/core: Order the PMU list to fix warning about unordered pmu_ctx_list +- !15360 USB: hub: Ignore non-compliant devices with too many configs or interfaces +- USB: hub: Ignore non-compliant devices with too many configs or interfaces +- !15597 CVE-2025-21853 +- bpf: avoid holding freeze_mutex during mmap operation +- bpf: unify VM_WRITE vs VM_MAYWRITE use in BPF map mmaping logic +- !15603 Mainline patch merged into OLK-6.6 +- RDMA/hns: Fix missing xa_destroy() +- RDMA/hns: Fix wrong value of max_sge_rd +- RDMA/hns: Fix a missing rollback in error path of hns_roce_create_qp_common() +- RDMA/hns: Fix invalid sq params not being blocked +- RDMA/hns: Fix unmatched condition in error path of alloc_user_qp_db() +- RDMA/hns: Fix soft lockup during bt pages loop +- !15430 cpufreq: Fix some boost errors related to CPU online and offline +- cpufreq: prevent NULL dereference in cpufreq_online() +- cpufreq: ACPI: Remove set_boost in acpi_cpufreq_cpu_init() +- cpufreq: CPPC: Fix wrong max_freq in policy initialization +- cpufreq: Introduce a more generic way to set default per-policy boost flag +- cpufreq: Fix re-boost issue after hotplugging a CPU +- !15568 [olk 6.6] some features for hibmcge driver +- net: hibmcge: Add support for ioctl +- net: hibmcge: Add support for BMC diagnose feature +- net: hibmcge: Add support for mac link exception handling feature +- net: hibmcge: Add support for abnormal irq handling feature +- net: hibmcge: Add support for checksum offload +- net: hibmcge: Add support for dump statistics +- ethtool: Implement ethtool_puts() +- !15357 blk-cgroup: Fix class @block_class's subsystem refcount leakage +- blk-cgroup: Fix class @block_class's subsystem refcount leakage +- !15333 VirtCCA Feature: cvm support UEFI boot +- Bug fix: Resolve the issue of host hard lockup +- VirtCCA: The cvm removes the vcpu binding limit. +- VirtCCA: Improve nfs workqueue bind method +- VirtCCA: Intercept the situation where devices with both NVM adn CVM passthrough under same PCIe root port +- VirtCCA: an interface to retrieve the TMI version is added +- VirtCCA: cvm support UEFI boot +- !15414 bnxt_en: Fix possible memory leak when hwrm_req_replace fails +- bnxt_en: Fix possible memory leak when hwrm_req_replace fails +- !15580 two bugfixs +- mm/migrate: fix shmem xarray update during migration +- mm: fix finish_fault() handling for large folios + +* Tue Mar 18 2025 ZhangPeng - 6.6.0-83.0.0.88 +- !15047 mm: Backport some mm patchs +- mm/memmap: prevent double scanning of memmap by kmemleak +- mm: kmemleak: fix upper boundary check for physical address objects +- mm/vmscan: fix hard LOCKUP in function isolate_lru_folios +- mm: compaction: use the proper flag to determine watermarks +- !15271 LoongArch: fix cacheinfo/unaligned/rtc/s3/hda/usb issues +- blutetooth/btusb: delay 1ms while suspending +- LoongArch: Correct the cacheinfo sharing information +- ice: fix unaligned access in ice_create_lag_recipe +- LoongArch: set CONFIG_CMA_SIZE_MBYTES to 0 +- rtc: loongson: clear TOY_MATCH0_REG in loongson_rtc_isr() +- hda/pci: Add AZX_DCAPS_NO_TCSEL flag for Loongson HDA devices +- LoongArch: Update the flush cache policy +- !15516 mm: shmem: remove unnecessary warning in shmem_writepage() +- mm: shmem: remove unnecessary warning in shmem_writepage() +- !15492 Add dp module in hibmc driver +- drm/hisilicon/hibmc: add dp module in hibmc +- drm/hisilicon/hibmc: refactored struct hibmc_drm_private +- drm/hisilicon/hibmc: add dp hw moduel in hibmc driver +- drm/hisilicon/hibmc: add dp link moduel in hibmc drivers +- drm/hisilicon/hibmc: add dp aux in hibmc drivers +- drm/hisilicon/hibmc: convert to struct drm_edid +- !15395 clk: qcom: dispcc-sm6350: Add missing parent_map for a clock +- clk: qcom: dispcc-sm6350: Add missing parent_map for a clock +- !15459 perf: Configure BRBE correctly on VHE host +- perf: Configure BRBE correctly on VHE host +- !15369 media: vidtv: Fix a null-ptr-deref in vidtv_mux_stop_thread +- media: vidtv: Fix a null-ptr-deref in vidtv_mux_stop_thread +- !15421 [OLK-6.6] [Backport] upstream fix for driver nvme +- nvme: handle connectivity loss in nvme_set_queue_count +- nvme: fix bogus kzalloc() return check in nvme_init_effects_log() +- nvme: Add error path for xa_store in nvme_init_effects +- nvme: Add error check for xa_store in nvme_get_effects_log +- !15370 wifi: brcmfmac: Check the return value of of_property_read_string_index() +- wifi: brcmfmac: Check the return value of of_property_read_string_index() +- !15483 CVE-2024-58069 +- rtc: pcf85063: fix potential OOB write in PCF85063 NVMEM read +- !15470 some readahead improvement +- readahead: properly shorten readahead when falling back to do_page_cache_ra() +- readahead: don't shorten readahead window in read_pages() +- readahead: make sure sync readahead reads needed page +- !15404 ipmr: do not call mr_mfc_uses_dev() for unres entries +- ipmr: do not call mr_mfc_uses_dev() for unres entries +- !15376 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.62-6.6.63 LTS Conflicts Patches +- mptcp: update local address flags when setting it +- NFSD: Async COPY result needs to return a write verifier +- !15164 [openEuler-24.03-LTS][linux-6.6.y sync] KVM: x86: Play nice with protected guests in complete_hypercall_exit() +- KVM: x86: Play nice with protected guests in complete_hypercall_exit() +- !15375 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.60-6.6.61 LTS Conflicts Patches +- posix-cpu-timers: Clear TICK_DEP_BIT_POSIX_TIMER on clone +- !15149 Fix northbridge init warning in guest for Hygon family 18h model 4h +- x86/amd_nb: Fix northbridge init warning in guest for Hygon family 18h model 4h +- !15147 Add device HID for Hygon GPIO controller +- pinctrl: Add device HID for Hygon GPIO controller +- !15438 fix HCT build err once VFIO disabled +- hct: fix build err once VFIO disabled + +* Fri Mar 14 2025 Mingzheng Xing - 6.6.0-82.0.0.87 +- RISC-V kernel upgrade to 6.6.0-82.0.0 +- riscv: Fixed conflict created by commit c7c8e8a76904 +- riscv: Enable zstd-compressed firmware support +- riscv: Enable AMDKFD driver for ROCm + +* Wed Mar 12 2025 ZhangPeng - 6.6.0-82.0.0.86 +- !15440 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.70-6.6.72 LTS Patches +- drm/mediatek: Only touch DISP_REG_OVL_PITCH_MSB if AFBC is supported +- riscv: Fix text patching when IPI are used +- mm: hugetlb: independent PMD page table shared count +- mm/hugetlb: enforce that PMD PT sharing has split PMD PT locks +- pgtable: fix s390 ptdesc field comments +- workqueue: Add rcu lock check at the end of work item execution +- pmdomain: imx: gpcv2: fix an OF node reference leak in imx_gpcv2_probe() +- pmdomain: imx: gpcv2: Simplify with scoped for each OF child loop +- arm64: dts: rockchip: add hevc power domain clock to rk3328 +- ARM: dts: imxrt1050: Fix clocks for mmc +- io_uring/eventfd: ensure io_eventfd_signal() defers another RCU period +- riscv: kprobes: Fix incorrect address calculation +- iio: adc: ad7124: Disable all channels at probe time +- iio: inkern: call iio_device_put() only on mapped devices +- iio: adc: ti-ads124s08: Use gpiod_set_value_cansleep() +- iio: gyro: fxas21002c: Fix missing data update in trigger handler +- iio: imu: kmx61: fix information leak in triggered buffer +- iio: dummy: iio_simply_dummy_buffer: fix information leak in triggered buffer +- iio: pressure: zpa2326: fix information leak in triggered buffer +- usb: gadget: configfs: Ignore trailing LF for user strings to cdev +- usb: gadget: f_fs: Remove WARN_ON in functionfs_bind +- usb: typec: tcpm/tcpci_maxim: fix error code in max_contaminant_read_resistance_kohm() +- usb: gadget: f_uac2: Fix incorrect setting of bNumEndpoints +- usb: chipidea: ci_hdrc_imx: decrement device's refcount in .remove() and in the error path of .probe() +- usb: gadget: midi2: Reverse-select at the right place +- usb: fix reference leak in usb_new_device() +- USB: core: Disable LPM only for non-suspended ports +- USB: usblp: return error when setting unsupported protocol +- usb: dwc3-am62: Disable autosuspend during remove +- usb: gadget: u_serial: Disable ep before setting port to null to fix the crash caused by port being null +- tty: serial: 8250: Fix another runtime PM usage counter underflow +- misc: microchip: pci1xxxx: Resolve return code mismatch during GPIO set config +- usb: dwc3: gadget: fix writing NYET threshold +- USB: serial: cp210x: add Phoenix Contact UPS Device +- usb-storage: Add max sectors quirk for Nokia 208 +- staging: iio: ad9832: Correct phase range check +- staging: iio: ad9834: Correct phase range check +- USB: serial: option: add Neoway N723-EA support +- USB: serial: option: add MeiG Smart SRM815 +- dm-verity FEC: Fix RS FEC repair for roots unaligned to block size (take 2) +- io_uring/timeout: fix multishot updates +- drm/amd/display: increase MAX_SURFACES to the value supported by hw +- drm/amdkfd: fixed page fault when enable MES shader debugger +- ACPI: resource: Add Asus Vivobook X1504VAP to irq1_level_low_skip_override[] +- ACPI: resource: Add TongFang GM5HG0A to irq1_edge_low_force_override[] +- smb: client: sync the root session and superblock context passwords before automounting +- thermal: of: fix OF node leak in of_thermal_zone_find() +- ksmbd: Implement new SMB3 POSIX type +- sctp: sysctl: plpmtud_probe_interval: avoid using current->nsproxy +- sctp: sysctl: udp_port: avoid using current->nsproxy +- sctp: sysctl: auth_enable: avoid using current->nsproxy +- sctp: sysctl: rto_min/max: avoid using current->nsproxy +- sctp: sysctl: cookie_hmac_alg: avoid using current->nsproxy +- mptcp: sysctl: sched: avoid using current->nsproxy +- dm-ebs: don't set the flag DM_TARGET_PASSES_INTEGRITY +- scsi: ufs: qcom: Power off the PHY if it was already powered on in ufs_qcom_power_up_sequence() +- dm thin: make get_first_thin use rcu-safe list first function +- riscv: mm: Fix the out of bound issue of vmemmap address +- cpuidle: riscv-sbi: fix device node release in early exit of for_each_possible_cpu +- ksmbd: fix unexpectedly changed path in ksmbd_vfs_kern_path_locked +- afs: Fix the maximum cell name length +- drm/mediatek: Add return value check when reading DPCD +- drm/mediatek: Fix mode valid issue for dp +- drm/mediatek: Fix YCbCr422 color format issue for DP +- drm/mediatek: stop selecting foreign drivers +- net/mlx5: Fix variable not being completed when function returns +- net: stmmac: dwmac-tegra: Read iommu stream id from device tree +- netfilter: nf_tables: imbalance in flowtable binding +- iio: imu: inv_icm42600: fix timestamps after suspend if sensor is on +- x86/mm/numa: Use NUMA_NO_NODE when calling memblock_set_node() +- memblock tests: fix implicit declaration of function 'numa_valid_node' +- riscv: Fix early ftrace nop patching +- tcp: Annotate data-race around sk->sk_mark in tcp_v4_send_reset +- Bluetooth: btnxpuart: Fix driver sending truncated data +- Bluetooth: MGMT: Fix Add Device to responding before completing +- Bluetooth: hci_sync: Fix not setting Random Address when required +- eth: gve: use appropriate helper to set xdp_features +- ipvlan: Fix use-after-free in ipvlan_get_iflink(). +- tls: Fix tls_sw_sendmsg error handling +- igc: return early when failing to read EECD register +- igc: field get conversion +- ice: fix incorrect PHY settings for 100 GB/s +- cxgb4: Avoid removal of uninserted tid +- bnxt_en: Fix possible memory leak when hwrm_req_replace fails +- pds_core: limit loop over fw name list +- net: libwx: fix firmware mailbox abnormal return +- tcp/dccp: allow a connection when sk_max_ack_backlog is zero +- tcp/dccp: complete lockless accesses to sk->sk_max_ack_backlog +- net: 802: LLC+SNAP OID:PID lookup on start of skb data +- ieee802154: ca8210: Add missing check for kfifo_alloc() in ca8210_probe() +- selftests/alsa: Fix circular dependency involving global-timer +- ASoC: mediatek: disable buffer pre-allocation +- ASoC: rt722: add delay time to wait for the calibration procedure +- erofs: fix PSI memstall accounting +- erofs: handle overlapped pclusters out of crafted images properly +- ovl: support encoding fid from inode with no alias +- ovl: pass realinode to ovl_encode_real_fh() instead of realdentry +- ovl: do not encode lower fh with upper sb_writers held +- exfat: fix the infinite loop in __exfat_free_cluster() +- dm array: fix cursor index when skipping across block boundaries +- dm array: fix unreleased btree blocks on closing a faulty array cursor +- dm array: fix releasing a faulty array block twice in dm_array_cursor_end +- jbd2: flush filesystem device before updating tail sequence +- jbd2: increase IO priority for writing revoke records +- memblock: make memblock_set_node() also warn about use of MAX_NUMNODES +- x86/hyperv: Fix hv tsc page based sched_clock for hibernation +- Revert "x86, crash: wrap crash dumping code into crash related ifdefs" +- Revert "x86/hyperv: Fix hv tsc page based sched_clock for hibernation" +- !15423 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.69-6.6.70 LTS Additional Patches +- !15423 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.69-6.6.70 LTS Additional Patches +- RDMA/bnxt_re: Fix max SGEs for the Work Request +- mptcp: don't always assume copied data in mptcp_cleanup_rbuf() +- mptcp: fix recvbuffer adjust on sleeping rcvmsg +- mm: vmscan: account for free pages to prevent infinite Loop in throttle_direct_reclaim() +- mm/kmemleak: fix sleeping function called from invalid context at print message +- mm/readahead: fix large folio support in async readahead +- fs/proc/task_mmu: fix pagemap flags with PMD THP entries on 32bit +- drm: adv7511: Fix use-after-free in adv7533_attach_dsi() +- dt-bindings: display: adi,adv7533: Drop single lane support +- drm: adv7511: Drop dsi single lane support +- net/sctp: Prevent autoclose integer overflow in sctp_association_init() +- sky2: Add device ID 11ab:4373 for Marvell 88E8075 +- RDMA/uverbs: Prevent integer overflow issue +- scripts/sorttable: fix orc_sort_cmp() to maintain symmetry and transitivity +- kcov: mark in_softirq_really() as __always_inline +- ALSA hda/realtek: Add quirk for Framework F111:000C +- ALSA: seq: Check UMP support for midi_version change +- modpost: fix the missed iteration for the max bit in do_input() +- modpost: fix input MODULE_DEVICE_TABLE() built for 64-bit on 32-bit host +- RDMA/bnxt_re: Fix the max WQE size for static WQE support +- ARC: build: Try to guess GCC variant of cross compiler +- irqchip/gic: Correct declaration of *percpu_base pointer in union gic_base +- Bluetooth: hci_core: Fix sleeping function called from invalid context +- net: usb: qmi_wwan: add Telit FE910C04 compositions +- smb: client: destroy cfid_put_wq on module exit +- ksmbd: set ATTR_CTIME flags when setting mtime +- ksmbd: retry iterate_dir in smb2_query_dir +- bpf: fix potential error return +- sound: usb: format: don't warn that raw DSD is unsupported +- sound: usb: enable DSD output for ddHiFi TC44C +- ALSA: hda/realtek: Add new alc2xx-fixup-headset-mic model +- drm/amdkfd: Correct the migration DMA map direction +- wifi: mac80211: wake the queues in case of failure in resume +- net: ti: icssg-prueth: Fix clearing of IEP_CMP_CFG registers during iep_init +- ila: serialize calls to nf_register_net_hooks() +- af_packet: fix vlan_get_protocol_dgram() vs MSG_PEEK +- af_packet: fix vlan_get_tci() vs MSG_PEEK +- net: wwan: iosm: Properly check for valid exec stage in ipc_mmio_init() +- net: restrict SO_REUSEPORT to inet sockets +- net: reenable NETIF_F_IPV6_CSUM offload for BIG TCP packets +- net: sfc: Correct key_len for efx_tc_ct_zone_ht_params +- RDMA/rtrs: Ensure 'ib_sge list' is accessible +- net: mv643xx_eth: fix an OF node reference leak +- eth: bcmsysport: fix call balance of priv->clk handling routines +- ALSA: usb-audio: US16x08: Initialize array before use +- net: llc: reset skb->transport_header +- netfilter: nft_set_hash: unaligned atomic read on struct nft_set_ext +- drm/i915/dg1: Fix power gate sequence. +- net/mlx5e: macsec: Maintain TX SA from encoding_sa +- net/mlx5: DR, select MSIX vector 0 for completion queue creation +- netrom: check buffer length before accessing it +- net: Fix netns for ip_tunnel_init_flow() +- ipv4: ip_tunnel: Unmask upper DSCP bits in ip_tunnel_xmit() +- ipv4: ip_tunnel: Unmask upper DSCP bits in ip_md_tunnel_xmit() +- ipv4: ip_tunnel: Unmask upper DSCP bits in ip_tunnel_bind_dev() +- ip_tunnel: annotate data-races around t->parms.link +- net: stmmac: restructure the error path of stmmac_probe_config_dt() +- net: stmmac: don't create a MDIO bus if unnecessary +- net: dsa: microchip: Fix LAN937X set_ageing_time function +- net: dsa: microchip: Fix KSZ9477 set_ageing_time function +- drm/bridge: adv7511_audio: Update Audio InfoFrame properly +- RDMA/bnxt_re: Fix the locking while accessing the QP table +- RDMA/bnxt_re: Fix MSN table size for variable wqe mode +- RDMA/bnxt_re: Add send queue size check for variable wqe +- RDMA/bnxt_re: Disable use of reserved wqes +- RDMA/bnxt_re: Add support for Variable WQE in Genp7 adapters +- RDMA/bnxt_re: Fix max_qp_wrs reported +- RDMA/bnxt_re: Fix reporting hw_ver in query_device +- RDMA/bnxt_re: Add check for path mtu in modify_qp +- RDMA/bnxt_re: Fix the check for 9060 condition +- nvme-pci: 512 byte aligned dma pool segment quirk +- RDMA/bnxt_re: Avoid sending the modify QP workaround for latest adapters +- RDMA/bnxt_re: Avoid initializing the software queue for user queues +- RDMA/mlx5: Enforce same type port association for multiport RoCE +- RDMA/bnxt_re: Remove always true dattr validity check +- RDMA/bnxt_re: Allow MSN table capability check +- powerpc: Remove initialisation of readpos +- net: mctp: handle skb cleanup on sock_queue failures +- ceph: give up on paths longer than PATH_MAX +- mmc: sdhci-msm: fix crypto key eviction +- !15418 ocfs2: handle a symlink read error correctly +- ocfs2: handle a symlink read error correctly +- !15299 ptp: Ensure info->enable callback is always set +- !15299 ptp: Ensure info->enable callback is always set +- ptp: Ensure info->enable callback is always set +- !15424 bpf: bpf_local_storage: Always use bpf_mem_alloc in PREEMPT_RT +- !15424 bpf: bpf_local_storage: Always use bpf_mem_alloc in PREEMPT_RT +- bpf: bpf_local_storage: Always use bpf_mem_alloc in PREEMPT_RT +- !15367 libbpf: Fix segfault due to libelf functions not setting errno +- !15367 libbpf: Fix segfault due to libelf functions not setting errno +- libbpf: Fix segfault due to libelf functions not setting errno +- !15417 [Intel-SIG] x86/cpu: Clearwater Forest support including CPU model and new ISAs +- x86: KVM: Advertise CPUIDs for new instructions in Clearwater Forest +- x86/cpu: Add model number for Intel Clearwater Forest processor +- !15225 cifs: Prevent NULL pointer dereference caused by cifs_sb->rsize is 0 +- !15225 cifs: Prevent NULL pointer dereference caused by cifs_sb->rsize is 0 +- cifs: Prevent NULL pointer dereference caused by cifs_sb->rsize is 0 +- !15390 backport mainline bugfix patch +- !15390 backport mainline bugfix patch +- irqchip/gic-v3: Handle CPU_PM_ENTER_FAILED correctly +- genirq: Make handle_enforce_irqctx() unconditionally available +- !15229 NFS: remove unused variable nfs_rpcstat +- NFS: remove unused variable nfs_rpcstat +- !14975 [OLK-6.6] [Backport] scsi: lpfc: Update phba link state conditional before sending CMF_SYNC_WQE +- scsi: lpfc: Update phba link state conditional before sending CMF_SYNC_WQE +- !15302 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.57-6.6.58 LTS Conflicts Patches +- selftests: mptcp: join: test for prohibited MPC to port-based endp +- !15292 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.56-6.6.57 LTS Conflicts Patches +- HID: mcp2200: added driver for GPIOs of MCP2200 +- !15326 [OLK-6.6]Hygon: Rename Hygon SM{3,4} feature macro to X86_FEATURE_HYGON_SM{3,4} +- !15326 [OLK-6.6]Hygon: Rename Hygon SM{3,4} feature macro to X86_FEATURE_HYGON_SM{3,4} +- x86/cpufeatures: Rename X86_FEATURE_SM{3,4} to X86_FEATURE_HYGON_SM{3,4} +- !15402 nilfs2: fix possible int overflows in nilfs_fiemap() +- !15402 nilfs2: fix possible int overflows in nilfs_fiemap() +- nilfs2: fix possible int overflows in nilfs_fiemap() +- !15043 wifi: mac80211: fix mbss changed flags corruption on 32 bit systems +- wifi: mac80211: fix mbss changed flags corruption on 32 bit systems +- !15283 wifi: rtw89: fix race between cancel_hw_scan and hw_scan completion +- wifi: rtw89: fix race between cancel_hw_scan and hw_scan completion +- !15308 CVE-2025-21732 +- RDMA/mlx5: Fix a race for an ODP MR which leads to CQE with error +- RDMA/mlx5: Ensure created mkeys always have a populated rb_key +- RDMA/mlx5: Change check for cacheable mkeys +- !15298 usbnet: ipheth: fix possible overflow in DPE length check +- usbnet: ipheth: fix possible overflow in DPE length check +- !15334 perf/core: Save raw sample data conditionally based on sample type +- perf/core: Save raw sample data conditionally based on sample type +- !15347 Mainline Patch +- bpf: Use preempt_count() directly in bpf_send_signal_common() +- bpf: Send signals asynchronously if !preemptible +- !15331 Fixes CVE-2025-21767 +- clocksource: Use migrate_disable() to avoid calling get_random_u32() in atomic context +- clocksource: Use pr_info() for "Checking clocksource synchronization" message +- !15221 smb: client: Fix netns refcount imbalance causing leaks and use-after-free +- smb: client: Fix netns refcount imbalance causing leaks and use-after-free +- !15364 LoongArch: csum: Fix OoB access in IP checksum code for negative lengths +- LoongArch: csum: Fix OoB access in IP checksum code for negative lengths +- !15345 USB: hub: Ignore non-compliant devices with too many configs or interfaces +- USB: hub: Ignore non-compliant devices with too many configs or interfaces +- !15363 gpiolib: Fix crash on error in gpiochip_get_ngpios() +- gpiolib: Fix crash on error in gpiochip_get_ngpios() +- !15385 orangefs: fix a oob in orangefs_debug_write +- orangefs: fix a oob in orangefs_debug_write +- !15265 NFSD: fix hang in nfsd4_shutdown_callback +- NFSD: fix hang in nfsd4_shutdown_callback +- !15263 NFS: Fix potential buffer overflowin nfs_sysfs_link_rpc_client() +- NFS: Fix potential buffer overflowin nfs_sysfs_link_rpc_client() +- !15266 nfsd: clear acl_access/acl_default after releasing them +- nfsd: clear acl_access/acl_default after releasing them +- !15346 block: mark GFP_NOIO around sysfs ->store() +- block: mark GFP_NOIO around sysfs ->store() +- !15309 usb: xhci: Fix NULL pointer dereference on certain command aborts +- !15309 usb: xhci: Fix NULL pointer dereference on certain command aborts +- usb: xhci: Fix NULL pointer dereference on certain command aborts +- !15204 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.69-6.6.70 LTS Patches +- !15204 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.69-6.6.70 LTS Patches +- platform/x86: mlx-platform: call pci_dev_put() to balance the refcount +- ALSA: ump: Shut up truncated string warning +- usb: xhci: Avoid queuing redundant Stop Endpoint commands +- cleanup: Remove address space of returned pointer +- crypto: ecc - Prevent ecc_digits_from_bytes from reading too many bytes +- Bluetooth: btusb: mediatek: add callback function in btusb_disconnect +- Bluetooth: btusb: add callback function in btusb suspend/resume +- scsi: mpi3mr: Start controller indexing from 0 +- scsi: mpi3mr: Use ida to manage mrioc ID +- ALSA: ump: Update legacy substream names upon FB info update +- ALSA: ump: Indicate the inactive group in legacy substream names +- ALSA: ump: Don't open legacy substream for an inactive group +- ALSA: ump: Use guard() for locking +- udf: Verify inode link counts before performing rename +- udf_rename(): only access the child content on cross-directory rename +- watchdog: rzg2l_wdt: Power on the watchdog domain in the restart handler +- watchdog: rzg2l_wdt: Rely on the reset driver for doing proper reset +- watchdog: rzg2l_wdt: Remove reset de-assert from probe +- of: address: Preserve the flags portion on 1:1 dma-ranges mapping +- of: address: Store number of bus flag cells rather than bool +- of: address: Remove duplicated functions +- x86/hyperv: Fix hv tsc page based sched_clock for hibernation +- x86, crash: wrap crash dumping code into crash related ifdefs +- thunderbolt: Don't display nvm_version unless upgrade supported +- thunderbolt: Add support for Intel Panther Lake-M/P +- thunderbolt: Add support for Intel Lunar Lake +- xhci: Turn NEC specific quirk for handling Stop Endpoint errors generic +- usb: xhci: Limit Stop Endpoint retries +- xhci: retry Stop Endpoint on buggy NEC controllers +- net: renesas: rswitch: fix possible early skb release +- softirq: Allow raising SCHED_SOFTIRQ from SMP-call-function on RT kernel +- net/mlx5: unique names for per device caches +- Revert "nvme: make keep-alive synchronous operation" +- nvme: use helper nvme_ctrl_state in nvme_keep_alive_finish function +- usb: typec: ucsi: add update_connector callback +- usb: typec: ucsi: add callback for connector status updates +- iio: adc: ad7192: properly check spi_get_device_match_data() +- iio: adc: ad7192: Convert from of specific to fwnode property handling +- usb: chipidea: udc: limit usb request length to max 16KB +- usb: chipidea: add CI_HDRC_HAS_SHORT_PKT_LIMIT flag +- usb: chipidea: add CI_HDRC_FORCE_VBUS_ACTIVE_ALWAYS flag +- fs/ntfs3: Fix warning in ni_fiemap +- fs/ntfs3: Implement fallocate for compressed files +- remoteproc: qcom: pas: enable SAR2130P audio DSP support +- remoteproc: qcom: pas: Add support for SA8775p ADSP, CDSP and GPDSP +- remoteproc: qcom: pas: Add sc7180 adsp +- mailbox: pcc: Check before sending MCTP PCC response ACK +- clk: qcom: clk-alpha-pll: Add NSS HUAYRA ALPHA PLL support for ipq9574 +- clk: qcom: clk-alpha-pll: Add support for zonda ole pll configure +- Bluetooth: btusb: Add new VID/PID 0489/e111 for MT7925 +- Bluetooth: btusb: Add USB HW IDs for MT7921/MT7922/MT7925 +- Bluetooth: btusb: Add new VID/PID 13d3/3602 for MT7925 +- Bluetooth: Add support ITTIM PE50-M75C +- Bluetooth: hci_conn: Reduce hci_conn_drop() calls in two functions +- i2c: i801: Add support for Intel Panther Lake +- i2c: i801: Add support for Intel Arrow Lake-H +- wifi: ath10k: avoid NULL pointer error during sdio remove +- wifi: ath10k: Update Qualcomm Innovation Center, Inc. copyrights +- media: uvcvideo: Force UVC version to 1.0a for 0408:4033 +- media: uvcvideo: Force UVC version to 1.0a for 0408:4035 +- cleanup: Adjust scoped_guard() macros to avoid potential warning +- cleanup: Add conditional guard support +- crypto: ecdsa - Avoid signed integer overflow on signature decoding +- crypto: ecdsa - Use ecc_digits_from_bytes to convert signature +- crypto: ecdsa - Rename keylen to bufsize where necessary +- crypto: ecdsa - Convert byte arrays with key coordinates to digits +- ext4: partial zero eof block on unaligned inode size extension +- ext4: convert to new timestamp accessors +- memblock: allow zero threshold in validate_numa_converage() +- NUMA: optimize detection of memory with no node id assigned by firmware +- sched: Initialize idle tasks only once +- usb: dwc3: gadget: Add missing check for single port RAM in TxFIFO resizing logic +- smb: client: fix use-after-free of signing key +- smb: client: stop flooding dmesg in smb2_calc_signature() +- fs/smb/client: implement chmod() for SMB3 POSIX Extensions +- smb/client: rename cifs_ace to smb_ace +- smb/client: rename cifs_acl to smb_acl +- smb/client: rename cifs_sid to smb_sid +- smb/client: rename cifs_ntsd to smb_ntsd +- x86/mm: Carve out INVLPG inline asm for use by others +- docs: media: update location of the media patches +- drm/amd/display: Fix incorrect DSC recompute trigger +- drm/amd/display: Fix DSC-re-computing + +* Wed Mar 05 2025 ZhangPeng - 6.6.0-80.0.0.85 +- !15278 tracing/osnoise: Fix resetting of tracepoints +- tracing/osnoise: Fix resetting of tracepoints +- !15227 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.54-6.6.55 LTS Conflicts Patches +- mm: z3fold: deprecate CONFIG_Z3FOLD +- null_blk: Remove usage of the deprecated ida_simple_xx() API +- iomap: handle a post-direct I/O invalidate race in iomap_write_delalloc_release +- proc: add config & param to block forcing mem writes +- HID: Ignore battery for all ELAN I2C-HID devices +- ata: sata_sil: Rename sil_blacklist to sil_quirks +- ata: pata_serverworks: Do not use the term blacklist +- scsi: aacraid: Rearrange order of struct aac_srb_unit +- uprobes: Fix race in uprobe_free_utask +- perf,x86: Fix kabi breakage in struct uprobe_task +- perf,x86: avoid missing caller address in stack traces captured in uprobe +- i2c: synquacer: Deal with optional PCLK correctly +- i2c: synquacer: Remove a clk reference from struct synquacer_i2c +- !15248 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.64-6.6.66 LTS Conflicts Patches +- mm/mempolicy: fix migrate_to_node() assuming there is at least one VMA in a MM +- selftests/ftrace: adjust offset for kprobe syntax error test +- bpf: Remove unnecessary kfree(im_node) in lpm_trie_update_elem +- iommu/arm-smmu: Defer probe of clients after smmu device bound +- gpio: grgpio: use a helper variable to store the address of ofdev->dev +- KVM: arm64: Change kvm_handle_mmio_return() return polarity +- !15270 v2 mm: Backport two patch +- arm64/mm: Drop setting PTE_TYPE_PAGE in pte_mkcont() +- arm64/mm: Re-organize arch_make_huge_pte() +- mm/hugetlb: use arch_make_huge_pte() in __hugetlb_insert_pfn() +- !14820 [OLK-6.6]Hygon: Support kernel boot verification and secret injection for Hygon CSV3 VM +- KVM: SVM: CSV: Support inject secret to Hygon CSV3 guest +- KVM: SVM: CSV: Support issue non-4K aligned CSV3_CMD_LAUNCH_ENCRYPT_DATA and more than once +- KVM: SVM: CSV: Provide KVM_CSV3_SET_GUEST_PRIVATE_MEMORY ioctl interface +- KVM: SVM: CSV: Provide KVM_CAP_HYGON_COCO_EXT interface +- crypto: ccp: Provide csv_get_extension_info() to present extensions of newer CSV firmware +- KVM: SVM: CSV: Ensure all the GPRs and some non-GPRs are synced before LAUNCH_ENCRYPT_VMCB +- crypto: ccp: Get api version again when update Hygon CSV firmware at runtime +- KVM: SVM: CSV: Fix the vm_size even if CSV3 feature is unsupported on Hygon CPUs +- !14819 [OLK-6.6]Hygon: Support CSV3 Attestation +- x86/configs: openeuler_defconfig: Set CONFIG_VIRT_DRIVERS=y and CONFIG_EFI_SECRET=m +- virt/csv-guest: Provide interface for request of CSV3 attestation report +- x86/csv: Add support for CSV3 ATTESTATION secure call +- x86/csv: Define ATTESTATION secure call command +- !15251 Intel: Backport upstream auto-tune per-CPU pageset size patchset +- mm and cache_info: remove unnecessary CPU cache info update +- !15185 hwtracing: hisi_ptt: Initialize the filter sysfs attribute when allocation +- hwtracing: hisi_ptt: Check duplicate filters before allocation +- hwtracing: hisi_ptt: Initialize the filter sysfs attribute when allocation + +* Wed Feb 26 2025 ZhangPeng - 6.6.0-79.0.0.84 +- !15203 usb: cdc-acm: Check control transfer buffer size before access +- usb: cdc-acm: Check control transfer buffer size before access +- !15214 USB: serial: quatech2: fix null-ptr-deref in qt2_process_read_urb() +- USB: serial: quatech2: fix null-ptr-deref in qt2_process_read_urb() +- !15178 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.53-6.6.54 LTS Conflicts Patches +- perf time-utils: Fix 32-bit nsec parsing +- perf sched timehist: Fixed timestamp error when unable to confirm event sched_in time +- perf stat: Display iostat headers correctly +- perf sched timehist: Fix missing free of session in perf_sched__timehist() +- perf report: Fix --total-cycles --stdio output error +- perf ui/browser/annotate: Use global annotation_options +- perf annotate: Move some source code related fields from 'struct annotation' to 'struct annotated_source' +- perf annotate: Split branch stack cycles info from 'struct annotation' +- perf inject: Fix leader sampling inserting additional samples +- perf mem: Free the allocated sort string, fixing a leak +- !14179 [OLK-6.6] update ITMT support patch for Zhaoxin CPUs +- cpufreq: acpi-cpufreq: Zhaoxin: fix incorrect max-freq issue +- cpufreq: ACPI: add ITMT support when CPPC enabled +- Revert "cpufreq: ACPI: add ITMT support when CPPC enabled" +- !15200 arm64/mpam: Try reading again if monitor instance returns not ready +- arm64/mpam: Try reading again if monitor instance returns not ready +- !15197 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.68-6.6.69 LTS Patches +- block: avoid to reuse `hctx` not removed from cpuhp callback list +- ALSA: hda/realtek: Fix spelling mistake "Firelfy" -> "Firefly" +- ALSA: sh: Fix wrong argument order for copy_from_iter() +- btrfs: sysfs: fix direct super block member reads +- btrfs: avoid monopolizing a core when activating a swap file +- power: supply: gpio-charger: Fix set charge current limits +- drm/amdkfd: pause autosuspend when creating pdd +- drm/amdkfd: Use device based logging for errors +- drm/amdkfd: drop struct kfd_cu_info +- drm/amdkfd: reduce stack size in kfd_topology_add_device() +- drm/amdgpu/hdp6.0: do a posting read when flushing HDP +- drm/amdgpu/hdp5.0: do a posting read when flushing HDP +- drm/amdgpu/hdp4.0: do a posting read when flushing HDP +- drm/amd/amdgpu: allow use kiq to do hdp flush under sriov +- pmdomain: core: Add missing put_device() +- ALSA: hda/realtek: fix micmute LEDs don't work on HP Laptops +- ALSA: hda/realtek: fix mute/micmute LEDs don't work for EliteBook X G1i +- MIPS: mipsregs: Set proper ISA level for virt extensions +- MIPS: Probe toolchain support of -msym32 +- blk-mq: move cpuhp callback registering out of q->sysfs_lock +- blk-mq: register cpuhp callback after hctx is added to xarray table +- drm/dp_mst: Ensure mst_primary pointer is valid in drm_dp_mst_handle_up_req() +- spi: omap2-mcspi: Fix the IS_ERR() bug for devm_clk_get_optional_enabled() +- scsi: storvsc: Do not flag MAINTENANCE_IN return of SRB_STATUS_DATA_OVERRUN as an error +- scsi: mpt3sas: Diag-Reset when Doorbell-In-Use bit is set during driver load time +- spi: intel: Add Panther Lake SPI controller support +- platform/x86: asus-nb-wmi: Ignore unknown event 0xCF +- LoongArch: BPF: Adjust the parameter of emit_jirl() +- LoongArch: Fix reserving screen info memory for above-4G firmware +- regmap: Use correct format specifier for logging range errors +- smb: server: Fix building with GCC 15 +- ALSA: sh: Use standard helper for buffer accesses +- ALSA: hda/conexant: fix Z60MR100 startup pop issue +- udf: Skip parent dir link count update if corrupted +- scsi: qla1280: Fix hw revision numbering for ISP1020/1040 +- watchdog: mediatek: Add support for MT6735 TOPRGU/WDT +- watchdog: it87_wdt: add PWRGD enable quirk for Qotom QCML04 +- stddef: make __struct_group() UAPI C++-friendly +- powerpc/pseries/vas: Add close() callback in vas_vm_ops struct +- mtd: rawnand: fix double free in atmel_pmecc_create_user() +- dmaengine: at_xdmac: avoid null_prt_deref in at_xdmac_prep_dma_memset +- dmaengine: apple-admac: Avoid accessing registers in probe +- dmaengine: fsl-edma: implement the cleanup path of fsl_edma3_attach_pd() +- dmaengine: tegra: Return correct DMA status when paused +- dmaengine: dw: Select only supported masters for ACPI devices +- dmaengine: mv_xor: fix child node refcount handling in early exit +- phy: rockchip: naneng-combphy: fix phy reset +- phy: usb: Toggle the PHY power during init +- phy: core: Fix that API devm_phy_destroy() fails to destroy the phy +- phy: core: Fix that API devm_of_phy_provider_unregister() fails to unregister the phy provider +- phy: core: Fix that API devm_phy_put() fails to release the phy +- phy: core: Fix an OF node refcount leakage in of_phy_provider_lookup() +- phy: core: Fix an OF node refcount leakage in _of_phy_get() +- phy: qcom-qmp: Fix register name in RX Lane config of SC8280XP +- mtd: rawnand: arasan: Fix missing de-registration of NAND +- mtd: rawnand: arasan: Fix double assertion of chip-select +- mtd: diskonchip: Cast an operand to prevent potential overflow +- nfsd: restore callback functionality for NFSv4.0 +- nfsd: Revert "nfsd: release svc_expkey/svc_export with rcu_work" +- bpf: Check negative offsets in __bpf_skb_min_len() +- tcp_bpf: Charge receive socket buffer in bpf_tcp_ingress() +- mm/vmstat: fix a W=1 clang compiler warning +- ceph: allocate sparse_ext map only for sparse reads +- ceph: fix memory leak in ceph_direct_read_write() +- ceph: try to allocate a smaller extent map for sparse read +- !15190 v2 fs/resctrl: Adapt to the hardware topology structures of RDT and MPAM +- arm64/mpam: Add write memory barrier to guarantee monitor results +- arm64/mpam: Improve conversion accuracy between percent and fixed-point fraction +- arm64/mpam: Fix out-of-bound access of cfg array +- arm64/mpam: fix bug in percent_to_mbw_max() +- arm64/mpam: fix MBA granularity conversion formula +- arm64/mpam: Fix out-of-bound access of mbwu_state array +- arm64/mpam: Add judgment to distinguish MSMON_MBWU_CAPTURE definition +- arm64/mpam: Support MATA monitor feature for MPAM +- fs/resctrl: Adapt to the hardware topology structures of RDT and MPAM +- !15192 scsi: megaraid_sas: Fix for a potential deadlock +- scsi: megaraid_sas: Fix for a potential deadlock +- !15191 gfs2: Truncate address space when flipping GFS2_DIF_JDATA flag +- gfs2: Truncate address space when flipping GFS2_DIF_JDATA flag +- !15170 The QoS feature adapts to the cfs bandwidth throttling +- sched/fair: Fix CPU bandwidth limit bypass during CPU hotplug +- sched/qos: Fix qos throttling in SMT expelled +- hungtask: fixed offline group hung task issue under high load scenarios +- !15176 virtio-blk: don't keep queue frozen during system suspend +- virtio-blk: don't keep queue frozen during system suspend +- !15175 block, bfq: fix waker_bfqq UAF after bfq_split_bfqq() +- block, bfq: fix waker_bfqq UAF after bfq_split_bfqq() +- !15167 Backport tracing fix patch +- trace: Fix kabi breakage for trace_event_fields +- tracing: Check "%s" dereference via the field and not the TP_printk format +- !15181 kobject: Fix global-out-of-bounds in kobject_action_type() +- kobject: Fix global-out-of-bounds in kobject_action_type() +- !15184 binfmt_flat: Fix integer overflow bug on 32 bit systems +- binfmt_flat: Fix integer overflow bug on 32 bit systems +- !15161 Revert "irqchip/gic-v3-its: Fix VSYNC referencing an unmapped VPE on GIC v4.0" +- Revert "irqchip/gic-v3-its: Fix VSYNC referencing an unmapped VPE on GIC v4.0" +- !15166 tracing: Have process_string() also allow arrays +- tracing: Have process_string() also allow arrays +- !14939 Fix the performance monitor relates to core for Hygon family 18h processor +- perf/x86/amd/core: Fix performance monitor for Hygon family 18h processor +- !15159 mm/thp: fix deferred split unqueue naming and locking +- mm/thp: fix deferred split unqueue naming and locking +- !15142 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.66-6.6.68 LTS Patches +- dt-bindings: net: fec: add pps channel property +- drm/amdgpu: Handle NULL bo->tbo.resource (again) in amdgpu_vm_bo_update +- net: fec: make PPS channel configurable +- net: fec: refactor PPS channel configuration +- io_uring/rw: avoid punting to io-wq directly +- io_uring/rw: treat -EOPNOTSUPP for IOCB_NOWAIT like -EAGAIN +- io_uring/rw: split io_read() into a helper +- epoll: Add synchronous wakeup support for ep_poll_callback +- ceph: fix memory leaks in __ceph_sync_read() +- ceph: improve error handling and short/overflow-read logic in __ceph_sync_read() +- ceph: validate snapdirname option length when mounting +- of: Fix refcount leakage for OF node returned by __of_get_dma_parent() +- of: Fix error path in of_parse_phandle_with_args_map() +- udmabuf: also check for F_SEAL_FUTURE_WRITE +- nilfs2: fix buffer head leaks in calls to truncate_inode_pages() +- of/irq: Fix using uninitialized variable @addr_len in API of_irq_parse_one() +- of/irq: Fix interrupt-map cell length check in of_irq_parse_imap_parent() +- NFS/pnfs: Fix a live lock between recalled layouts and layoutget +- io_uring: Fix registered ring file refcount leak +- selftests/bpf: Use asm constraint "m" for LoongArch +- selftests/memfd: run sysctl tests when PID namespace support is enabled +- tracing: Add "%s" check in test_event_printk() +- tracing: Add missing helper functions in event pointer dereference check +- tracing: Fix test_event_printk() to process entire print argument +- smb: client: fix TCP timers deadlock after rmmod +- Drivers: hv: util: Avoid accessing a ringbuffer not initialized yet +- btrfs: tree-checker: reject inline extent items with 0 ref count +- vmalloc: fix accounting with i915 +- zram: fix uninitialized ZRAM not releasing backing device +- zram: refuse to use zero sized block device as backing device +- hwmon: (tmp513) Fix interpretation of values of Temperature Result and Limit Registers +- hwmon: (tmp513) Fix Current Register value interpretation +- hwmon: (tmp513) Fix interpretation of values of Shunt Voltage and Limit Registers +- hwmon: (tmp513) Use SI constants from units.h +- hwmon: (tmp513) Simplify with dev_err_probe() +- hwmon: (tmp513) Don't use "proxy" headers +- drm/amdgpu: don't access invalid sched +- i915/guc: Accumulate active runtime on gt reset +- i915/guc: Ensure busyness counter increases motonically +- i915/guc: Reset engine utilization buffer before registration +- drm/panel: novatek-nt35950: fix return value check in nt35950_probe() +- thunderbolt: Improve redrive mode handling +- USB: serial: option: add Telit FE910C04 rmnet compositions +- USB: serial: option: add MediaTek T7XX compositions +- USB: serial: option: add Netprisma LCUK54 modules for WWAN Ready +- USB: serial: option: add MeiG Smart SLM770A +- USB: serial: option: add TCL IK512 MBIM & ECM +- hexagon: Disable constant extender optimization for LLVM prior to 19.1.0 +- efivarfs: Fix error on non-existent file +- i2c: riic: Always round-up when calculating bus period +- chelsio/chtls: prevent potential integer overflow on 32bit +- net: tun: fix tun_napi_alloc_frags() +- KVM: x86: Cache CPUID.0xD XSTATE offsets+sizes during module init +- EDAC/amd64: Simplify ECC check on unified memory controllers +- mmc: mtk-sd: disable wakeup in .remove() and in the error path of .probe() +- mmc: sdhci-tegra: Remove SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC quirk +- net: mdiobus: fix an OF node reference leak +- selftests: openvswitch: fix tcpdump execution +- netfilter: ipset: Fix for recursive locking warning +- ipvs: Fix clamp() of ip_vs_conn_tab on small memory systems +- net: ethernet: bgmac-platform: fix an OF node reference leak +- ksmbd: fix broken transfers when exceeding max simultaneous operations +- ksmbd: count all requests in req_running counter +- net: renesas: rswitch: rework ts tags management +- ionic: use ee->offset when returning sprom data +- netdevsim: prevent bad user input in nsim_dev_health_break_write() +- net: mscc: ocelot: fix incorrect IFH SRC_PORT field in ocelot_ifh_set_basic() +- net/smc: check sndbuf_space again after NOSPACE flag is set in smc_poll +- cxl/region: Fix region creation for greater than x2 switches +- cxl/pci: Fix potential bogus return value upon successful probing +- tools: hv: change permissions of NetworkManager configuration file +- xfs: reset rootdir extent size hint after growfsrt +- xfs: take m_growlock when running growfsrt +- xfs: Fix the owner setting issue for rmap query in xfs fsmap +- xfs: conditionally allow FS_XFLAG_REALTIME changes if S_DAX is set +- xfs: attr forks require attr, not attr2 +- xfs: remove unused parameter in macro XFS_DQUOT_LOGRES +- xfs: fix file_path handling in tracepoints +- xfs: convert comma to semicolon +- xfs: Fix xfs_prepare_shift() range for RT +- xfs: Fix xfs_flush_unmap_range() range for RT +- xfs: create a new helper to return a file's allocation unit +- xfs: declare xfs_file.c symbols in xfs_file.h +- xfs: use consistent uid/gid when grabbing dquots for inodes +- xfs: verify buffer, inode, and dquot items every tx commit +- xfs: fix the contact address for the sysfs ABI documentation +- i2c: pnx: Fix timeout in wait functions +- p2sb: Do not scan and remove the P2SB device when it is unhidden +- p2sb: Move P2SB hide and unhide code to p2sb_scan_and_cache() +- p2sb: Introduce the global flag p2sb_hidden_by_bios +- p2sb: Factor out p2sb_read_from_cache() +- platform/x86: p2sb: Make p2sb_get_devfn() return void +- usb: cdns3: Add quirk flag to enable suspend residency +- PCI/AER: Disable AER service on suspend +- PCI: Use preserve_config in place of pci_flags +- ASoC: Intel: sof_sdw: add quirk for Dell SKU 0B8C +- ASoC: Intel: sof_sdw: fix jack detection on ADL-N variant RVP +- MIPS: Loongson64: DTS: Fix msi node for ls7a +- usb: cdns3-ti: Add workaround for Errata i2409 +- PCI: Add ACS quirk for Broadcom BCM5760X NIC +- PCI: vmd: Create domain symlink before pci_bus_add_devices() +- usb: dwc2: gadget: Don't write invalid mapped sg entries into dma_desc with iommu enabled +- net: sched: fix ordering of qlen adjustment +- x86/static-call: fix 32-bit build +- x86/xen: remove hypercall page +- x86/xen: use new hypercall functions instead of hypercall page +- x86/xen: add central hypercall functions +- x86/static-call: provide a way to do very early static-call updates +- objtool/x86: allow syscall instruction +- x86: make get_cpu_vendor() accessible from Xen code +- xen/netfront: fix crash when removing device +- net: rswitch: Avoid use-after-free in rswitch_poll() +- selftests/bpf: remove use of __xlated() +- tracing/kprobes: Skip symbol counting logic for module symbols in create_local_trace_kprobe() +- kselftest/arm64: abi: fix SVCR detection +- blk-iocost: Avoid using clamp() on inuse in __propagate_weights() +- net: dsa: microchip: KSZ9896 register regmap alignment to 32 bit boundaries +- net: renesas: rswitch: fix initial MPIC register setting +- Bluetooth: SCO: Add support for 16 bits transparent voice setting +- Bluetooth: iso: Fix recursive locking warning +- ACPICA: events/evxfregn: don't release the ContextMutex that was never acquired +- team: Fix feature propagation of NETIF_F_GSO_ENCAP_ALL +- bonding: Fix feature propagation of NETIF_F_GSO_ENCAP_ALL +- net/sched: netem: account for backlog updates from child qdisc +- net: dsa: felix: fix stuck CPU-injected packets with short taprio windows +- netfilter: nf_tables: do not defer rule destruction via call_rcu +- net: renesas: rswitch: handle stop vs interrupt race +- net: renesas: rswitch: fix leaked pointer on error path +- net: renesas: rswitch: fix race window between tx start and complete +- net: rswitch: Add jumbo frames handling for TX +- net: rswitch: Add a setting ext descriptor function +- net: rswitch: Add unmap_addrs instead of dma address in each desc +- net: rswitch: Use build_skb() for RX +- net: rswitch: Use unsigned int for desc related array index +- net: rswitch: Drop unused argument/return value +- Documentation: PM: Clarify pm_runtime_resume_and_get() return value +- ASoC: amd: yc: Fix the wrong return value +- ALSA: control: Avoid WARN() for symlink errors +- qca_spi: Make driver probing reliable +- qca_spi: Fix clock speed for multiple QCA7000 +- cxgb4: use port number to set mac addr +- ACPI: resource: Fix memory resource type union access +- net: sparx5: fix the maximum frame length register +- net: sparx5: fix FDMA performance issue +- spi: aspeed: Fix an error handling path in aspeed_spi_[read|write]_user() +- net: mscc: ocelot: perform error cleanup in ocelot_hwstamp_set() +- net: mscc: ocelot: be resilient to loss of PTP packets during transmission +- net: mscc: ocelot: ocelot->ts_id_lock and ocelot_port->tx_skbs.lock are IRQ-safe +- net: mscc: ocelot: improve handling of TX timestamp for unknown skb +- net: mscc: ocelot: fix memory leak on ocelot_port_add_txtstamp_skb() +- net: lapb: increase LAPB_HEADER_LEN +- ptp: kvm: x86: Return EOPNOTSUPP instead of ENODEV from kvm_arch_ptp_init() +- selftests: mlxsw: sharedbuffer: Ensure no extra packets are counted +- selftests: mlxsw: sharedbuffer: Remove duplicate test cases +- selftests: mlxsw: sharedbuffer: Remove h1 ingress test case +- wifi: cfg80211: sme: init n_channels before channels[] access +- tipc: fix NULL deref in cleanup_bearer() +- batman-adv: Do not let TT changes list grows indefinitely +- batman-adv: Remove uninitialized data in full table TT response +- batman-adv: Do not send uninitialized TT changes +- amdgpu/uvd: get ring reference from rq scheduler +- wifi: mac80211: fix station NSS capability initialization order +- wifi: mac80211: clean up 'ret' in sta_link_apply_parameters() +- wifi: mac80211: init cnt before accessing elem in ieee80211_copy_mbssid_beacon +- rtla/timerlat: Make timerlat_hist_cpu->*_count unsigned long long +- bpf, sockmap: Fix update element with same +- bpf, sockmap: Fix race between element replace and close() +- xfs: only run precommits once per transaction object +- xfs: fix scrub tracepoints when inode-rooted btrees are involved +- xfs: return from xfs_symlink_verify early on V4 filesystems +- xfs: don't drop errno values when we fail to ficlone the entire range +- xfs: update btree keys correctly when _insrec splits an inode root block +- drm/i915: Fix memory leak by correcting cache object name in error handler +- usb: dwc3: xilinx: make sure pipe clock is deselected in usb2 only mode +- usb: gadget: u_serial: Fix the issue that gs_start_io crashed due to accessing null pointer +- usb: typec: anx7411: fix OF node reference leaks in anx7411_typec_switch_probe() +- usb: typec: anx7411: fix fwnode_handle reference leak +- usb: ehci-hcd: fix call balance of clocks handling routines +- usb: gadget: midi2: Fix interpretation of is_midi1 bits +- scsi: ufs: core: Update compl_time_stamp_local_clock after completing a cqe +- usb: dwc2: Fix HCD port connection race +- usb: dwc2: hcd: Fix GetPortStatus & SetPortFeature +- usb: dwc2: Fix HCD resume +- ata: sata_highbank: fix OF node reference leak in highbank_initialize_phys() +- usb: host: max3421-hcd: Correctly abort a USB request. +- riscv: Fix IPIs usage in kfence_protect_page() +- ALSA: usb-audio: Add implicit feedback quirk for Yamaha THR5 +- riscv: Fix wrong usage of __pa() on a fixmap address +- tcp: check space before adding MPTCP SYN options +- splice: do not checksum AF_UNIX sockets +- ksmbd: fix racy issue from session lookup and expire +- !15144 hwmon: (drivetemp) Fix driver producing garbage data when SCSI errors occur +- hwmon: (drivetemp) Fix driver producing garbage data when SCSI errors occur +- !15143 scsi: mpi3mr: Fix corrupt config pages PHY state is switched in sysfs +- scsi: mpi3mr: Fix corrupt config pages PHY state is switched in sysfs +- !15132 CVE-2024-56607 fix +- wifi: ath12k: fix atomic calls in ath12k_mac_op_set_bitrate_mask() +- wifi: mac80211: Add non-atomic station iterator +- !15140 iio: adc: rockchip_saradc: fix information leak in triggered buffer +- iio: adc: rockchip_saradc: fix information leak in triggered buffer + +* Wed Feb 19 2025 ZhangPeng - 6.6.0-78.0.0.83 +- !15076 [olk 6.6] backport some feature patches for hibmcge driver +- net: hibmcge: Add nway_reset supported in this module +- net: hibmcge: Add reset supported in this module +- net: hibmcge: Add pauseparam supported in this module +- net: hibmcge: Add register dump supported in this module +- net: hibmcge: Add unicast frame filter supported in this module +- net: hibmcge: Add irq_info file to debugfs +- net: hibmcge: Add debugfs supported in this module +- lib/string_choices: Add str_true_false()/str_false_true() helper +- !15109 net: Fix icmp host relookup triggering ip_rt_bug +- net: Fix icmp host relookup triggering ip_rt_bug +- !15105 ipv6: Fix soft lockups in fib6_select_path under high next hop churn +- ipv6: Fix soft lockups in fib6_select_path under high next hop churn +- !15106 netfilter: conntrack: clamp maximum hashtable size to INT_MAX +- netfilter: conntrack: clamp maximum hashtable size to INT_MAX +- !15101 net/mlx5e: Don't call cleanup on profile rollback failure +- net/mlx5e: Don't call cleanup on profile rollback failure +- !15103 ethtool: fail closed if we can't get max channel used in indirection tables +- ethtool: fail closed if we can't get max channel used in indirection tables +- !15102 v2 net: Fix CVE-2024-56658 +- net: Fix kabi breakage in struct net +- net: defer final 'struct net' free in netns dismantle +- !15107 netfilter: IDLETIMER: Fix for possible ABBA deadlock +- netfilter: IDLETIMER: Fix for possible ABBA deadlock +- !15110 net/mlx5e: Skip restore TC rules for vport rep without loaded flag +- net/mlx5e: Skip restore TC rules for vport rep without loaded flag +- !15111 net: wwan: t7xx: Fix FSM command timeout issue +- net: wwan: t7xx: Fix FSM command timeout issue +- !15112 net_sched: cls_flow: validate TCA_FLOW_RSHIFT attribute +- net_sched: cls_flow: validate TCA_FLOW_RSHIFT attribute +- !15113 sched: sch_cake: add bounds checks to host bulk flow fairness counts +- sched: sch_cake: add bounds checks to host bulk flow fairness counts +- !15119 mm: resolve faulty mmap_region() error path behaviour +- mm: resolve faulty mmap_region() error path behaviour +- !15118 iio: adc: at91: call input_free_device() on allocated iio_dev +- iio: adc: at91: call input_free_device() on allocated iio_dev +- !15123 mm, madvise: fix potential workingset node list_lru leaks +- mm, madvise: fix potential workingset node list_lru leaks +- mm: move mapping_set_update out of +- !15055 iomap: avoid avoid truncating 64-bit offset to 32 bits +- iomap: avoid avoid truncating 64-bit offset to 32 bits +- !15128 ALSA: seq: oss: Fix races at processing SysEx messages +- ALSA: seq: oss: Fix races at processing SysEx messages +- !14972 selinux: ignore unknown extended permissions +- selinux: ignore unknown extended permissions +- !15121 KVM: arm64: Get rid of userspace_irqchip_in_use +- KVM: arm64: Get rid of userspace_irqchip_in_use +- !13446 [OLK-6.6]crypto: montage - fix variable 'device_handle' is uninitialized warning for Mont-TSSE driver +- crypto: montage - fix variable 'device_handle' is unintialized warning for Mont-TSSE driver +- !15133 Upgrade Hygon HCT driver to version 0.6 +- hct: remove the unused variables from hct shared structure. +- hct: add write support for node use of MDEV devices. +- hct: fix vm_start issue caused by pre allocated vma changes. +- hct: compatible with hct.ko modules in versions 0.5 and earlier. +- hct: update pasid type from int to long. +- hct: sharing CCP resources between host and virtual machines. +- hct: adding node to identifies hct mdev device usage status. +- !15083 mm/vmalloc: combine all TLB flush operations of KASAN shadow virtual address into one operation +- mm/vmalloc: combine all TLB flush operations of KASAN shadow virtual address into one operation +- !15080 [OLK-6.6] crypto: zhaoxin - Fix incorrect parameter in sm4 driver +- crypto: zhaoxin - Fix incorrect parameter in sm4 driver +- !15078 [OLK-6.6] crypto: zhaoxin - Optimize the assembly code in the SM3 driver +- crypto: zhaoxin - Optimize the assembly code in the SM3 driver +- !15068 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.64-6.6.66 LTS Patches +- i3c: master: svc: fix possible assignment of the same address to two devices +- i3c: master: Remove i3c_dev_disable_ibi_locked(olddev) on device hotjoin +- serial: amba-pl011: fix build regression +- platform/x86: asus-wmi: Fix thermal profile initialization +- ALSA: usb-audio: Update UMP group attributes for GTB blocks, too +- usb: dwc3: ep0: Don't reset resource alloc flag +- drm/amdgpu: rework resume handling for display (v2) +- ASoC: Intel: avs: Fix return status of avs_pcm_hw_constraints_init() +- ocfs2: Revert "ocfs2: fix the la space leak when unmounting an ocfs2 volume" +- btrfs: add cancellation points to trim loops +- iio: invensense: fix multiple odr switch when FIFO is off +- xhci: dbc: Fix STALL transfer event handling +- mm/damon/vaddr: fix issue in damon_va_evenly_split_region() +- KVM: x86/mmu: Ensure that kvm_release_pfn_clean() takes exact pfn from kvm_faultin_pfn() +- vfio/mlx5: Align the page tracking max message size with the device capability +- x86: Fix build regression with CONFIG_KEXEC_JUMP enabled +- tracing/eprobe: Fix to release eprobe when failed to add dyn_event +- clk: en7523: Initialize num before accessing hws in en7523_register_clocks() +- btrfs: fix missing snapshot drew unlock when root is dead during swap activation +- setlocalversion: work around "git describe" performance +- smb: client: don't try following DFS links in cifs_tree_connect() +- serial: 8250_dw: Add Sophgo SG2044 quirk +- rtc: cmos: avoid taking rtc_lock for extended period of time +- misc: eeprom: eeprom_93cx6: Add quirk for extra read clock cycle +- powerpc/prom_init: Fixup missing powermac #size-cells +- ASoC: amd: yc: Add quirk for microphone on Lenovo Thinkpad T14s Gen 6 21M1CTO1WW +- MIPS: Loongson64: DTS: Really fix PCIe port nodes for ls7a +- drm/amdgpu/vcn: reset fw_shared when VCPU buffers corrupted on vcn v4.0.3 +- ASoC: amd: yc: fix internal mic on Redmi G 2022 +- iio: light: ltr501: Add LTER0303 to the supported devices +- usb: chipidea: udc: handle USB Error Interrupt if IOC not set +- fs/ntfs3: Fix case when unmarked clusters intersect with zone +- ACPI: x86: Clean up Asus entries in acpi_quirk_skip_dmi_ids[] +- ACPI: x86: Add skip i2c clients quirk for Acer Iconia One 8 A1-840 +- f2fs: fix to shrink read extent node in batches +- f2fs: print message if fscorrupted was found in f2fs_new_node_page() +- PCI: Add ACS quirk for Wangxun FF5xxx NICs +- PCI: Add 'reset_subordinate' to reset hierarchy below bridge +- PCI: Detect and trust built-in Thunderbolt chips +- PCI: vmd: Set devices to D0 before enabling PM L1 Substates +- PCI: vmd: Add DID 8086:B06F and 8086:B60B for Intel client SKUs +- PCI: qcom: Add support for IPQ9574 +- KMSAN: uninit-value in inode_go_dump (5) +- verification/dot2: Improve dot parser robustness +- smb: client: memcpy() with surrounding object base address +- nvdimm: rectify the illogical code within nd_dax_probe() +- thermal/drivers/qcom/tsens-v1: Add support for MSM8937 tsens +- pinctrl: qcom: spmi-mpp: Add PM8937 compatible +- pinctrl: qcom-pmic-gpio: add support for PM8937 +- scsi: st: Add MTIOCGET and MTLOAD to ioctls allowed after device reset +- scsi: st: Don't modify unknown block number in MTIOCGET +- leds: class: Protect brightness_show() with led_cdev->led_access mutex +- clk: qcom: tcsrcc-sm8550: add SAR2130P support +- clk: qcom: rpmh: add support for SAR2130P +- clk: qcom: rcg2: add clk_rcg2_shared_floor_ops +- scsi: lpfc: Call lpfc_sli4_queue_unset() in restart and rmmod paths +- scsi: ufs: core: Make DMA mask configuration more flexible +- rtla/timerlat: Make timerlat_top_cpu->*_count unsigned long long +- tools/rtla: fix collision with glibc sched_attr/sched_set_attr +- tracing: Use atomic64_inc_return() in trace_clock_counter() +- tracing/ftrace: disable preemption in syscall probe +- pinctrl: freescale: fix COMPILE_TEST error with PINCTRL_IMX_SCU +- netpoll: Use rcu_access_pointer() in __netpoll_setup +- net/neighbor: clear error in case strict check is not set +- rocker: fix link status detection in rocker_carrier_init() +- ASoC: hdmi-codec: reorder channel allocation list +- Bluetooth: Set quirks for ATS2851 +- Bluetooth: Support new quirks for ATS2851 +- Bluetooth: Add new quirks for ATS2851 +- Bluetooth: btusb: Add RTL8852BE device 0489:e123 to device tables +- dsa: qca8k: Use nested lock to avoid splat +- wifi: brcmfmac: Fix oops due to NULL pointer dereference in brcmf_sdiod_sglist_rw() +- wifi: ipw2x00: libipw_rx_any(): fix bad alignment +- ALSA: usb-audio: Make mic volume workarounds globally applicable +- virtio-net: fix overflow inside virtnet_rq_alloc +- drm/amdgpu: skip amdgpu_device_cache_pci_state under sriov +- fsl/fman: Validate cell-index value obtained from Device Tree +- dma-debug: fix a possible deadlock on radix_lock +- drm/amdgpu: refine error handling in amdgpu_ttm_tt_pin_userptr +- netlink: specs: Add missing bitset attrs to ethtool spec +- drm/amdgpu: Dereference the ATCS ACPI buffer +- drm/amdgpu: clear RB_OVERFLOW bit when enabling interrupts for vega20_ih +- drm/sched: memset() 'job' in drm_sched_job_init() +- net: stmmac: Programming sequence for VLAN packets with split header +- net: sfp: change quirks for Alcatel Lucent G-010S-P +- drm/panel: simple: Add Microchip AC69T88A LVDS Display panel +- fanotify: allow reporting errors on failure to open fd +- wifi: ath5k: add PCI ID for Arcadyan devices +- wifi: ath5k: add PCI ID for SX76X +- net: inet6: do not leave a dangling sk pointer in inet6_create() +- net: inet: do not leave a dangling sk pointer in inet_create() +- net: ieee802154: do not leave a dangling sk pointer in ieee802154_create() +- net: af_can: do not leave a dangling sk pointer in can_create() +- af_packet: avoid erroring out after sock_init_data() in packet_create() +- net/sched: cbs: Fix integer overflow in cbs_set_port_rate() +- net: ethernet: fs_enet: Use %pa to format resource_size_t +- net: fec_mpc52xx_phy: Use %pa to format resource_size_t +- samples/bpf: Fix a resource leak +- r8169: don't apply UDP padding quirk on RTL8126A +- drm/display: Fix building with GCC 15 +- dlm: fix possible lkb_resource null dereference +- drm/radeon/r600_cs: Fix possible int overflow in r600_packet3_check() +- drm/mcde: Enable module autoloading +- drm/bridge: it6505: Enable module autoloading +- drm: panel-orientation-quirks: Add quirk for AYA NEO GEEK +- drm: panel-orientation-quirks: Add quirk for AYA NEO Founder edition +- drm: panel-orientation-quirks: Add quirk for AYA NEO 2 model +- drm/vc4: hvs: Set AXI panic modes for the HVS +- drm/vc4: hdmi: Avoid log spam for audio start failure +- HID: magicmouse: Apple Magic Trackpad 2 USB-C driver support +- selftests/resctrl: Protect against array overflow when reading strings +- regmap: maple: Provide lockdep (sub)class for maple tree's internal lock +- mmc: sdhci-esdhc-imx: enable quirks SDHCI_QUIRK_NO_LED +- mmc: core: Add SD card quirk for broken poweroff notification +- media: cx231xx: Add support for Dexatek USB Video Grabber 1d19:6108 +- media: uvcvideo: Add a quirk for the Kaiweets KTI-W02 infrared camera +- media: uvcvideo: RealSense D421 Depth module metadata +- HID: add per device quirk to force bind to hid-generic +- spi: spi-fsl-lpspi: Adjust type of scldiv +- perf/x86/amd: Warn only on new bits set +- ACPI: x86: Make UART skip quirks work on PCI UARTs without an UID +- hwmon: (nct6775) Add 665-ACE/600M-CL to ASUS WMI monitoring list +- kselftest/arm64: Don't leak pipe fds in pac.exec_sign_all() +- btrfs: do not clear read-only when adding sprout device +- btrfs: avoid unnecessary device path update for the same device +- btrfs: don't take dev_replace rwsem on task already holding it +- kselftest/arm64: Log fp-stress child startup errors to stdout +- epoll: annotate racy check +- x86/mm: Add _PAGE_NOPTISHADOW bit to avoid updating userspace page tables +- kasan: make report_lock a raw spinlock +- lib: stackinit: hide never-taken branch from compiler +- ocfs2: update seq_file index in ocfs2_dlm_seq_next +- mmc: core: Further prevent card detect during shutdown +- mmc: sdhci-pci: Add DMI quirk for missing CD GPIO on Vexia Edu Atla 10 tablet +- regmap: detach regmap from dev on regmap_exit +- dma-fence: Use kernel's sort for merging fences +- dma-fence: Fix reference leak on fence merge failure path +- dma-buf: fix dma_fence_array_signaled v4 +- x86/kexec: Restore GDT on return from ::preserve_context kexec +- modpost: Add .irqentry.text to OTHER_SECTIONS +- drm/amdgpu/hdp5.2: do a posting read when flushing HDP +- drm/dp_mst: Fix resetting msg rx state after topology removal +- drm/dp_mst: Verify request type in the corresponding down message reply +- drm/dp_mst: Fix MST sideband message body length check +- bcache: revert replacing IS_ERR_OR_NULL with IS_ERR again +- smb3.1.1: fix posix mounts to older servers +- fs/smb/client: cifs_prime_dcache() for SMB3 POSIX reparse points +- fs/smb/client: Implement new SMB3 POSIX type +- fs/smb/client: avoid querying SMB2_OP_QUERY_WSL_EA for SMB3 POSIX +- scsi: ufs: core: Add missing post notify for power mode change +- scsi: qla2xxx: Remove check req_sg_cnt should be equal to rsp_sg_cnt +- scsi: qla2xxx: Supported speed displayed incorrectly for VPorts +- scsi: qla2xxx: Fix NVMe and NPIV connect issue +- scsi: qla2xxx: Fix abort in bsg timeout +- ALSA: hda/realtek: Add support for Samsung Galaxy Book3 360 (NP730QFG) +- ALSA: hda/realtek: Enable mute and micmute LED on HP ProBook 430 G8 +- ALSA: usb-audio: add mixer mapping for Corsair HS80 +- arm64: Ensure bits ASID[15:8] are masked out when the kernel uses 8-bit ASIDs +- tracing: Fix cmp_entries_dup() to respect sort() comparison rules +- can: mcp251xfd: mcp251xfd_get_tef_len(): work around erratum DS80000789E 6. +- can: dev: can_set_termination(): allow sleeping GPIOs +- watchdog: rti: of: honor timeout-sec property +- LoongArch: Add architecture specific huge_pte_clear() +- x86/CPU/AMD: WARN when setting EFER.AUTOIBRS if and only if the WRMSR fails +- bpf: Fix exact match conditions in trie_get_next_key() +- iio: magnetometer: yas530: use signed integer type for clamp limits +- scatterlist: fix incorrect func name in kernel-doc +- drm/v3d: Enable Performance Counters before clearing them +- ASoC: mediatek: mt8188-mt6359: Remove hardcoded dmic codec +- scsi: scsi_debug: Fix hrtimer support for ndelay +- ASoC: SOF: ipc3-topology: fix resource leaks in sof_ipc3_widget_setup_comp_dai() +- ASoC: SOF: ipc3-topology: Convert the topology pin index to ALH dai index +- tools: Override makefile ARCH variable if defined, but empty +- ALSA: usb-audio: Notify xrun for low-latency mode +- ALSA: seq: ump: Fix seq port updates per FB info notify +- ALSA: ump: Update substream name from assigned FB names +- ALSA: seq: ump: Use automatic cleanup of kfree() +- drm/sti: Add __iomem for mixer_dbg_mxn's parameter +- bpftool: fix potential NULL pointer dereferencing in prog_dump() +- xsk: always clear DMA mapping information when unmapping the pool +- bpf, vsock: Invoke proto::close on close() +- bpf, vsock: Fix poll() missing a queue +- scsi: ufs: core: Add ufshcd_send_bsg_uic_cmd() for UFS BSG +- scsi: ufs: core: Always initialize the UIC done completion +- drm/bridge: it6505: Fix inverted reset polarity +- drm/bridge: it6505: update usleep_range for RC circuit charge time +- i3c: master: Fix dynamic address leak when 'assigned-address' is present +- i3c: master: Extend address status bit to 4 and add I3C_ADDR_SLOT_EXT_DESIRED +- i3c: master: Replace hard code 2 with macro I3C_ADDR_SLOT_STATUS_BITS +- i3c: master: svc: Modify enabled_events bit 7:0 to act as IBI enable counter +- i3c: master: svc: use slow speed for first broadcast address +- i3c: master: support to adjust first broadcast address speed +- i3c: master: fix kernel-doc check warning +- i3c: master: svc: add hot join support +- i3c: master: add enable(disable) hot join in sys entry +- f2fs: fix to drop all discards after creating snapshot on lvm device +- driver core: fw_devlink: Stop trying to optimize cycle detection logic +- driver core: Add FWLINK_FLAG_IGNORE to completely ignore a fwnode link +- driver core: fw_devlink: Improve logs for cycle detection +- ntp: Remove invalid cast in time offset math +- powerpc/vdso: Drop -mstack-protector-guard flags in 32-bit files with clang +- powerpc/vdso: Refactor CFLAGS for CVDSO build +- mmc: mtk-sd: Fix MMC_CAP2_CRYPTO flag setting +- mmc: mtk-sd: fix devm_clk_get_optional usage +- mmc: mtk-sd: Fix error handle of probe function +- mmc: mtk-sd: use devm_mmc_alloc_host +- usb: dwc3: ep0: Don't clear ep0 DWC3_EP_TRANSFER_STARTED +- usb: dwc3: ep0: Don't reset resource alloc flag (including ep0) +- usb: dwc3: gadget: Rewrite endpoint allocation flow +- soc: fsl: cpm1: qmc: Set the ret error code on platform_get_irq() failure +- soc: fsl: cpm1: qmc: Introduce qmc_{init,exit}_xcc() and their CPM1 version +- soc: fsl: cpm1: qmc: Introduce qmc_init_resource() and its CPM1 version +- soc: fsl: cpm1: qmc: Re-order probe() operations +- soc: fsl: cpm1: qmc: Fix blank line and spaces +- soc/fsl: cpm: qmc: Convert to platform remove callback returning void +- serial: amba-pl011: Fix RX stall when DMA is used +- serial: amba-pl011: Use port lock wrappers +- xhci: Fix control transfer error on Etron xHCI host +- xhci: Don't issue Reset Device command to Etron xHCI host +- xhci: Combine two if statements for Etron xHCI host +- xhci: remove XHCI_TRUST_TX_LENGTH quirk +- xhci: Allow RPM on the USB controller (1022:43f7) by default +- net: avoid potential UAF in default_operstate() +- net/mlx5e: Remove workaround to avoid syndrome for internal port +- geneve: do not assume mac header is set in geneve_xmit_skb() +- mlxsw: spectrum_acl_flex_keys: Use correct key block on Spectrum-4 +- mlxsw: spectrum_acl_flex_keys: Constify struct mlxsw_afk_element_inst +- mlxsw: Mark high entropy key blocks +- mlxsw: Edit IPv6 key blocks to use one less block for multicast forwarding +- mlxsw: spectrum_acl_flex_keys: Add 'ipv4_5b' flex key +- mlxsw: Add 'ipv4_5' flex key +- ethtool: Fix wrong mod state in case of verbose and no_mask bitset +- netfilter: nft_set_hash: skip duplicated elements pending gc run +- netfilter: ipset: Hold module reference while requesting a module +- net: sched: fix erspan_opt settings in cls_flower +- ixgbe: downgrade logging of unsupported VF API version to debug +- ixgbevf: stop attempting IPSEC offload on Mailbox API 1.5 +- net/qed: allow old cards not supporting "num_images" to work +- tipc: Fix use-after-free of kernel socket in cleanup_bearer(). +- dccp: Fix memory leak in dccp_feat_change_recv +- net/ipv6: release expired exception dst cached in socket +- ipv6: introduce dst_rt6_info() helper +- net-timestamp: make sk_tskey more predictable in error path +- platform/x86: asus-wmi: Ignore return value when writing thermal policy +- platform/x86: asus-wmi: Fix inconsistent use of thermal policies +- platform/x86: asus-wmi: add support for vivobook fan profiles +- can: j1939: j1939_session_new(): fix skb reference counting +- net: hsr: avoid potential out-of-bound access in fill_frame_info() +- net/sched: tbf: correct backlog statistic for GSO packets +- ptp: Add error handling for adjfine callback in ptp_clock_adjtime +- selftests: hid: fix typo and exit code +- netfilter: nft_socket: remove WARN_ON_ONCE on maximum cgroup level +- netfilter: x_tables: fix LED ID check in led_tg_check() +- ipvs: fix UB due to uninitialized stack access in ip_vs_protocol_init() +- can: f81604: f81604_handle_can_bus_errors(): fix {rx,tx}_errors statistics +- can: ems_usb: ems_usb_rx_err(): fix {rx,tx}_errors statistics +- can: sun4i_can: sun4i_can_err(): fix {rx,tx}_errors statistics +- can: sja1000: sja1000_err(): fix {rx,tx}_errors statistics +- can: hi311x: hi3110_can_ist(): fix {rx,tx}_errors statistics +- can: ifi_canfd: ifi_canfd_handle_lec_err(): fix {rx,tx}_errors statistics +- can: m_can: m_can_handle_lec_err(): fix {rx,tx}_errors statistics +- can: hi311x: hi3110_can_ist(): fix potential use-after-free +- can: sun4i_can: sun4i_can_err(): call can_change_state() even if cf is NULL +- can: c_can: c_can_handle_bus_err(): update statistics if skb allocation fails +- can: gs_usb: add usb endpoint address detection at driver probe step +- can: gs_usb: add VID/PID for Xylanta SAINT3 product family +- watchdog: mediatek: Make sure system reset gets asserted in mtk_wdt_restart() +- watchdog: apple: Actually flush writes after requesting watchdog restart +- watchdog: xilinx_wwdt: Calculate max_hw_heartbeat_ms using clock frequency +- iTCO_wdt: mask NMI_NOW bit for update_no_reboot_bit() call +- openrisc: Use asm-generic's version of fix_to_virt() & virt_to_fix() +- Revert "x86/pvh: Set phys_base when calling xen_prepare_pvh()" +- Revert "x86/pvh: Call C code via the kernel virtual mapping" +- !15069 fs/proc: fix softlockup in __read_vmcore (part 2) +- fs/proc: fix softlockup in __read_vmcore (part 2) +- !15067 irqchip/gic-v3-its: Don't enable interrupts in its_irq_set_vcpu_affinity() +- irqchip/gic-v3-its: Don't enable interrupts in its_irq_set_vcpu_affinity() +- !15065 fix CVE-2025-21668 +- pmdomain: imx8mp-blk-ctrl: add missing loop break condition +- !15056 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.63-6.6.64 LTS Patches +- posix-timers: Target group sigqueue to current task only if not exiting +- drm/amd/pm: update current_socclk and current_uclk in gpu_metrics on smu v13.0.7 +- drm/amdgpu: fix usage slab after free +- drm/amdkfd: Use the correct wptr size +- drm: xlnx: zynqmp_dpsub: fix hotplug detection +- drm/etnaviv: flush shader L1 cache after user commandstream +- drm/mediatek: Fix child node refcount handling in early exit +- drm/sti: avoid potential dereference of error pointers in sti_hqvdp_atomic_check +- udmabuf: use vmf_insert_pfn and VM_PFNMAP for handling mmap +- powerpc: Adjust adding stack protector flags to KBUILD_CLAGS for clang +- powerpc: Fix stack protector Kconfig test for clang +- iio: gts: fix infinite loop for gain_to_scaletables() +- iio: Fix fwnode_handle in __fwnode_iio_channel_get_by_name() +- iio: accel: kx022a: Fix raw read format +- nfsd: make sure exp active before svc_export_show +- PCI: rockchip-ep: Fix address translation unit programming +- PCI: of_property: Assign PCI instead of CPU bus address to dynamic PCI nodes +- dm thin: Add missing destroy_work_on_stack() +- dm: Fix typo in error message +- ovl: properly handle large files in ovl_security_fileattr +- leds: flash: mt6360: Fix device_for_each_child_node() refcounting in error paths +- thermal: int3400: Fix reading of current_uuid for active policy +- fs/proc/kcore.c: Clear ret value in read_kcore_iter after successful iov_iter_zero +- PCI: endpoint: Clear secondary (not primary) EPC in pci_epc_remove_epf() +- PCI: keystone: Add link up check to ks_pcie_other_map_bus() +- PCI: keystone: Set mode as Root Complex for "ti,keystone-pcie" compatible +- i3c: master: svc: Fix pm_runtime_set_suspended() with runtime pm enabled +- scsi: ufs: exynos: Fix hibern8 notify callbacks +- util_macros.h: fix/rework find_closest() macros +- ceph: extract entity name from device id +- mm/slub: Avoid list corruption when removing a slab from the full list +- ARM: 9431/1: mm: Pair atomic_set_release() with _read_acquire() +- ARM: 9430/1: entry: Do a dummy read from VMAP shadow +- s390/entry: Mark IRQ entries to fix stack depot warnings +- ARM: 9429/1: ioremap: Sync PGDs for VMALLOC shadow +- clk: qcom: gcc-qcs404: fix initial rate of GPLL3 +- leds: lp55xx: Remove redundant test for invalid channel number +- iommu/io-pgtable-arm: Fix stage-2 map/unmap for concatenated tables +- zram: clear IDLE flag after recompression +- vmstat: call fold_vm_zone_numa_events() before show per zone NUMA event +- maple_tree: refine mas_store_root() on storing NULL +- media: uvcvideo: Require entities to have a non-zero unique ID +- media: uvcvideo: Stop stream during unregister +- media: gspca: ov534-ov772x: Fix off-by-one error in set_frame_rate() +- media: venus: Fix pm_runtime_set_suspended() with runtime pm enabled +- media: amphion: Fix pm_runtime_set_suspended() with runtime pm enabled +- efi/libstub: Free correct pointer on failure +- media: platform: exynos4-is: Fix an OF node reference leak in fimc_md_is_isp_available +- media: ts2020: fix null-ptr-deref in ts2020_probe() +- media: verisilicon: av1: Fix reference video buffer pointer assignment +- media: i2c: tc358743: Fix crash in the probe error path when using polling +- arm64: dts: freescale: imx8mp-verdin: Fix SD regulator startup delay +- media: i2c: dw9768: Fix pm_runtime_set_suspended() with runtime pm enabled +- media: mtk-jpeg: Fix null-ptr-deref during unload module +- media: amphion: Set video drvdata before register video device +- arm64: dts: ti: k3-am62-verdin: Fix SD regulator startup delay +- arm64: dts: freescale: imx8mm-verdin: Fix SD regulator startup delay +- arm64: dts: allwinner: pinephone: Add mount matrix to accelerometer +- md/md-bitmap: Add missing destroy_work_on_stack() +- btrfs: add a sanity check for btrfs root in btrfs_search_slot() +- btrfs: don't loop for nowait writes when checking for cross references +- quota: flush quota_release_work upon quota writeback +- xfs: remove unknown compat feature check in superblock write validation +- SUNRPC: timeout and cancel TLS handshake with -ETIMEDOUT +- cifs: unlock on error in smb3_reconfigure() +- cifs: during remount, make sure passwords are in sync +- modpost: remove incorrect code in do_eisa_entry() +- Rename .data.once to .data..once to fix resetting WARN*_ONCE +- Rename .data.unlikely to .data..unlikely +- init/modpost: conditionally check section mismatch to __meminit* +- modpost: squash ALL_{INIT,EXIT}_TEXT_SECTIONS to ALL_TEXT_SECTIONS +- modpost: use ALL_INIT_SECTIONS for the section check from DATA_SECTIONS +- modpost: disallow the combination of EXPORT_SYMBOL and __meminit* +- modpost: remove EXIT_SECTIONS macro +- modpost: remove MEM_INIT_SECTIONS macro +- modpost: disallow *driver to reference .meminit* sections +- modpost: remove ALL_EXIT_DATA_SECTIONS macro +- rtc: ab-eoz9: don't fail temperature reads on undervoltage notification +- cifs: Fix parsing reparse point with native symlink in SMB1 non-UNICODE session +- cifs: Fix parsing native symlinks relative to the export +- smb: client: disable directory caching when dir_cache_timeout is zero +- perf/arm-cmn: Ensure port and device id bits are set properly +- perf/arm-smmuv3: Fix lockdep assert in ->event_init() +- 9p/xen: fix release of IRQ +- 9p/xen: fix init sequence +- block: return unsigned int from bdev_io_min +- nvme/multipath: Fix RCU list traversal to use SRCU primitive +- nvme-multipath: avoid hang on inaccessible namespaces +- nvme-multipath: implement "queue-depth" iopolicy +- nvme-multipath: prepare for "queue-depth" iopolicy +- rtc: rzn1: fix BCD to rtc_time conversion errors +- jffs2: fix use of uninitialized variable +- ubi: fastmap: Fix duplicate slab cache names while attaching +- ubifs: Correct the total block count by deducting journal reservation +- ubi: fastmap: wl: Schedule fm_work if wear-leveling pool is empty +- rtc: abx80x: Fix WDT bit position of the status register +- rtc: st-lpc: Use IRQF_NO_AUTOEN flag in request_irq() +- um: Always dump trace for specified task in show_stack +- um: Fix the return value of elf_core_copy_task_fpregs +- SUNRPC: make sure cache entry active before cache_show +- ipc: fix memleak if msg_init_ns failed in create_ipc_ns +- lib: string_helpers: silence snprintf() output truncation warning +- ublk: fix error code for unsupported command +- usb: dwc3: gadget: Fix checking for number of TRBs left +- smb: During unmount, ensure all cached dir instances drop their dentry +- smb: prevent use-after-free due to open_cached_dir error paths +- smb: Don't leak cfid when reconnect races with open_cached_dir +- smb: client: handle max length for SMB symlinks +- smb3: request handle caching when caching directories +- ALSA: hda/realtek: Apply quirk for Medion E15433 +- ALSA: hda/realtek: Fix Internal Speaker and Mic boost of Infinix Y4 Max +- ALSA: hda/realtek: Set PCBeep to default value for ALC274 +- ALSA: hda/realtek: Update ALC225 depop procedure +- ALSA: ump: Fix evaluation of MIDI 1.0 FB info +- media: v4l2-core: v4l2-dv-timings: check cvt/gtf result +- soc: fsl: rcpm: fix missing of_node_put() in copy_ippdexpcr1_setting() +- HID: wacom: Interpret tilt data from Intuos Pro BT as signed values +- blk-mq: Make blk_mq_quiesce_tagset() hold the tag list mutex less long +- block: fix ordering between checking BLK_MQ_S_STOPPED request adding +- block: fix ordering between checking QUEUE_FLAG_QUIESCED request adding +- block: fix missing dispatching request when queue is started or unquiesced +- arm64: tls: Fix context-switching of tpidrro_el0 when kpti is enabled +- ublk: fix ublk_ch_mmap() for 64K page size +- iio: gts: Fix uninitialized symbol 'ret' +- um: vector: Do not use drvdata in release +- serial: 8250: omap: Move pm_runtime_get_sync +- serial: 8250_fintek: Add support for F81216E +- dt-bindings: serial: rs485: Fix rs485-rts-delay property +- um: net: Do not use drvdata in release +- um: ubd: Do not use drvdata in release +- ubi: wl: Put source PEB into correct list if trying locking LEB failed +- platform/chrome: cros_ec_typec: fix missing fwnode reference decrement +- smb: client: fix NULL ptr deref in crypto_aead_setkey() +- parisc/ftrace: Fix function graph tracing disablement +- cifs: support mounting with alternate password to allow password rotation +- cpufreq: mediatek-hw: Fix wrong return value in mtk_cpufreq_get_cpu_power() +- mtd: spi-nor: core: replace dummy buswidth from addr to data +- spi: Fix acpi deferred irq probe +- netfilter: ipset: add missing range check in bitmap_ip_uadt +- gpio: exar: set value when external pull-up or pull-down is present +- Revert "serial: sh-sci: Clean sci_ports[0] after at earlycon exit" +- serial: sh-sci: Clean sci_ports[0] after at earlycon exit +- Revert "usb: gadget: composite: fix OS descriptors w_value logic" +- wifi: brcmfmac: release 'root' node in all execution paths +- wifi: ath12k: fix crash when unbinding +- ARM: dts: omap36xx: declare 1GHz OPP as turbo again +- usb: xhci: Fix TD invalidation under pending Set TR Dequeue +- Compiler Attributes: disable __counted_by for clang < 19.1.3 +- Bluetooth: Fix type of len in rfcomm_sock_getsockopt{,_old}() +- exfat: fix uninit-value in __exfat_get_dentry_set +- dt-bindings: iio: dac: ad3552r: fix maximum spi speed +- pinctrl: qcom: spmi: fix debugfs drive strength +- tools/nolibc: s390: include std.h +- locking/lockdep: Avoid creating new name string literals in lockdep_set_subclass() +- tty: ldsic: fix tty_ldisc_autoload sysctl's proc_handler +- comedi: Flush partial mappings in error case +- fsnotify: fix sending inotify event with unexpected filename +- KVM: arm64: Ignore PMCNTENSET_EL0 while checking for overflow status +- KVM: arm64: vgic-v3: Sanitise guest writes to GICR_INVLPIR +- powerpc/pseries: Fix KVM guest detection for disabling hardlockup detector +- KVM: x86/mmu: Skip the "try unsync" path iff the old SPTE was a leaf SPTE +- crypto: x86/aegis128 - access 32-bit arguments as 32-bit +- perf/x86/intel/pt: Fix buffer full but size is 0 case +- ASoC: codecs: Fix atomicity violation in snd_soc_component_get_drvdata() +- ASoC: amd: yc: Add a quirk for microfone on Lenovo ThinkPad P14s Gen 5 21MES00B00 +- jfs: xattr: check invalid xattr size more strictly +- ext4: fix FS_IOC_GETFSMAP handling +- ext4: supress data-race warnings in ext4_free_inodes_{count,set}() +- soc: qcom: socinfo: fix revision check in qcom_socinfo_probe() +- ASoC: Intel: sst: Fix used of uninitialized ctx to log an error +- arm64: dts: mediatek: mt8195-cherry: Mark USB 3.0 on xhci1 as disabled +- Revert "arm64: dts: mediatek: mt8195-cherry: Mark USB 3.0 on xhci1 as disabled" +- btrfs: do not BUG_ON() when freeing tree block after error +- dm-bufio: fix warnings about duplicate slab caches +- dm-cache: fix warnings about duplicate slab caches +- closures: Change BUG_ON() to WARN_ON() +- usb: ehci-spear: fix call balance of sehci clk handling routines +- xen: Fix the issue of resource not being properly released in xenbus_dev_probe() +- nvme: fix metadata handling in nvme-passthrough +- mailbox: mtk-cmdq: Move devm_mbox_controller_register() after devm_pm_runtime_enable() +- f2fs: fix fiemap failure issue when page size is 16KB +- gfs2: Don't set GLF_LOCK in gfs2_dispose_glock_lru +- ASoC: amd: yc: Fix for enabling DMIC on acp6x via _DSD entry +- apparmor: fix 'Do simple duplicate message elimination' +- drm/amd/display: Fix null check for pipe_ctx->plane_state in hwss_setup_dpp +- drm/radeon: Fix spurious unplug event on radeon HDMI +- drm/radeon: change rdev->ddev to rdev_to_drm(rdev) +- drm/radeon: add helper rdev_to_drm(rdev) +- ALSA: hda/realtek: Update ALC256 depop procedure +- counter: ti-ecap-capture: Add check for clk_enable() +- counter: stm32-timer-cnt: Add check for clk_enable() +- misc: apds990x: Fix missing pm_runtime_disable() +- USB: chaoskey: Fix possible deadlock chaoskey_list_lock +- USB: chaoskey: fail open after removal +- usb: yurex: make waiting on yurex_write interruptible +- usb: using mutex lock and supporting O_NONBLOCK flag in iowarrior_read() +- iio: light: al3010: Fix an error handling path in al3010_probe() +- ipmr: fix tables suspicious RCU usage +- ip6mr: fix tables suspicious RCU usage +- rxrpc: Improve setsockopt() handling of malformed user input +- llc: Improve setsockopt() handling of malformed user input +- bnxt_en: Unregister PTP during PCI shutdown and suspend +- bnxt_en: Refactor bnxt_ptp_init() +- bnxt_en: Reserve rings after PCIe AER recovery if NIC interface is down +- net: hsr: fix hsr_init_sk() vs network/transport headers. +- spi: atmel-quadspi: Fix register name in verbose logging function +- octeontx2-af: Quiesce traffic before NIX block reset +- octeontx2-af: RPM: fix stale FCFEC counters +- octeontx2-af: RPM: fix stale RSFEC counters +- octeontx2-pf: Reset MAC stats during probe +- octeontx2-af: RPM: Fix low network performance +- octeontx2-af: RPM: Fix mismatch in lmac type +- net: stmmac: dwmac-socfpga: Set RX watchdog interrupt as broken +- marvell: pxa168_eth: fix call balance of pep->clk handling routines +- net: mdio-ipq4019: add missing error check +- net/ipv6: delete temporary address if mngtmpaddr is removed or unmanaged +- s390/iucv: MSG_PEEK causes memory leak in iucv_sock_destruct() +- net: microchip: vcap: Add typegroup table terminators in kunit tests +- net: usb: lan78xx: Fix refcounting and autosuspend on invalid WoL configuration +- tg3: Set coherent DMA mask bits to 31 for BCM57766 chipsets +- net: usb: lan78xx: Fix memory leak on device unplug by freeing PHY device +- power: supply: rt9471: Use IC status regfield to report real charger status +- power: supply: rt9471: Fix wrong WDT function regfield declaration +- power: supply: bq27xxx: Fix registers of bq27426 +- power: supply: core: Remove might_sleep() from power_supply_put() +- LoongArch: BPF: Sign-extend return values +- LoongArch: Fix build failure with GCC 15 (-std=gnu23) +- fs_parser: update mount_api doc to match function signature +- vfio/pci: Properly hide first-in-list PCIe extended capability +- gpio: zevio: Add missed label initialisation +- selftests/mount_setattr: Fix failures on 64K PAGE_SIZE kernels +- vdpa/mlx5: Fix suboptimal range on iotlb iteration +- hwmon: (tps23861) Fix reporting of negative temperatures +- NFSD: Fix nfsd4_shutdown_copy() +- nfsd: release svc_expkey/svc_export with rcu_work +- NFSD: Cap the number of bytes copied by nfs4_reset_recoverydir() +- NFSD: Prevent NULL dereference in nfsd4_process_cb_update() +- PCI: endpoint: epf-mhi: Avoid NULL dereference if DT lacks 'mmio' +- remoteproc: qcom_q6v5_mss: Re-order writes to the IMEM region +- rpmsg: glink: use only lower 16-bits of param2 for CMD_OPEN name length +- remoteproc: qcom: pas: add minidump_id to SM8350 resources +- perf trace: Avoid garbage when not printing a syscall's arguments +- perf trace: Do not lose last events in a race +- perf trace: Fix tracing itself, creating feedback loops +- perf list: Fix topic and pmu_name argument order +- m68k: coldfire/device.c: only build FEC when HW macros are defined +- m68k: mcfgpio: Fix incorrect register offset for CONFIG_M5441x +- perf trace: avoid garbage when not printing a trace event's arguments +- f2fs: fix to avoid forcing direct write to use buffered IO on inline_data inode +- PCI: j721e: Deassert PERST# after a delay of PCIE_T_PVPERL_MS milliseconds +- PCI: j721e: Add suspend and resume support +- PCI: j721e: Use T_PERST_CLK_US macro +- PCI: j721e: Add reset GPIO to struct j721e_pcie +- PCI: cadence: Set cdns_pcie_host_init() global +- PCI: cadence: Extract link setup sequence from cdns_pcie_host_setup() +- PCI: j721e: Add PCIe 4x lane selection support +- PCI: j721e: Add per platform maximum lane settings +- PCI: Add T_PVPERL macro +- f2fs: fix to avoid use GC_AT when setting gc_mode as GC_URGENT_LOW or GC_URGENT_MID +- f2fs: check curseg->inited before write_sum_page in change_curseg +- f2fs: fix the wrong f2fs_bug_on condition in f2fs_do_replace_block +- perf ftrace latency: Fix unit on histogram first entry when using --use-nsec +- PCI: cpqphp: Fix PCIBIOS_* return value confusion +- PCI: cpqphp: Use PCI_POSSIBLE_ERROR() to check config reads +- rust: macros: fix documentation of the paste! macro +- perf probe: Correct demangled symbols in C++ program +- perf probe: Fix libdw memory leak +- f2fs: fix to account dirty data in __get_secs_required() +- f2fs: compress: fix inconsistent update of i_blocks in release_compress_blocks and reserve_compress_blocks +- perf test attr: Add back missing topdown events +- perf trace: Keep exited threads for summary +- perf stat: Fix affinity memory leaks on error path +- perf stat: Close cork_fd when create_perf_stat_counter() failed +- gfs2: Allow immediate GLF_VERIFY_DELETE work +- gfs2: Rename GLF_VERIFY_EVICT to GLF_VERIFY_DELETE +- gfs2: Get rid of gfs2_glock_queue_put in signal_our_withdraw +- perf cs-etm: Don't flush when packet_queue fills up +- mailbox: arm_mhuv2: clean up loop in get_irq_chan_comb() +- smb: cached directories can be more than root file handle +- pinctrl: k210: Undef K210_PC_DEFAULT +- arm64: dts: qcom: sc8180x: Add a SoC-specific compatible to cpufreq-hw +- clk: clk-axi-clkgen: make sure to enable the AXI bus clock +- dt-bindings: clock: axi-clkgen: include AXI clk +- fbdev: sh7760fb: Fix a possible memory leak in sh7760fb_alloc_mem() +- powerpc/kexec: Fix return of uninitialized variable +- powerpc/sstep: make emulate_vsx_load and emulate_vsx_store static +- KVM: PPC: Book3S HV: Avoid returning to nested hypervisor on pending doorbells +- KVM: PPC: Book3S HV: Stop using vc->dpdes for nested KVM guests +- dax: delete a stale directory pmem +- cpufreq: CPPC: Fix wrong return value in cppc_get_cpu_power() +- x86/tdx: Dynamically disable SEPT violations from causing #VEs +- x86/tdx: Rename tdx_parse_tdinfo() to tdx_setup() +- x86/tdx: Introduce wrappers to read and write TD metadata +- x86/tdx: Pass TDCALL/SEAMCALL input/output registers via a structure +- x86/tdx: Rename __tdx_module_call() to __tdcall() +- x86/tdx: Make macros of TDCALLs consistent with the spec +- x86/tdx: Skip saving output regs when SEAMCALL fails with VMFailInvalid +- scsi: fusion: Remove unused variable 'rc' +- fs/proc/kcore.c: fix coccinelle reported ERROR instances +- mfd: rt5033: Fix missing regmap_del_irq_chip() +- checkpatch: always parse orig_commit in fixes tag +- checkpatch: check for missing Fixes tags +- iommu/vt-d: Fix checks and print in pgtable_walk() +- iommu/vt-d: Fix checks and print in dmar_fault_dump_ptes() +- clk: imx: imx8-acm: Fix return value check in clk_imx_acm_attach_pm_domains() +- clk: imx: clk-scu: fix clk enable state save and restore +- clk: imx: fracn-gppll: fix pll power up +- clk: imx: fracn-gppll: correct PLL initialization flow +- clk: imx: lpcg-scu: SW workaround for errata (e10858) +- RISC-V: KVM: Fix APLIC in_clrip and clripnum write emulation +- RDMA/rxe: Set queue pair cur_qp_state when being queried +- clk: renesas: rzg2l: Fix FOUTPOSTDIV clk +- clk: sunxi-ng: d1: Fix PLL_AUDIO0 preset +- RDMA/bnxt_re: Check cqe flags to know imm_data vs inv_irkey +- mtd: spi-nor: spansion: Use nor->addr_nbytes in octal DTR mode in RD_ANY_REG_OP +- powerpc/mm/fault: Fix kfence page fault reporting +- mtd: rawnand: atmel: Fix possible memory leak +- mtd: hyperbus: rpc-if: Add missing MODULE_DEVICE_TABLE +- mtd: hyperbus: rpc-if: Convert to platform remove callback returning void +- cpufreq: loongson2: Unregister platform_driver on failure +- mfd: intel_soc_pmic_bxtwc: Use IRQ domain for PMIC devices +- mfd: intel_soc_pmic_bxtwc: Use IRQ domain for TMU device +- mfd: da9052-spi: Change read-mask to write-mask +- mfd: tps65010: Use IRQF_NO_AUTOEN flag in request_irq() to fix race +- powerpc/vdso: Flag VDSO64 entry points as functions +- pinctrl: zynqmp: drop excess struct member description +- trace/trace_event_perf: remove duplicate samples on the first tracepoint event +- clk: mediatek: drop two dead config options +- unicode: Fix utf8_load() error path +- bpf: fix recursive lock when verdict program return SK_PASS +- wireguard: selftests: load nf_conntrack if not present +- netpoll: Use rcu_access_pointer() in netpoll_poll_lock +- ALSA: caiaq: Use snd_card_free_when_closed() at disconnection +- ALSA: us122l: Use snd_card_free_when_closed() at disconnection +- ALSA: usx2y: Use snd_card_free_when_closed() at disconnection +- net: rfkill: gpio: Add check for clk_enable() +- bpf: Force uprobe bpf program to always return 0 +- drm/amdkfd: Fix wrong usage of INIT_WORK() +- selftests: net: really check for bg process completion +- ipv6: release nexthop on device removal +- net: use unrcu_pointer() helper +- sock_diag: allow concurrent operation in sock_diag_rcv_msg() +- sock_diag: allow concurrent operations +- sock_diag: add module pointer to "struct sock_diag_handler" +- bpf, sockmap: Fix sk_msg_reset_curr +- bpf, sockmap: Several fixes to bpf_msg_push_data +- selftests/bpf: Add push/pop checking for msg_verify_data in test_sockmap +- selftests/bpf: Fix total_bytes in msg_loop_rx in test_sockmap +- selftests/bpf: Fix SENDPAGE data logic in test_sockmap +- selftests/bpf: Add txmsg_pass to pull/push/pop in test_sockmap +- netlink: typographical error in nlmsg_type constants definition +- netfilter: nf_tables: must hold rcu read lock while iterating object type list +- netfilter: nf_tables: skip transaction if update object is not implemented +- netfilter: nf_tables: must hold rcu read lock while iterating expression type list +- netfilter: nf_tables: avoid false-positive lockdep splat on rule deletion +- netfilter: nf_tables: Introduce nf_tables_getrule_single() +- netfilter: nf_tables: Open-code audit log call in nf_tables_getrule() +- drm: use ATOMIC64_INIT() for atomic64_t +- drm/vkms: Drop unnecessary call to drm_crtc_cleanup() +- bpf, bpftool: Fix incorrect disasm pc +- drm/msm/dpu: cast crtc_clk calculation to u64 in _dpu_core_perf_calc_clk() +- wifi: wfx: Fix error handling in wfx_core_init() +- drm/etnaviv: hold GPU lock across perfmon sampling +- drm/etnaviv: Request pages from DMA32 zone on addressing_limited +- drm/msm/gpu: Check the status of registration to PM QoS +- drm/msm/adreno: Use IRQF_NO_AUTOEN flag in request_irq() +- bpf, arm64: Remove garbage frame for struct_ops trampoline +- drm/panfrost: Remove unused id_mask from struct panfrost_model +- selftests/bpf: fix test_spin_lock_fail.c's global vars usage +- octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_dcbnl.c +- octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_dmac_flt.c +- octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_flows.c +- drm/msm/dpu: drop LM_3 / LM_4 on MSM8998 +- drm/msm/dpu: drop LM_3 / LM_4 on SDM845 +- drm/msm/dpu: on SDM845 move DSPP_3 to LM_5 block +- drm: fsl-dcu: enable PIXCLK on LS1021A +- wifi: mwifiex: Fix memcpy() field-spanning write warning in mwifiex_config_scan() +- selftests/bpf: Fix txmsg_redir of test_txmsg_pull in test_sockmap +- selftests/bpf: Fix msg_verify_data in test_sockmap +- drm/bridge: tc358767: Fix link properties discovery +- netdevsim: copy addresses for both in and out paths +- libbpf: never interpret subprogs in .text as entry programs +- ASoC: rt722-sdca: Remove logically deadcode in rt722-sdca.c +- libbpf: fix sym_is_subprog() logic for weak global subprogs +- drm/vc4: Match drm_dev_enter and exit calls in vc4_hvs_atomic_flush +- ice: consistently use q_idx in ice_vc_cfg_qs_msg() +- ice: Support FCS/CRC strip disable for VF +- virtchnl: Add CRC stripping capability +- wifi: ath11k: Fix CE offset address calculation for WCN6750 in SSR +- bpf: Fix the xdp_adjust_tail sample prog issue +- libbpf: Fix expected_attach_type set handling in program load callback +- drm/bridge: it6505: Drop EDID cache on bridge power off +- drm/bridge: anx7625: Drop EDID cache on bridge power off +- ASoC: dt-bindings: mt6359: Update generic node name and dmic-mode +- ASoC: fsl_micfil: fix regmap_write_bits usage +- dt-bindings: vendor-prefixes: Add NeoFidelity, Inc +- wifi: ath10k: fix invalid VHT parameters in supported_vht_mcs_rate_nss2 +- wifi: ath10k: fix invalid VHT parameters in supported_vht_mcs_rate_nss1 +- drm/v3d: Address race-condition in MMU flush +- drm/imx/ipuv3: Use IRQF_NO_AUTOEN flag in request_irq() +- drm/imx/dcss: Use IRQF_NO_AUTOEN flag in request_irq() +- wifi: mwifiex: Use IRQF_NO_AUTOEN flag in request_irq() +- wifi: p54: Use IRQF_NO_AUTOEN flag in request_irq() +- drm/omap: Fix locking in omap_gem_new_dmabuf() +- drm/omap: Fix possible NULL dereference +- wifi: ath9k: add range check for conn_rsp_epid in htc_connect_service() +- drm/vc4: hvs: Correct logic on stopping an HVS channel +- drm/vc4: hvs: Remove incorrect limit from hvs_dlist debugfs function +- drm/vc4: hvs: Fix dlist debug not resetting the next entry pointer +- drm/vc4: hvs: Don't write gamma luts on 2711 +- drm/mm: Mark drm_mm_interval_tree*() functions with __maybe_unused +- platform/x86: panasonic-laptop: Return errno correctly in show callback +- HID: hyperv: streamline driver probe to avoid devres issues +- arm64: dts: rockchip: correct analog audio name on Indiedroid Nova +- selftests/resctrl: Protect against array overrun during iMC config parsing +- selftests/resctrl: Fix memory overflow due to unhandled wraparound +- selftests/resctrl: Refactor fill_buf functions +- selftests/resctrl: Split fill_buf to allow tests finer-grained control +- arm64: dts: mediatek: mt8183-kukui-jacuzzi: Add supplies for fixed regulators +- arm64: dts: mediatek: mt8183-kukui-jacuzzi: Fix DP bridge supply names +- arm64: dts: mediatek: mt6358: fix dtbs_check error +- arm64: dts: mediatek: Add ADC node on MT6357, MT6358, MT6359 PMICs +- arm64: dts: renesas: hihope: Drop #sound-dai-cells +- regmap: irq: Set lockdep class for hierarchical IRQ domains +- spi: zynqmp-gqspi: Undo runtime PM changes at driver exit time​ +- spi: tegra210-quad: Avoid shift-out-of-bounds +- pmdomain: ti-sci: Add missing of_node_put() for args.np +- of/fdt: add dt_phys arg to early_init_dt_scan and early_init_dt_verify +- x86/of: Unconditionally call unflatten_and_copy_device_tree() +- um: Unconditionally call unflatten_device_tree() +- arm64: dts: ti: k3-j721s2: Fix clock IDs for MCSPI instances +- arm64: dts: ti: k3-j721e: Fix clock IDs for MCSPI instances +- arm64: dts: ti: k3-j7200: Fix clock ids for MCSPI instances +- arm64: dts: ti: k3-j7200: Fix register map for main domain pmx +- arm64: dts: ti: k3-j7200: use ti,j7200-padconf compatible +- ARM: dts: cubieboard4: Fix DCDC5 regulator constraints +- pwm: imx27: Workaround of the pwm output bug when decrease the duty cycle +- arm64: dts: mt8183: Damu: add i2c2's i2c-scl-internal-delay-ns +- arm64: dts: mt8183: cozmo: add i2c2's i2c-scl-internal-delay-ns +- arm64: dts: mt8183: burnet: add i2c2's i2c-scl-internal-delay-ns +- arm64: dts: mt8183: fennel: add i2c2's i2c-scl-internal-delay-ns +- regulator: rk808: Restrict DVS GPIOs to the RK808 variant only +- arm64: dts: mediatek: mt8173-elm-hana: Add vdd-supply to second source trackpad +- arm64: dts: mt8183: kukui: Fix the address of eeprom at i2c4 +- arm64: dts: mt8183: krane: Fix the address of eeprom at i2c4 +- media: i2c: ds90ub960: Fix missing return check on ub960_rxport_read call +- tpm: fix signed/unsigned bug when checking event logs +- efi/libstub: fix efi_parse_options() ignoring the default command line +- openrisc: Implement fixmap to fix earlycon +- mmc: mmc_spi: drop buggy snprintf() +- ARM: dts: microchip: sam9x60: Add missing property atmel,usart-mode +- arm64: dts: qcom: sm6350: Fix GPU frequencies missing on some speedbins +- soc: ti: smartreflex: Use IRQF_NO_AUTOEN flag in request_irq() +- arm64: dts: mt8195: Fix dtbs_check error for infracfg_ao node +- arm64: dts: mt8195: Fix dtbs_check error for mutex node +- microblaze: Export xmb_manager functions +- spi: spi-fsl-lpspi: Use IRQF_NO_AUTOEN flag in request_irq() +- clocksource/drivers/timer-ti-dm: Fix child node refcount handling +- clocksource/drivers:sp804: Make user selectable +- kcsan, seqlock: Fix incorrect assumption in read_seqbegin() +- kcsan, seqlock: Support seqcount_latch_t +- time: Fix references to _msecs_to_jiffies() handling of values +- time: Partially revert cleanup on msecs_to_jiffies() documentation +- x86/unwind/orc: Fix unwind for newly forked tasks +- thermal/lib: Fix memory leak on error in thermal_genl_auto() +- tools/lib/thermal: Make more generic the command encoding function +- rcuscale: Do a proper cleanup if kfree_scale_init() fails +- crypto: cavium - Fix an error handling path in cpt_ucode_load_fw() +- crypto: caam - add error check to caam_rsa_set_priv_key_form +- hwmon: (pmbus/core) clear faults after setting smbalert mask +- hwmon: (pmbus_core) Allow to hook PMBUS_SMBALERT_MASK +- doc: rcu: update printed dynticks counter bits +- crypto: inside-secure - Fix the return value of safexcel_xcbcmac_cra_init() +- amd-pstate: Set min_perf to nominal_perf for active mode performance gov +- crypto: cavium - Fix the if condition to exit loop after timeout +- EDAC/{skx_common,i10nm}: Fix incorrect far-memory error source indicator +- EDAC/skx_common: Differentiate memory error sources +- EDAC/fsl_ddr: Fix bad bit shift operations +- thermal: core: Initialize thermal zones before registering them +- crypto: qat - remove faulty arbiter config reset +- firmware: google: Unregister driver_info on failure +- crypto: qat/qat_4xxx - fix off by one in uof_get_name() +- crypto: qat - remove check after debugfs_create_dir() +- crypto: caam - Fix the pointer passed to caam_qi_shutdown() +- virtio_blk: reverse request order in virtio_queue_rqs +- nvme-pci: reverse request order in nvme_queue_rqs +- ext4: remove calls to to set/clear the folio error flag +- s390/syscalls: Avoid creation of arch/arch/ directory +- block: fix bio_split_rw_at to take zone_write_granularity into account +- acpi/arm64: Adjust error handling procedure in gtdt_parse_timer_block() +- arm64: fix .data.rel.ro size assertion when CONFIG_LTO_CLANG +- m68k: mvme147: Reinstate early console +- m68k: mvme16x: Add and use "mvme16x.h" +- m68k: mvme147: Fix SCSI controller IRQ numbers +- mips: asm: fix warning when disabling MIPS_FP_SUPPORT +- x86/pvh: Call C code via the kernel virtual mapping +- x86/pvh: Set phys_base when calling xen_prepare_pvh() +- s390/pageattr: Implement missing kernel_page_present() +- s390/cio: Do not unregister the subchannel based on DNV +- kselftest/arm64: mte: fix printf type warnings about longs +- kselftest/arm64: mte: fix printf type warnings about __u64 +- fs/proc: do_task_stat: use sig->stats_lock to gather the threads/children stats +- drm/amd/display: Don't refer to dc_sink in is_dsc_need_re_compute +- x86/stackprotector: Work around strict Clang TLS symbol requirements +- ipmr: Fix access to mfc_cache_list without lock held +- ARM: 9420/1: smp: Fix SMP for xip kernels +- ALSA: usb-audio: Fix Yamaha P-125 Quirk Entry +- LoongArch: Define a default value for VM_DATA_DEFAULT_FLAGS +- ASoC: audio-graph-card2: Purge absent supplies for device tree nodes +- proc/softirqs: replace seq_printf with seq_put_decimal_ull_width +- drm: panel-orientation-quirks: Make Lenovo Yoga Tab 3 X90F DMI match less strict +- ASoC: stm: Prevent potential division by zero in stm32_sai_get_clk_div() +- ASoC: stm: Prevent potential division by zero in stm32_sai_mclk_round_rate() +- ASoC: amd: yc: Support dmic on another model of Lenovo Thinkpad E14 Gen 6 +- platform/x86: thinkpad_acpi: Fix for ThinkPad's with ECFW showing incorrect fan speed +- can: j1939: fix error in J1939 documentation. +- tools/lib/thermal: Remove the thermal.h soft link when doing make clean +- ASoC: tas2781: Add new driver version for tas2563 & tas2781 qfn chip +- platform/x86: dell-wmi-base: Handle META key Lock/Unlock events +- platform/x86: dell-smbios-base: Extends support to Alienware products +- regulator: rk808: Add apply_bit for BUCK3 on RK809 +- firmware: arm_scmi: Reject clear channel request on A2P +- soc: qcom: Add check devm_kasprintf() returned value +- net: usb: qmi_wwan: add Quectel RG650V +- bpf: fix filed access without lock +- x86/amd_nb: Fix compile-testing without CONFIG_AMD_NB +- ALSA: hda/realtek: Add subwoofer quirk for Infinix ZERO BOOK 13 +- usb: typec: use cleanup facility for 'altmodes_node' +- usb: add support for new USB device ID 0x17EF:0x3098 for the r8152 driver +- mac80211: fix user-power when emulating chanctx +- wifi: iwlwifi: mvm: Use the sync timepoint API in suspend +- ASoC: Intel: sst: Support LPE0F28 ACPI HID +- ASoC: Intel: bytcr_rt5640: Add DMI quirk for Vexia Edu Atla 10 tablet +- ASoC: Intel: bytcr_rt5640: Add support for non ACPI instantiated codec +- ASoC: codecs: rt5640: Always disable IRQs from rt5640_cancel_work() +- wifi: radiotap: Avoid -Wflex-array-member-not-at-end warnings + +* Tue Feb 18 2025 ZhangPeng - 6.6.0-77.0.0.82 +- kernel.spec: build iso with 64KB page size kernel + +* Wed Feb 12 2025 ZhangPeng - 6.6.0-77.0.0.81 +- !15062 gpio: xilinx: Convert gpio_lock to raw spinlock +- gpio: xilinx: Convert gpio_lock to raw spinlock +- !15060 RDMA/hns: Fix mbox timing out by adding retry mechanism +- RDMA/hns: Fix mbox timing out by adding retry mechanism +- !15052 [OLK-6.6] Intel: Backport KVM Fix for Clearing SGX EDECCSSA to OLK-6.6 +- KVM: VMX: Also clear SGX EDECCSSA in KVM CPU caps when SGX is disabled +- !15048 net: hns3: add sync command to sync io-pgtable +- net: hns3: add sync command to sync io-pgtable +- Revert "net: hns3: add sync command to sync io-pgtable" +- net: hns3: fix oops when unload drivers paralleling +- !15031 nbd: don't allow reconnect after disconnect +- nbd: don't allow reconnect after disconnect +- !15002 riscv: Fix sleeping in invalid context in die() +- riscv: Fix sleeping in invalid context in die() +- !14989 ksmbd: fix a missing return value check bug +- ksmbd: fix a missing return value check bug +- !15027 pinmux: Use sequential access to access desc->pinmux data +- pinmux: Use sequential access to access desc->pinmux data +- !14321 [OLK-6.6] rcu: Provide a boot time parameter to control lazy RCU +- rcu: Provide a boot time parameter to control lazy RCU +- rcu/tree: Remove superfluous return from void call_rcu* functions +- !14900 LeapIOraid: update leapioraid driver version +- LeapIOraid: update leapioraid driver version +- !14325 LeapIOraid: Fix hiding ugood disk problem +- LeapIOraid: Fix hiding ugood disk problem +- !15016 spi: mpc52xx: Add cancel_work_sync before module remove +- spi: mpc52xx: Add cancel_work_sync before module remove +- !15041 gve: guard XDP xmit NDO on existence of xdp queues +- gve: guard XDP xmit NDO on existence of xdp queues +- !15040 gve: guard XSK operations on the existence of queues +- gve: guard XSK operations on the existence of queues +- !15039 mptcp: fix TCP options overflow. +- mptcp: fix TCP options overflow. +- !15038 net: fix memory leak in tcp_conn_request() +- net: fix memory leak in tcp_conn_request() +- !15037 net/smc: check smcd_v2_ext_offset when receiving proposal msg +- net/smc: check smcd_v2_ext_offset when receiving proposal msg +- !15036 net/smc: check return value of sock_recvmsg when draining clc data +- net/smc: check return value of sock_recvmsg when draining clc data +- !15025 net/smc: check iparea_offset and ipv6_prefixes_cnt when receiving proposal msg +- net/smc: check iparea_offset and ipv6_prefixes_cnt when receiving proposal msg +- !15019 platform/x86/amd/pmc: Only disable IRQ1 wakeup where i8042 actually enabled it +- platform/x86/amd/pmc: Only disable IRQ1 wakeup where i8042 actually enabled it +- !15020 drm/modes: Avoid divide by zero harder in drm_mode_vrefresh() +- drm/modes: Avoid divide by zero harder in drm_mode_vrefresh() +- !14979 igb: Fix potential invalid memory access in igb_init_module() +- igb: Fix potential invalid memory access in igb_init_module() +- !14963 exfat: fix the infinite loop in exfat_readdir() +- exfat: fix the infinite loop in exfat_readdir() +- !14988 net/smc: check v2_ext_offset/eid_cnt/ism_gid_cnt when receiving proposal msg +- net/smc: check v2_ext_offset/eid_cnt/ism_gid_cnt when receiving proposal msg +- !14987 workqueue: Do not warn when cancelling WQ_MEM_RECLAIM work from !WQ_MEM_RECLAIM worker +- workqueue: Do not warn when cancelling WQ_MEM_RECLAIM work from !WQ_MEM_RECLAIM worker +- !14985 i3c: Use i3cdev->desc->info instead of calling i3c_device_get_info() to avoid deadlock +- i3c: Use i3cdev->desc->info instead of calling i3c_device_get_info() to avoid deadlock +- !14892 s390/cpum_sf: Handle CPU hotplug remove during sampling +- s390/cpum_sf: Handle CPU hotplug remove during sampling +- !15013 v2 nilfs2: prevent use of deleted inode +- nilfs2: prevent use of deleted inode +- !14845 merge HULK-6.6 patches into OLK-6.6 +- mm: refactor map_deny_write_exec() +- drm/mediatek: Fix potential NULL dereference in mtk_crtc_destroy() +- nommu: pass NULL argument to vma_iter_prealloc() +- mm: page_alloc: move mlocked flag clearance into free_pages_prepare() +- cacheinfo: Allocate memory during CPU hotplug if not done from the primary CPU +- kcsan: Turn report_filterlist_lock into a raw_spinlock +- ionic: Fix netdev notifier unregister on failure +- !15018 misc: microchip: pci1xxxx: Resolve kernel panic during GPIO IRQ handling +- misc: microchip: pci1xxxx: Resolve kernel panic during GPIO IRQ handling +- !15009 i3c: mipi-i3c-hci: Mask ring interrupts before ring stop request +- i3c: mipi-i3c-hci: Mask ring interrupts before ring stop request +- !15008 nvme-rdma: unquiesce admin_q before destroy it +- nvme-rdma: unquiesce admin_q before destroy it + +* Thu Feb 06 2025 ZhangPeng - 6.6.0-76.0.0.80 +- !14997 wifi: rtw89: check return value of ieee80211_probereq_get() for RNR +- wifi: rtw89: check return value of ieee80211_probereq_get() for RNR +- !14941 CVE-2024-57906 +- iio: adc: ti-ads8688: fix information leak in triggered buffer +- !14990 mainline patches +- cgroup/bpf: only cgroup v2 can be attached by bpf programs +- Revert "cgroup: Fix memory leak caused by missing cgroup_bpf_offline" +- cgroup/cpuset: Prevent leakage of isolated CPUs into sched domains +- !14993 arm64: ptrace: fix partial SETREGSET for NT_ARM_TAGGED_ADDR_CTRL +- arm64: ptrace: fix partial SETREGSET for NT_ARM_TAGGED_ADDR_CTRL +- !14952 drm/mediatek: Set private->all_drm_private[i +- drm/mediatek: Set private->all_drm_private[i]->drm to NULL if mtk_drm_bind returns err +- !14835 v2 media: dvb-frontends: dib3000mb: fix uninit-value in dib3000_write_reg +- media: dvb-frontends: dib3000mb: fix uninit-value in dib3000_write_reg +- !14994 mm/compaction: fix UBSAN shift-out-of-bounds warning +- mm/compaction: fix UBSAN shift-out-of-bounds warning +- !14893 btrfs: flush delalloc workers queue before stopping cleaner kthread during unmount +- btrfs: flush delalloc workers queue before stopping cleaner kthread during unmount +- !14876 *** jffs2: Prevent rtime decompress memory corruption *** +- jffs2: Fix rtime decompressor +- jffs2: Prevent rtime decompress memory corruption +- !14833 btrfs: check folio mapping after unlock in relocate_one_folio() +- btrfs: check folio mapping after unlock in relocate_one_folio() +- !14588 smb: Initialize cfid->tcon before performing network ops +- smb: Initialize cfid->tcon before performing network ops +- !14977 PCI: imx6: Fix suspend/resume support on i.MX6QDL +- PCI: imx6: Fix suspend/resume support on i.MX6QDL +- !14984 vsock/bpf: return early if transport is not assigned +- vsock/bpf: return early if transport is not assigned +- !14976 x86/fpu: Ensure shadow stack is active before "getting" registers +- x86/fpu: Ensure shadow stack is active before "getting" registers +- !14682 CVE-2024-56701 +- powerpc/pseries: Fix dtl_access_lock to be a rw_semaphore +- !14780 CVE-2024-56677 +- powerpc/fadump: Move fadump_cma_init to setup_arch() after initmem_init() +- powerpc/fadump: Refactor and prepare fadump_cma_init for late init +- !14681 CVE-2024-56567 +- ad7780: fix division by zero in ad7780_write_raw() +- !14981 nfsd: fix nfs4_openowner leak when concurrent nfsd4_open occur +- nfsd: fix nfs4_openowner leak when concurrent nfsd4_open occur +- !14916 CVE-2024-56641 +- net/smc: initialize close_work early to avoid warning +- net/smc: refactoring initialization of smc sock +- !14914 bnxt_en: Fix receive ring space parameters when XDP is active +- bnxt_en: Fix receive ring space parameters when XDP is active +- !14915 net/smc: protect link down work from execute after lgr freed +- net/smc: protect link down work from execute after lgr freed +- !14913 xsk: fix OOB map writes when deleting elements +- xsk: fix OOB map writes when deleting elements +- !14911 net/smc: fix LGR and link use-after-free issue +- net/smc: fix LGR and link use-after-free issue +- !14912 net: fix data-races around sk->sk_forward_alloc +- net: fix data-races around sk->sk_forward_alloc +- !14907 ipv4: ip_tunnel: Fix suspicious RCU usage warning in ip_tunnel_find() +- ipv4: ip_tunnel: Fix suspicious RCU usage warning in ip_tunnel_find() +- !14908 net: dsa: improve shutdown sequence +- net: dsa: improve shutdown sequence +- !14877 sched: qos sched support online task to preempt offline task +- sched: qos sched support online task to preempt offline task +- !14974 BMA: Fix Oops and spin lock deadlock problem, and change the version number. +- BMA: Fix Oops and spin lock deadlock problem, and change the version number. +- !14973 v2 fix CVE-2024-54193 +- accel/ivpu: Fix WARN in ivpu_ipc_send_receive_internal() +- accel/ivpu: Move set autosuspend delay to HW specific code +- !14901 Fix CVE-2024-47794 +- bpf: Fix kabi breakage in struct bpf_prog_aux +- selftests/bpf: Add test to verify tailcall and freplace restrictions +- selftests/bpf: Add testcase for updating attached freplace prog to prog_array map +- bpf: Prevent tailcall infinite loop caused by freplace +- !14874 bpf: Check size for BTF-based ctx access of pointer members +- bpf: Check size for BTF-based ctx access of pointer members +- !14847 bpf: put bpf_link's program when link is safe to be deallocated +- bpf: put bpf_link's program when link is safe to be deallocated +- !14945 btrfs: avoid NULL pointer dereference if no valid extent tree +- btrfs: avoid NULL pointer dereference if no valid extent tree +- !14823 mm：some shmem bugfix +- mm: shmem: fix ShmemHugePages at swapout +- mm: shmem: extend shmem_unused_huge_shrink() to all sizes +- mm: shmem: fix minor off-by-one in shrinkable calculation +- docs: mm: fix the incorrect 'FileHugeMapped' field +- mm: shmem: fix the update of 'shmem_falloc->nr_unswapped' +- mm: shmem: fix incorrect index alignment for within_size policy +- !14970 drm/amd/display: Add check for granularity in dml ceil/floor helpers +- drm/amd/display: Add check for granularity in dml ceil/floor helpers +- !14969 topology: Keep the cpumask unchanged when printing cpumap +- topology: Keep the cpumask unchanged when printing cpumap +- !14935 Two CVE fixes for ksmbd +- ksmbd: fix potencial out-of-bounds when buffer offset is invalid +- ksmbd: fix slab-out-of-bounds in smb_strndup_from_utf16() +- !14930 tracing/kprobe: Make trace_kprobe's module callback called after jump_label update +- tracing/kprobe: Make trace_kprobe's module callback called after jump_label update +- !14931 bpf: Fix theoretical prog_array UAF in __uprobe_perf_func() +- bpf: Fix theoretical prog_array UAF in __uprobe_perf_func() +- !14953 mm: khugepaged: fix call hpage_collapse_scan_file() for anonymous vma +- mm: khugepaged: fix call hpage_collapse_scan_file() for anonymous vma +- !14767 mm: shmem: control THP support through the kernel command line +- mm: shmem: remove __shmem_huge_global_enabled() +- mm: shmem: override mTHP shmem default with a kernel parameter +- mm: move ``get_order_from_str()`` to internal.h +- mm: shmem: control THP support through the kernel command line +- !14962 memcg: fix memcg_swapmax_read() not declared warning +- memcg: fix memcg_swapmax_read() not declared warning +- !14944 ocfs2: fix slab-use-after-free due to dangling pointer dqi_priv +- ocfs2: fix slab-use-after-free due to dangling pointer dqi_priv +- !14946 swiotlb: fix swiotlb_bounce() to do partial sync's correctly +- iommu/dma: fix zeroing of bounce buffer padding used by untrusted devices +- swiotlb: remove alloc_size argument to swiotlb_tbl_map_single() +- swiotlb: fix swiotlb_bounce() to do partial sync's correctly +- swiotlb: extend buffer pre-padding to alloc_align_mask if necessary +- !14897 perf/x86/intel/ds: Unconditionally drain PEBS DS when changing PEBS_DATA_CFG +- perf/x86/intel/ds: Unconditionally drain PEBS DS when changing PEBS_DATA_CFG +- !14920 iio: light: vcnl4035: fix information leak in triggered buffer +- iio: light: vcnl4035: fix information leak in triggered buffer +- !14964 v2 fix BUG: unable to handle kernel paging request in build_id_parse +- lib/buildid: Handle memfd_secret() files in build_id_parse() +- mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed() +- !14951 Mainline patch to olk-6.6 +- RDMA/hns: Fix UAF for cq async event +- Revert "RDMA/hns: Fix UAF for cq async event" +- RDMA/hns: Check atomic wr length +- Revert "RDMA/hns: Fix a missing check of atomic wr length" +- Revert "RDMA/hns: Check atomic wr length" +- !14925 RDMA/hns: Fix unmatch exception handling when request_irq() fails +- RDMA/hns: Fix unmatch exception handling when request_irq() fails +- !14884 um: Fix potential integer overflow during physmem setup +- um: Fix potential integer overflow during physmem setup +- !14886 Bluetooth: hci_core: Fix not checking skb length on hci_acldata_packet +- Bluetooth: hci_core: Fix not checking skb length on hci_acldata_packet +- !14889 clk: ralink: mtmips: clock fixes for Ralink SoCs +- clk: ralink: mtmips: fix clocks probe order in oldest ralink SoCs +- clk: ralink: mtmips: fix clock plan for Ralink SoC RT3883 +- !14888 usb: dwc3: gadget: Fix looping of queued SG entries +- usb: dwc3: gadget: Fix looping of queued SG entries +- !14924 soc: imx8m: Probe the SoC driver as platform driver +- soc: imx8m: Probe the SoC driver as platform driver +- !14813 drm/i915/hdcp: Add encoder check in intel_hdcp_get_capability +- drm/i915/hdcp: Add encoder check in intel_hdcp_get_capability + +* Tue Jan 21 2025 ZhangPeng - 6.6.0-75.0.0.79 +- !14904 net: renesas: rswitch: avoid use-after-put for a device tree node +- net: renesas: rswitch: avoid use-after-put for a device tree node +- !14898 Revert "Revert "driver core: Fix uevent_show() vs driver detach race"" +- Revert "Revert "driver core: Fix uevent_show() vs driver detach race"" +- !14741 io_uring: check for overflows in io_pin_pages +- io_uring: check for overflows in io_pin_pages +- !14863 drm/sti: avoid potential dereference of error pointers in sti_gdp_atomic_check +- drm/sti: avoid potential dereference of error pointers in sti_gdp_atomic_check +- !14882 usb: typec: fix potential array underflow in ucsi_ccg_sync_control() +- usb: typec: fix potential array underflow in ucsi_ccg_sync_control() +- !14879 CVE-2024-57872 +- scsi: ufs: pltfrm: Dellocate HBA during ufshcd_pltfrm_remove() +- scsi: ufs: pltfrm: Drop PM runtime reference count after ufshcd_remove() +- scsi: ufs: pltfrm: Disable runtime PM during removal of glue drivers +- scsi: ufs: Convert all platform drivers to return void +- !14880 pinctrl: mcp23s08: Fix sleeping in atomic context due to regmap locking +- pinctrl: mcp23s08: Fix sleeping in atomic context due to regmap locking +- !14873 [openEuler-24.03-LTS][linux-6.6.y sync] Backport 6.6.62-6.6.63 LTS Patches +- mm/damon/core: copy nr_accesses when splitting region +- mm/damon/core: handle zero schemes apply interval +- mm/damon/core: check apply interval in damon_do_apply_schemes() +- mm: refactor map_deny_write_exec() +- mm: unconditionally close VMAs on error +- drm/amd/pm: Vangogh: Fix kernel memory out of bounds write +- mptcp: pm: use _rcu variant under rcu_read_lock +- mptcp: hold pm lock when deleting entry +- NFSD: initialize copy->cp_clp early in nfsd4_copy for use by trace point +- lib/buildid: Fix build ID parsing logic +- staging: vchiq_arm: Use devm_kzalloc() for vchiq_arm_state allocation +- staging: vchiq_arm: Get the rid off struct vchiq_2835_state +- mm/damon/core: handle zero {aggregation,ops_update} intervals +- mm/damon/core: implement scheme-specific apply interval +- drm/amd: Fix initialization mistake for NBIO 7.7.0 +- nouveau: fw: sync dma after setup is called. +- drm/bridge: tc358768: Fix DSI command tx +- mmc: sunxi-mmc: Fix A100 compatible description +- LoongArch: Make KASAN work with 5-level page-tables +- LoongArch: Disable KASAN if PGDIR_SIZE is too large for cpu_vabits +- ocfs2: fix UBSAN warning in ocfs2_verify_volume() +- ALSA: hda/realtek: fix mute/micmute LEDs for a HP EliteBook 645 G10 +- ALSA: hda/realtek - Fixed Clevo platform headset Mic issue +- nommu: pass NULL argument to vma_iter_prealloc() +- KVM: VMX: Bury Intel PT virtualization (guest/host mode) behind CONFIG_BROKEN +- KVM: x86: Unconditionally set irr_pending when updating APICv state +- KVM: nVMX: Treat vpid01 as current if L2 is active, but with VPID disabled +- vdpa/mlx5: Fix PA offset with unaligned starting iotlb map +- mm: fix NULL pointer dereference in alloc_pages_bulk_noprof +- x86/mm: Fix a kdump kernel failure on SME system when CONFIG_IMA_KEXEC=y +- tools/mm: fix compile error +- ARM: 9419/1: mm: Fix kernel memory mapping for xip kernels +- bonding: add ns target multicast address to slave device +- net: ti: icssg-prueth: Fix 1 PPS sync +- stmmac: dwmac-intel-plat: fix call balance of tx_clk handling routines +- net: stmmac: rename stmmac_pltfr_remove_no_dt to stmmac_pltfr_remove +- net: stmmac: dwmac-visconti: use devm_stmmac_probe_config_dt() +- net: stmmac: dwmac-intel-plat: use devm_stmmac_probe_config_dt() +- net: Make copy_safe_from_sockptr() match documentation +- net: stmmac: dwmac-mediatek: Fix inverted handling of mediatek,mac-wol +- samples: pktgen: correct dev to DEV +- net: sched: cls_u32: Fix u32's systematic failure to free IDR entries for hnodes. +- net/sched: cls_u32: replace int refcounts with proper refcounts +- Bluetooth: btintel: Direct exception event to bluetooth stack +- Bluetooth: hci_core: Fix calling mgmt_device_connected +- Revert "RDMA/core: Fix ENODEV error for iWARP test over vlan" +- virtio/vsock: Fix accept_queue memory leak +- net/mlx5e: CT: Fix null-ptr-deref in add rule err flow +- net/mlx5e: clear xdp features on non-uplink representors +- net/mlx5e: kTLS, Fix incorrect page refcounting +- net/mlx5: fs, lock FTE when checking if active +- mptcp: cope racing subflow creation in mptcp_rcv_space_adjust +- mptcp: error out earlier on disconnect +- net: vertexcom: mse102x: Fix tx_bytes calculation +- sctp: fix possible UAF in sctp_v6_available() +- netlink: terminate outstanding dump on socket close +- !14856 sched: psi: fix bogus pressure spikes from aggregation race +- sched: psi: fix bogus pressure spikes from aggregation race +- !14861 drm/sti: avoid potential dereference of error pointers +- drm/sti: avoid potential dereference of error pointers +- !14846 net: stmmac: fix TSO DMA API usage causing oops +- net: stmmac: fix TSO DMA API usage causing oops +- !14789 erofs: handle NONHEAD !delta[1 +- erofs: handle NONHEAD !delta[1] lclusters gracefully +- !14848 LoongArch: Fix sleeping in atomic context for PREEMPT_RT +- LoongArch: Fix sleeping in atomic context for PREEMPT_RT +- !14840 [OLK-6.6][bugfix]Hygon: Remove 'CONFIG_' prefix under Kconfig entry HYGON_CSV +- x86/config: Remove 'CONFIG_' prefix under Kconfig entry HYGON_CSV +- !14690 Improve the tmpfs large folio read performance +- mm: shmem: fallback to page size splice if large folio has poisoned pages +- mm: shmem: improve the tmpfs large folio read performance +- mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic +- !14466 bpf: Call free_htab_elem() after htab_unlock_bucket() +- bpf: Call free_htab_elem() after htab_unlock_bucket() +- !14818 olk-6.6 +- cpufreq: CPPC: Fix possible null-ptr-deref for cpufreq_cpu_get_raw() +- !14738 LoongArch: fix some dwmac/pci/SE/i2c/interrupt issues +- LoongArch: adjust the calc method of number of packages. +- LoongArch: eiointc: fix ext irq route error +- drivers/char: add ACPI firmware support for Loongson SE driver +- LoongArch: fix i2c related issues +- net: stmmac: dwmac-loongson: fix Mac DMA reset +- pci/quirks: LS7A2000: fix pm transition of devices under pcie port +- !14573 RDMA/rxe: Fix the qp flush warnings in req +- RDMA/rxe: Fix the qp flush warnings in req +- !14574 ALSA: pcm: Add sanity NULL check for the default mmap fault handler +- ALSA: pcm: Add sanity NULL check for the default mmap fault handler +- !14035 Backport fs optimizing patch from mainline +- fs/file.c: conditionally clear full_fds +- !14828 cgroup/cpuset: remove kernfs active break +- cgroup/cpuset: remove kernfs active break +- !14589 ksmbd: fix Out-of-Bounds Write in ksmbd_vfs_stream_write +- ksmbd: fix Out-of-Bounds Write in ksmbd_vfs_stream_write +- !14800 Bluetooth: fix use-after-free in device_for_each_child() +- Bluetooth: fix use-after-free in device_for_each_child() +- !14792 jfs: fix shift-out-of-bounds in dbSplit +- jfs: fix shift-out-of-bounds in dbSplit +- !14791 scsi: ufs: core: sysfs: Prevent div by zero +- scsi: ufs: core: sysfs: Prevent div by zero +- !14787 acpi: nfit: vmalloc-out-of-bounds Read in acpi_nfit_ctl +- acpi: nfit: vmalloc-out-of-bounds Read in acpi_nfit_ctl +- !14786 x86/xen: don't do PV iret hypercall through hypercall page +- x86/xen: don't do PV iret hypercall through hypercall page +- !14785 drm/i915: Fix NULL pointer dereference in capture_engine +- drm/i915: Fix NULL pointer dereference in capture_engine +- !14817 olk-6.6 +- wifi: rtlwifi: Drastically reduce the attempts to read efuse in case of failures +- !14784 wifi: nl80211: fix NL80211_ATTR_MLO_LINK_ID off-by-one +- wifi: nl80211: fix NL80211_ATTR_MLO_LINK_ID off-by-one +- !14807 btrfs: fix use-after-free in btrfs_encoded_read_endio() +- btrfs: fix use-after-free in btrfs_encoded_read_endio() +- !14591 ubifs: authentication: Fix use-after-free in ubifs_tnc_end_commit +- ubifs: authentication: Fix use-after-free in ubifs_tnc_end_commit +- !14587 jfs: add a check to prevent array-index-out-of-bounds in dbAdjTree +- jfs: add a check to prevent array-index-out-of-bounds in dbAdjTree +- !14747 net: enetc: Do not configure preemptible TCs if SIs do not support +- net: enetc: Do not configure preemptible TCs if SIs do not support +- !14796 btrfs: fix use-after-free when COWing tree bock and tracing is enabled +- btrfs: fix use-after-free when COWing tree bock and tracing is enabled +- !14655 vfio/mlx5: Fix an unwind issue in mlx5vf_add_migration_pages() +- vfio/mlx5: Fix an unwind issue in mlx5vf_add_migration_pages() +- !14656 iio: adc: ad7923: Fix buffer overflow for tx_buf and ring_xfer +- iio: adc: ad7923: Fix buffer overflow for tx_buf and ring_xfer +- !14659 firmware: arm_scpi: Check the DVFS OPP count returned by the firmware +- firmware: arm_scpi: Check the DVFS OPP count returned by the firmware +- !14712 CVE-2024-56654 +- Bluetooth: hci_event: Fix using rcu_read_(un)lock while iterating +- Bluetooth: ISO: Reassociate a socket with an active BIS +- !14761 Add Huawei Intelligent Network Card Driver:hibifur +- Add Huawei Intelligent Network Card Driver:hibifur +- !14772 tcp_bpf: Fix the sk_mem_uncharge logic in tcp_bpf_sendmsg +- tcp_bpf: Fix the sk_mem_uncharge logic in tcp_bpf_sendmsg +- !14769 svcrdma: fix miss destroy percpu_counter in svc_rdma_proc_init() +- svcrdma: fix miss destroy percpu_counter in svc_rdma_proc_init() +- !14770 bpf, sockmap: Several fixes to bpf_msg_pop_data +- bpf, sockmap: Several fixes to bpf_msg_pop_data +- !14730 sched: Fix bpf cpustats refcount leak +- sched: Fix bpf cpustats refcount leak +- !14739 mfd: intel_soc_pmic_bxtwc: Use IRQ domain for USB Type-C device +- mfd: intel_soc_pmic_bxtwc: Use IRQ domain for USB Type-C device +- !14731 octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_common.c +- octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_common.c +- !14759 wifi: ath12k: Skip Rx TID cleanup for self peer +- wifi: ath12k: Skip Rx TID cleanup for self peer +- !14757 media: platform: allegro-dvt: Fix possible memory leak in allocate_buffers_internal() +- media: platform: allegro-dvt: Fix possible memory leak in allocate_buffers_internal() +- !14605 scsi: qedi: Fix a possible memory leak in qedi_alloc_and_init_sb() +- scsi: qedi: Fix a possible memory leak in qedi_alloc_and_init_sb() +- !14490 nilfs2: fix potential out-of-bounds memory access in nilfs_find_entry() +- nilfs2: fix potential out-of-bounds memory access in nilfs_find_entry() +- !14489 f2fs: fix f2fs_bug_on when uninstalling filesystem call f2fs_evict_inode. +- f2fs: fix f2fs_bug_on when uninstalling filesystem call f2fs_evict_inode. +- !14488 ovl: Filter invalid inodes with missing lookup function +- ovl: Filter invalid inodes with missing lookup function +- !14486 ocfs2: fix uninitialized value in ocfs2_file_read_iter() +- ocfs2: fix uninitialized value in ocfs2_file_read_iter() +- !14754 i3c: master: Fix miss free init_dyn_addr at i3c_master_put_i3c_addrs() +- i3c: master: Fix miss free init_dyn_addr at i3c_master_put_i3c_addrs() +- !14776 tracing: Prevent bad count for tracing_cpumask_write +- tracing: Prevent bad count for tracing_cpumask_write +- !14766 drm/i915/hdcp: Add encoder check in hdcp2_get_capability +- drm/i915/hdcp: Add encoder check in hdcp2_get_capability +- !14765 nilfs2: fix null-ptr-deref in block_touch_buffer tracepoint +- nilfs2: fix null-ptr-deref in block_touch_buffer tracepoint +- !14764 octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_ethtool.c +- octeontx2-pf: handle otx2_mbox_get_rsp errors in otx2_ethtool.c +- !14729 KVM: arm64: Don't retire aborted MMIO instruction +- KVM: arm64: Don't retire aborted MMIO instruction +- !14763 usb: musb: Fix hardware lockup on first Rx endpoint request +- usb: musb: Fix hardware lockup on first Rx endpoint request +- !14762 sched/numa: fix memory leak due to the overwritten vma->numab_state +- sched/numa: fix memory leak due to the overwritten vma->numab_state +- !14749 net/mlx5: DR, prevent potential error pointer dereference +- net/mlx5: DR, prevent potential error pointer dereference +- !14746 nvme-pci: fix freeing of the HMB descriptor table +- nvme-pci: fix freeing of the HMB descriptor table +- !14722 Bluetooth: MGMT: Fix possible deadlocks +- Bluetooth: MGMT: Fix possible deadlocks +- !14670 Bluetooth: L2CAP: do not leave dangling sk pointer on error in l2cap_sock_create() +- Bluetooth: L2CAP: do not leave dangling sk pointer on error in l2cap_sock_create() +- !14608 scsi: qla2xxx: Fix use after free on unload +- scsi: qla2xxx: Fix use after free on unload +- !14724 etmem: fix use-after-free of mm in the scan release process +- etmem: fix use-after-free of mm in the scan release process +- !14384 virtiofs: use pages instead of pointer for kernel direct IO +- virtiofs: use pages instead of pointer for kernel direct IO +- !14673 ALSA: 6fire: Release resources at card release +- ALSA: 6fire: Release resources at card release +- !14666 Bluetooth: btmtk: avoid UAF in btmtk_process_coredump +- Bluetooth: btmtk: avoid UAF in btmtk_process_coredump +- !14634 backport mainline bugfix patch +- genirq/msi: Fix off-by-one error in msi_domain_alloc() +- !14611 merge mainline patches into OLK-6.6 +- mm/vmscan: wake up flushers conditionally to avoid cgroup OOM +- mm/mglru: reset page lru tier bits when activating +- tmpfs: don't enable large folios if not supported +- mm: migrate: annotate data-race in migrate_folio_unmap() +- mm: Kconfig: fixup zsmalloc configuration +- mm: ignore data-race in __swap_writepage +- !14602 blk-cgroup: Fix UAF in blkcg_unpin_online() +- blk-cgroup: Fix UAF in blkcg_unpin_online() +- !14720 rtc: check if __rtc_read_time was successful in rtc_timer_do_work() +- rtc: check if __rtc_read_time was successful in rtc_timer_do_work() +- !14719 rcu/kvfree: Fix data-race in __mod_timer / kvfree_call_rcu +- rcu/kvfree: Fix data-race in __mod_timer / kvfree_call_rcu +- !14718 PCI: Fix reset_method_store() memory leak +- PCI: Fix reset_method_store() memory leak +- !14455 f2fs: fix race in concurrent f2fs_stop_gc_thread +- f2fs: fix race in concurrent f2fs_stop_gc_thread +- !14542 net: usb: lan78xx: Fix double free issue with interrupt buffer allocation +- net: usb: lan78xx: Fix double free issue with interrupt buffer allocation +- !14531 sched/deadline: Fix warning in migrate_enable for boosted tasks +- sched/deadline: Fix warning in migrate_enable for boosted tasks +- !14528 drm/amdgpu: set the right AMDGPU sg segment limitation +- drm/amdgpu: set the right AMDGPU sg segment limitation +- !14525 CVE-2024-56609 +- wifi: rtw88: use ieee80211_purge_tx_queue() to purge TX skb +- wifi: mac80211: export ieee80211_purge_tx_queue() for drivers +- !14667 sched: fix warning in sched_setaffinity +- sched: fix warning in sched_setaffinity +- !14412 [OLK-6.6]Workaround ARM PMUs cpu maps having offline cpus +- perf arm: Workaround ARM PMUs cpu maps having offline cpus +- !14698 bpf: Fix narrow scalar spill onto 64-bit spilled scalar slots +- bpf: Fix narrow scalar spill onto 64-bit spilled scalar slots +- !14694 arm64: probes: Disable kprobes/uprobes on MOPS instructions +- arm64: probes: Disable kprobes/uprobes on MOPS instructions +- !14677 drm/amd/display: Fix null check for pipe_ctx->plane_state in dcn20_program_pipe +- drm/amd/display: Fix null check for pipe_ctx->plane_state in dcn20_program_pipe +- !14569 PCI: Fix use-after-free of slot->bus on hot remove +- PCI: Fix use-after-free of slot->bus on hot remove +- !14568 wifi: ath12k: fix warning when unbinding +- wifi: ath12k: fix warning when unbinding +- !14424 octeontx2-pf: handle otx2_mbox_get_rsp errors in cn10k.c +- octeontx2-pf: handle otx2_mbox_get_rsp errors in cn10k.c +- !14714 Bluetooth: MGMT: Fix slab-use-after-free Read in set_powered_sync +- Bluetooth: MGMT: Fix slab-use-after-free Read in set_powered_sync +- !14715 media: atomisp: Add check for rgby_data memory allocation failure +- media: atomisp: Add check for rgby_data memory allocation failure +- !14699 CVE-2024-53158 +- soc: qcom: geni-se: fix array underflow in geni_se_clk_tbl_get() +- !14710 CVE-2024-56546 +- drivers: soc: xilinx: add the missing kfree in xlnx_add_cb_for_suspend() +- !14507 sched/task_stack: fix object_is_on_stack() for KASAN tagged pointers +- sched/task_stack: fix object_is_on_stack() for KASAN tagged pointers +- !14511 netfilter: nft_inner: incorrect percpu area handling under softirq +- netfilter: nft_inner: incorrect percpu area handling under softirq +- !14472 jfs: fix array-index-out-of-bounds in jfs_readdir +- jfs: fix array-index-out-of-bounds in jfs_readdir +- !14664 v2 CVE-2024-53224 +- RDMA/mlx5: Move events notifier registration to be after device registration +- IB/mlx5: Allocate resources just before first QP/SRQ is created + +* Thu Jan 16 2025 Mingzheng Xing - 6.6.0-73.0.0.78 +- riscv kernel upgrade to 6.6.0-73.0.0 +- Backport RISC-V AIA driver from mainline linux +- Fix kernel-mode FPU support on riscv +- Add cache info for SG2042 dts + +* Mon Jan 06 2025 ZhangPeng - 6.6.0-73.0.0.77 +- !14618 drm: zynqmp_kms: Unplug DRM device before removal +- drm: zynqmp_kms: Unplug DRM device before removal +- !14621 brd: defer automatic disk creation until module initialization succeeds +- brd: defer automatic disk creation until module initialization succeeds +- !14617 block: fix uaf for flush rq while iterating tags +- block: fix uaf for flush rq while iterating tags +- !14614 scsi: sg: Fix slab-use-after-free read in sg_release() +- scsi: sg: Fix slab-use-after-free read in sg_release() +- !14519 bpf,perf: Fix invalid prog_array access in perf_event_detach_bpf_prog +- bpf,perf: Fix invalid prog_array access in perf_event_detach_bpf_prog +- !14593 CVE-2024-56702 +- bpf: Augment raw_tp arguments with PTR_MAYBE_NULL +- bpf: Revert "bpf: Mark raw_tp arguments with PTR_MAYBE_NULL" +- bpf: Mark raw_tp arguments with PTR_MAYBE_NULL +- !14415 ksmbd: fix Out-of-Bounds Read in ksmbd_vfs_stream_read +- ksmbd: fix Out-of-Bounds Read in ksmbd_vfs_stream_read +- !14439 sh: intc: Fix use-after-free bug in register_intc_controller() +- sh: intc: Fix use-after-free bug in register_intc_controller() +- !14635 Bluetooth: RFCOMM: avoid leaving dangling sk pointer in rfcomm_sock_alloc() +- Bluetooth: RFCOMM: avoid leaving dangling sk pointer in rfcomm_sock_alloc() +- !14648 memcg: fix soft lockup in the OOM process +- memcg: fix soft lockup in the OOM process +- !14562 f2fs: fix to do sanity check on node blkaddr in truncate_node() +- f2fs: fix to do sanity check on node blkaddr in truncate_node() +- !14524 The bugfix for LPM trie +- bpf: Handle in-place update for full LPM trie correctly +- bpf: Handle BPF_EXIST and BPF_NOEXIST for LPM trie +- !14459 sh: cpuinfo: Fix a warning for CONFIG_CPUMASK_OFFSTACK +- sh: cpuinfo: Fix a warning for CONFIG_CPUMASK_OFFSTACK +- !14454 drm/vc4: hdmi: Avoid hang with debug registers when suspended +- drm/vc4: hdmi: Avoid hang with debug registers when suspended +- !14385 media: imx-jpeg: Ensure power suppliers be suspended before detach them +- media: imx-jpeg: Ensure power suppliers be suspended before detach them +- !14546 media: wl128x: Fix atomicity violation in fmc_send_cmd() +- media: wl128x: Fix atomicity violation in fmc_send_cmd() +- !14244 sunrpc: clear XPRT_SOCK_UPD_TIMEOUT when reset transport +- sunrpc: clear XPRT_SOCK_UPD_TIMEOUT when reset transport +- !14243 sunrpc: fix one UAF issue caused by sunrpc kernel tcp socket +- sunrpc: fix one UAF issue caused by sunrpc kernel tcp socket +- !14504 apparmor: test: Fix memory leak for aa_unpack_strdup() +- apparmor: test: Fix memory leak for aa_unpack_strdup() +- !14418 LoongArch: Add CPU HWMon platform driver +- LoongArch: Add CPU HWMon platform driver +- !14387 LoongArch: compile loongson drm driver as module +- drm/loongson: compile loongson drm driver as module +- LoongArch: cleanup loongson3 defconfig +- !14441 crypto: bcm - add error check in the ahash_hmac_init function +- crypto: bcm - add error check in the ahash_hmac_init function +- !14438 media: imx-jpeg: Set video drvdata before register video device +- media: imx-jpeg: Set video drvdata before register video device +- !14389 CVE-2024-53173 +- NFSv4.0: Fix a use-after-free problem in the asynchronous open() +- NFSv4.0: Fix the wake up of the next waiter in nfs_release_seqid() +- Revert "NFSv4: release seqid when open failed for nfs4.0" +- Revert "nfs: fix rpc_task use-after-free when open and close different files concurrently" +- !14469 io_uring/tctx: work around xa_store() allocation error issue +- io_uring/tctx: work around xa_store() allocation error issue +- !14447 Fix CVE-2024-56744 +- f2fs: fix to avoid potential deadlock in f2fs_record_stop_reason() +- !14394 scsi: ufs: qcom: Only free platform MSIs when ESI is enabled +- scsi: ufs: qcom: Only free platform MSIs when ESI is enabled +- !14518 hfsplus: don't query the device logical block size multiple times +- hfsplus: don't query the device logical block size multiple times +- !14517 scsi: bfa: Fix use-after-free in bfad_im_module_exit() +- scsi: bfa: Fix use-after-free in bfad_im_module_exit() +- !14516 f2fs: fix null-ptr-deref in f2fs_submit_page_bio() +- f2fs: fix null-ptr-deref in f2fs_submit_page_bio() +- !14475 ext4: correct behaviors under errors=remount-ro mode +- Revert "ext4: drop EXT4_MF_FS_ABORTED flag" +- Revert "ext4: warn on read-only filesystem in ext4_journal_check_start()" +- Revert "ext4: drop read-only check in ext4_init_inode_table()" +- Revert "ext4: drop read-only check in ext4_write_inode()" +- Revert "ext4: drop read-only check from ext4_force_commit()" +- Revert "ext4: replace read-only check for shutdown check in mmp code" +- Revert "ext4: don't set SB_RDONLY after filesystem errors" +- sb_writers: fix kabi broken +- fs: fix a hungtask problem when freeze/unfreeze fs +- !14506 iommufd: Fix out_fput in iommufd_fault_alloc() +- iommufd: Fix out_fput in iommufd_fault_alloc() +- !14369 crypto: qat/qat_420xx - fix off by one in uof_get_name() +- crypto: qat/qat_420xx - fix off by one in uof_get_name() +- !14417 v2 drm/nouveau/gr/gf100: Fix missing unlock in gf100_gr_chan_new() +- drm/nouveau/gr/gf100: Fix missing unlock in gf100_gr_chan_new() +- !14423 v2 EDAC/igen6: Avoid segmentation fault on module unload +- EDAC/igen6: Avoid segmentation fault on module unload +- !14365 EDAC/bluefield: Fix potential integer overflow +- EDAC/bluefield: Fix potential integer overflow +- !14364 clk: clk-apple-nco: Add NULL check in applnco_probe +- clk: clk-apple-nco: Add NULL check in applnco_probe +- !14408 bpf: fix OOB devmap writes when deleting elements +- bpf: fix OOB devmap writes when deleting elements +- !14367 io_uring: check if iowq is killed before queuing +- io_uring: check if iowq is killed before queuing +- !14355 btrfs: ref-verify: fix use-after-free after invalid ref action +- btrfs: ref-verify: fix use-after-free after invalid ref action +- !14350 jfs: array-index-out-of-bounds fix in dtReadFirst +- jfs: array-index-out-of-bounds fix in dtReadFirst +- !14348 ksmbd: fix use-after-free in SMB request handling +- ksmbd: fix use-after-free in SMB request handling +- !14102 OLK-6.6-fix-CVE-2024-49906 +- drm/amd/display: Check null pointer before try to access it +- drm/amd/display: Check null pointer before try to access it +- !14346 drm/amd/display: Fix out-of-bounds access in 'dcn21_link_encoder_create' +- drm/amd/display: Fix out-of-bounds access in 'dcn21_link_encoder_create' +- !14420 HID: wacom: fix when get product name maybe null pointer +- HID: wacom: fix when get product name maybe null pointer +- !14301 Revert "unicode: Don't special case ignorable code points" +- Revert "unicode: Don't special case ignorable code points" +- !14332 mm: avoid unconditional one-tick sleep when swapcache_prepare fails +- mm: avoid unconditional one-tick sleep when swapcache_prepare fails +- !14362 CVE-2024-53217 +- nfsd: restore callback functionality for NFSv4.0 +- NFSD: Prevent NULL dereference in nfsd4_process_cb_update() +- !14319 exfat: fix out-of-bounds access of directory entries +- exfat: fix out-of-bounds access of directory entries +- !14405 NFSD: Prevent a potential integer overflow +- NFSD: Prevent a potential integer overflow +- !14395 scsi: qedf: Fix a possible memory leak in qedf_alloc_and_init_sb() +- scsi: qedf: Fix a possible memory leak in qedf_alloc_and_init_sb() +- !14401 gpio: grgpio: Add NULL check in grgpio_probe +- gpio: grgpio: Add NULL check in grgpio_probe +- !14400 bpf: Fix UAF via mismatching bpf_prog/attachment RCU flavors +- bpf: Fix UAF via mismatching bpf_prog/attachment RCU flavors +- !14329 svcrdma: Address an integer overflow +- svcrdma: Address an integer overflow +- !14376 fix CVE-2024-53197 +- ALSA: usb-audio: Fix a DMA to stack memory bug +- ALSA: usb-audio: Fix potential out-of-bound accesses for Extigy and Mbox devices +- !14206 v4 erofs/fscache/cachefiles: Add ondemand loading support +- cachefiles: Fix missing pos updates in cachefiles_ondemand_fd_write_iter() +- cachefiles: Fix incorrect length return value in cachefiles_ondemand_fd_write_iter() +- cachefiles: Clean up in cachefiles_commit_tmpfile() +- cachefiles: Fix NULL pointer dereference in object->file +- cachefiles: add support for buffer I/O in ondemand mode +- fscache: clean up for fscache_clear_volume_priv +- fscache: Add the synchronous waiting mechanism for the volume unhash in erofs ondemand mode +- cachefiles: Introduce "dir_has_put" in cachefiles_volume +- erofs: remove EXPERIMENTAL feature warning for fscache-based +- cachefiles: Add restrictions to cachefiles_daemon_cull() +- erofs: add erofs_ondemand switch +- erofs: add erofs switch to better control it +- fscache: modify the waiting mechanism with duplicate volumes +- cachefiles: modify inappropriate error return value in cachefiles_daemon_secctx +- fscache: add a memory barrier for FSCACHE_VOLUME_CREATING +- !14377 ftrace: Fix regression with module command in stack_trace_filter +- ftrace: Fix regression with module command in stack_trace_filter +- !14373 [OLK-6.6]Hygon: Fix objtool warning cause by csv3_active() +- x86: mm: Fix objtool warning about bool noinstr csv3_active() +- !14358 firmware_loader: Fix possible resource leak in fw_log_firmware_info() +- firmware_loader: Fix possible resource leak in fw_log_firmware_info() +- !14342 hwmon: (nct6775-core) Fix overflows seen when writing limit attributes +- hwmon: (nct6775-core) Fix overflows seen when writing limit attributes +- !14271 iomap: fix zero padding data issue in concurrent append writes +- iomap: fix zero padding data issue in concurrent append writes +- !14340 ocfs2: free inode when ocfs2_get_init_inode() fails +- ocfs2: free inode when ocfs2_get_init_inode() fails +- !14312 OLK-6.6: Some patches of RDMA/hns from Linux to olk-6.6 v2 +- RDMA/hns: Fix missing flush CQE for DWQE +- RDMA/hns: Fix warning storm caused by invalid input in IO path +- RDMA/hns: Fix accessing invalid dip_ctx during destroying QP +- Revert "RDMA/hns: Fix accessing invalid dip_ctx during destroying QP" +- RDMA/hns: Fix mapping error of zero-hop WQE buffer +- !14338 CVE-2024-53230 +- cpufreq: CPPC: Fix wrong return value in cppc_get_cpu_cost() +- cpufreq: CPPC: Fix possible null-ptr-deref for cppc_get_cpu_cost() +- !14336 mm: Backport mainline patch +- mm: don't install PMD mappings when THPs are disabled by the hw/process/vma +- mm: huge_memory: add vma_thp_disabled() and thp_disabled_by_hw() +- mm: fix arithmetic for max_prop_frac when setting max_ratio +- mm: fix arithmetic for bdi min_ratio +- mm/kmemleak: fix partially freeing unknown object warning +- mm: kmemleak: add __find_and_remove_object() +- mm: kmemleak: use mem_pool_free() to free object +- mm: kmemleak: split __create_object into two functions +- mm/hugetlb: fix nodes huge page allocation when there are surplus pages +- !14337 ALSA: usb-audio: Fix out of bounds reads when finding clock sources +- ALSA: usb-audio: Fix out of bounds reads when finding clock sources +- !14322 back some mainline patches for damon +- mm/damon/core: fix return value from damos_wmark_metric_value +- mm/damon/core: remove unnecessary si_meminfo invoke. +- !14323 zram: fix NULL pointer in comp_algorithm_show() +- zram: fix NULL pointer in comp_algorithm_show() +- !14201 【OLK-6.6】 drivers: arch_topology: Refactor do-while loops +- drivers: arch_topology: Refactor do-while loops +- !14295 pmdomain: imx93-blk-ctrl: correct remove path +- pmdomain: imx93-blk-ctrl: correct remove path +- !14190 Revert "mmc: dw_mmc: Fix IDMAC operation with pages bigger than 4K" +- Revert "mmc: dw_mmc: Fix IDMAC operation with pages bigger than 4K" +- !14134 x86/CPU/AMD: Clear virtualized VMLOAD/VMSAVE on Zen4 client +- x86/CPU/AMD: Clear virtualized VMLOAD/VMSAVE on Zen4 client +- !14214 mm: revert "mm: shmem: fix data-race in shmem_getattr()" +- mm: revert "mm: shmem: fix data-race in shmem_getattr()" +- !14229 Fix extioi restart issue in vm +- LoongArch: arch/loongarch/kvm: Fix extioi restart issue +- !14127 MPTCP Upstream part 24 +- mptcp: refer to 'MPTCP' socket in comments +- mptcp: add mptcp_space_from_win helper +- mptcp: use mptcp_win_from_space helper +- doc: new 'mptcp' page in 'networking' +- doc: mptcp: alphabetical order +- doc: mptcp: add missing 'available_schedulers' entry +- selftests: mptcp: join: mark 'fastclose' tests as flaky +- selftests: mptcp: lib: support flaky subtests +- selftests: mptcp: export ip_mptcp to mptcp_lib +- !13359 [OLK-6.6] [Backport] individual submission +- MIPS: RB532: Declare prom_setup_cmdline() and rb532_gpio_init() static +- MIPS: BCM47XX: Declare early_tlb_init() static +- MIPS: BCM47XX: include header for bcm47xx_prom_highmem_init() prototype +- ocfs2: spelling fix +- !14291 vdpa: solidrun: Fix UB bug with devres +- vdpa: solidrun: Fix UB bug with devres +- !14256 RDMA/hns: Fix creating GSI QP in non-extended SGE QP bank +- RDMA/hns: Fix creating GSI QP in non-extended SGE QP bank + +* Sat Dec 21 2024 ZhangPeng - 6.6.0-72.0.0.76 +- !14213 v2 bpf: Add kabi reserve padding for uapi struct bpf_link_info +- bpf: Add kabi reserve padding for uapi struct bpf_link_info +- !14225 kabi OLK-6.6 reverve space +- kabi: reserve space for uprobes.h +- kabi: reserve space for internal.h +- kabi: reserve space for perf_event.h +- !14259 【OLK-6.6】 Hygon family 18h model 6h~fh L3 PMU config incorrect issue fix +- perf/x86/uncore: Correct the L3 PMU config for Hygon processors +- !14251 kabi reserver space for i2c +- kabi reserver space for i2c +- !14258 v2 scsi: hisi_sas: Enable force phy when SATA disk directly connected +- scsi: hisi_sas: Enable force phy when SATA disk directly connected + * Thu Dec 19 2024 ZhangPeng - 6.6.0-69.0.0.75 - !14221 KABI for cgroup - cgroup: add more reserve kabi