#include <linux/anon_inodes.h>

#include <linux/slab.h>

#include <linux/sched/autogroup.h>

#include <linux/sched/mm.h>

#include <linux/sched/user.h>

#include <linux/sched/numa_balancing.h>

#include <linux/sched/stat.h>

#include <linux/sched/task.h>

#include <linux/sched/task_stack.h>

#include <linux/sched/cputime.h>

#include <linux/sched/ext.h>

#include <linux/seq_file.h>

#include <linux/rtmutex.h>

#include <linux/init.h>

#include <linux/unistd.h>

#include <linux/module.h>

#include <linux/vmalloc.h>

#include <linux/completion.h>

#include <linux/personality.h>

#include <linux/mempolicy.h>

#include <linux/sem.h>

#include <linux/file.h>

#include <linux/fdtable.h>

#include <linux/iocontext.h>

#include <linux/key.h>

#include <linux/kmsan.h>

#include <linux/binfmts.h>

#include <linux/mman.h>

#include <linux/mmu_notifier.h>

#include <linux/fs.h>

#include <linux/mm.h>

#include <linux/mm_inline.h>

#include <linux/memblock.h>

#include <linux/nsproxy.h>

#include <linux/capability.h>

#include <linux/cpu.h>

#include <linux/cgroup.h>

#include <linux/security.h>

#include <linux/hugetlb.h>

#include <linux/seccomp.h>

#include <linux/swap.h>

#include <linux/syscalls.h>

#include <linux/syscall_user_dispatch.h>

#include <linux/jiffies.h>

#include <linux/futex.h>

#include <linux/compat.h>

#include <linux/kthread.h>

#include <linux/task_io_accounting_ops.h>

#include <linux/rcupdate.h>

#include <linux/ptrace.h>

#include <linux/mount.h>

#include <linux/audit.h>

#include <linux/memcontrol.h>

#include <linux/ftrace.h>

#include <linux/proc_fs.h>

#include <linux/profile.h>

#include <linux/rmap.h>

#include <linux/ksm.h>

#include <linux/acct.h>

#include <linux/userfaultfd_k.h>

#include <linux/tsacct_kern.h>

#include <linux/cn_proc.h>

#include <linux/freezer.h>

#include <linux/delayacct.h>

#include <linux/taskstats_kern.h>

#include <linux/tty.h>

#include <linux/fs_struct.h>

#include <linux/magic.h>

#include <linux/perf_event.h>

#include <linux/posix-timers.h>

#include <linux/user-return-notifier.h>

#include <linux/oom.h>

#include <linux/khugepaged.h>

#include <linux/signalfd.h>

#include <linux/uprobes.h>

#include <linux/aio.h>

#include <linux/compiler.h>

#include <linux/sysctl.h>

#include <linux/kcov.h>

#include <linux/livepatch.h>

#include <linux/thread_info.h>

#include <linux/stackleak.h>

#include <linux/kasan.h>

#include <linux/scs.h>

#include <linux/io_uring.h>

#include <linux/bpf.h>

#include <linux/stackprotector.h>

#include <linux/user_events.h>

#include <linux/iommu.h>

#include <linux/rseq.h>

#include <uapi/linux/pidfd.h>

#include <linux/pidfs.h>

#include <linux/tick.h>

#include <asm/pgalloc.h>

#include <linux/uaccess.h>

#include <asm/mmu_context.h>

#include <asm/cacheflush.h>

#include <asm/tlbflush.h>

#include "../mm/internal.h"

#include <trace/events/sched.h>

#define CREATE_TRACE_POINTS

#include <trace/events/task.h>

#include <kunit/visibility.h>

#define MIN_THREADS 20

#define MAX_THREADS FUTEX_TID_MASK

unsigned long total_forks; /* Handle normal Linux uptimes. */

int nr_threads; /* The idle threads do not count.. */

static int max_threads; /* tunable limit on nr_threads */

#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)

static const char * const resident_page_types[] = {

NAMED_ARRAY_INDEX(MM_FILEPAGES),

NAMED_ARRAY_INDEX(MM_ANONPAGES),

NAMED_ARRAY_INDEX(MM_SWAPENTS),

NAMED_ARRAY_INDEX(MM_SHMEMPAGES),

};

DEFINE_PER_CPU(unsigned long, process_counts) = 0;

__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */

#ifdef CONFIG_PROVE_RCU

int lockdep_tasklist_lock_is_held(void)

{

return lockdep_is_held(&tasklist_lock);

}

EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);

#endif /* #ifdef CONFIG_PROVE_RCU */

int nr_processes(void)

{

int cpu;

int total = 0;

for_each_possible_cpu(cpu)

total += per_cpu(process_counts, cpu);

return total;

}

void __weak arch_release_task_struct(struct task_struct *tsk)

{

}

static struct kmem_cache *task_struct_cachep;

static inline struct task_struct *alloc_task_struct_node(int node)

{

return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);

}

static inline void free_task_struct(struct task_struct *tsk)

{

kmem_cache_free(task_struct_cachep, tsk);

}

# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)

# ifdef CONFIG_VMAP_STACK

#define NR_CACHED_STACKS 2

static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);

struct vm_stack {

struct rcu_head rcu;

struct vm_struct *stack_vm_area;

};

static bool try_release_thread_stack_to_cache(struct vm_struct *vm)

{

unsigned int i;

for (i = 0; i < NR_CACHED_STACKS; i++) {

struct vm_struct *tmp = NULL;

if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm))

return true;

}

return false;

}

static void thread_stack_free_rcu(struct rcu_head *rh)

{

struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);

if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))

return;

vfree(vm_stack);

}

static void thread_stack_delayed_free(struct task_struct *tsk)

{

struct vm_stack *vm_stack = tsk->stack;

vm_stack->stack_vm_area = tsk->stack_vm_area;

call_rcu(&vm_stack->rcu, thread_stack_free_rcu);

}

static int free_vm_stack_cache(unsigned int cpu)

{

struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);

int i;

for (i = 0; i < NR_CACHED_STACKS; i++) {

struct vm_struct *vm_stack = cached_vm_stacks[i];

if (!vm_stack)

continue;

vfree(vm_stack->addr);

cached_vm_stacks[i] = NULL;

}

return 0;

}

static int memcg_charge_kernel_stack(struct vm_struct *vm)

{

int i;

int ret;

int nr_charged = 0;

BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);

for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {

ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0);

if (ret)

goto err;

nr_charged++;

}

return 0;

err:

for (i = 0; i < nr_charged; i++)

memcg_kmem_uncharge_page(vm->pages[i], 0);

return ret;

}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)

{

struct vm_struct *vm;

void *stack;

int i;

for (i = 0; i < NR_CACHED_STACKS; i++) {

struct vm_struct *s;

s = this_cpu_xchg(cached_stacks[i], NULL);

if (!s)

continue;

/* Reset stack metadata. */

kasan_unpoison_range(s->addr, THREAD_SIZE);

stack = kasan_reset_tag(s->addr);

/* Clear stale pointers from reused stack. */

memset(stack, 0, THREAD_SIZE);

if (memcg_charge_kernel_stack(s)) {

vfree(s->addr);

return -ENOMEM;

}

tsk->stack_vm_area = s;

tsk->stack = stack;

return 0;

}

/*

stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,

THREADINFO_GFP & ~__GFP_ACCOUNT,

node, __builtin_return_address(0));

if (!stack)

return -ENOMEM;

vm = find_vm_area(stack);

if (memcg_charge_kernel_stack(vm)) {

vfree(stack);

return -ENOMEM;

}

/*

tsk->stack_vm_area = vm;

stack = kasan_reset_tag(stack);

tsk->stack = stack;

return 0;

}

static void free_thread_stack(struct task_struct *tsk)

{

if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))

thread_stack_delayed_free(tsk);

tsk->stack = NULL;

tsk->stack_vm_area = NULL;

}

# else /* !CONFIG_VMAP_STACK */

static void thread_stack_free_rcu(struct rcu_head *rh)

{

__free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);

}

static void thread_stack_delayed_free(struct task_struct *tsk)

{

struct rcu_head *rh = tsk->stack;

call_rcu(rh, thread_stack_free_rcu);

}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)

{

struct page *page = alloc_pages_node(node, THREADINFO_GFP,

THREAD_SIZE_ORDER);

if (likely(page)) {

tsk->stack = kasan_reset_tag(page_address(page));

return 0;

}

return -ENOMEM;

}

static void free_thread_stack(struct task_struct *tsk)

{

thread_stack_delayed_free(tsk);

tsk->stack = NULL;

}

# endif /* CONFIG_VMAP_STACK */

# else /* !(THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)) */

static struct kmem_cache *thread_stack_cache;

static void thread_stack_free_rcu(struct rcu_head *rh)

{

kmem_cache_free(thread_stack_cache, rh);

}

static void thread_stack_delayed_free(struct task_struct *tsk)

{

struct rcu_head *rh = tsk->stack;

call_rcu(rh, thread_stack_free_rcu);

}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)

{

unsigned long *stack;

stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);

stack = kasan_reset_tag(stack);

tsk->stack = stack;

return stack ? 0 : -ENOMEM;

}

static void free_thread_stack(struct task_struct *tsk)

{

thread_stack_delayed_free(tsk);

tsk->stack = NULL;

}

void thread_stack_cache_init(void)

{

thread_stack_cache = kmem_cache_create_usercopy("thread_stack",

THREAD_SIZE, THREAD_SIZE, 0, 0,

THREAD_SIZE, NULL);

BUG_ON(thread_stack_cache == NULL);

}

# endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */

static struct kmem_cache *signal_cachep;

struct kmem_cache *sighand_cachep;

struct kmem_cache *files_cachep;

struct kmem_cache *fs_cachep;

static struct kmem_cache *mm_cachep;

static void account_kernel_stack(struct task_struct *tsk, int account)

{

if (IS_ENABLED(CONFIG_VMAP_STACK)) {

struct vm_struct *vm = task_stack_vm_area(tsk);

int i;

for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)

mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB,

account * (PAGE_SIZE / 1024));

} else {

void *stack = task_stack_page(tsk);

/* All stack pages are in the same node. */

mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,

account * (THREAD_SIZE / 1024));

}

}

void exit_task_stack_account(struct task_struct *tsk)

{

account_kernel_stack(tsk, -1);

if (IS_ENABLED(CONFIG_VMAP_STACK)) {

struct vm_struct *vm;

int i;

vm = task_stack_vm_area(tsk);

for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)

memcg_kmem_uncharge_page(vm->pages[i], 0);

}

}

static void release_task_stack(struct task_struct *tsk)

{

if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))

return; /* Better to leak the stack than to free prematurely */

free_thread_stack(tsk);

}

#ifdef CONFIG_THREAD_INFO_IN_TASK

void put_task_stack(struct task_struct *tsk)

{

if (refcount_dec_and_test(&tsk->stack_refcount))

release_task_stack(tsk);

}

void free_task(struct task_struct *tsk)

{

#ifdef CONFIG_SECCOMP

WARN_ON_ONCE(tsk->seccomp.filter);

release_user_cpus_ptr(tsk);

scs_release(tsk);

#ifndef CONFIG_THREAD_INFO_IN_TASK

/*

release_task_stack(tsk);

/*

WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);

rt_mutex_debug_task_free(tsk);

ftrace_graph_exit_task(tsk);

arch_release_task_struct(tsk);

if (tsk->flags & PF_KTHREAD)

free_kthread_struct(tsk);

bpf_task_storage_free(tsk);

free_task_struct(tsk);

}

EXPORT_SYMBOL(free_task);

void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)

{

struct file *exe_file;

exe_file = get_mm_exe_file(oldmm);

RCU_INIT_POINTER(mm->exe_file, exe_file);

/*

if (exe_file && exe_file_deny_write_access(exe_file))

pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);

}

#ifdef CONFIG_MMU

static inline int mm_alloc_pgd(struct mm_struct *mm)

{

mm->pgd = pgd_alloc(mm);

if (unlikely(!mm->pgd))

return -ENOMEM;

return 0;

}

static inline void mm_free_pgd(struct mm_struct *mm)

{

pgd_free(mm, mm->pgd);

}

#define mm_alloc_pgd(mm) (0)

#define mm_free_pgd(mm)

#endif /* CONFIG_MMU */

#ifdef CONFIG_MM_ID

static DEFINE_IDA(mm_ida);

static inline int mm_alloc_id(struct mm_struct *mm)

{

int ret;

ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);

if (ret < 0)

return ret;

mm->mm_id = ret;

return 0;

}

static inline void mm_free_id(struct mm_struct *mm)

{

const mm_id_t id = mm->mm_id;

mm->mm_id = MM_ID_DUMMY;

if (id == MM_ID_DUMMY)

return;

if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX))

return;

ida_free(&mm_ida, id);

}

#else /* !CONFIG_MM_ID */

static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }

static inline void mm_free_id(struct mm_struct *mm) {}

#endif /* CONFIG_MM_ID */

static void check_mm(struct mm_struct *mm)

{

int i;

BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,

"Please make sure 'struct resident_page_types[]' is updated as well");

for (i = 0; i < NR_MM_COUNTERS; i++) {

long x = percpu_counter_sum(&mm->rss_stat[i]);

if (unlikely(x))

pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",

mm, resident_page_types[i], x);

}

if (mm_pgtables_bytes(mm))

pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",

mm_pgtables_bytes(mm));

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)

VM_BUG_ON_MM(mm->pmd_huge_pte, mm);

}

#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))

#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))

static void do_check_lazy_tlb(void *arg)

{

struct mm_struct *mm = arg;

WARN_ON_ONCE(current->active_mm == mm);

}

static void do_shoot_lazy_tlb(void *arg)

{

struct mm_struct *mm = arg;

if (current->active_mm == mm) {

WARN_ON_ONCE(current->mm);

current->active_mm = &init_mm;

switch_mm(mm, &init_mm, current);

}

}

static void cleanup_lazy_tlbs(struct mm_struct *mm)

{

if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {

/*

return;

}

/*

on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1);

if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))

on_each_cpu(do_check_lazy_tlb, (void *)mm, 1);

}

void __mmdrop(struct mm_struct *mm)

{

BUG_ON(mm == &init_mm);

WARN_ON_ONCE(mm == current->mm);

/* Ensure no CPUs are using this as their lazy tlb mm */

cleanup_lazy_tlbs(mm);

WARN_ON_ONCE(mm == current->active_mm);

mm_free_pgd(mm);

mm_free_id(mm);

destroy_context(mm);

mmu_notifier_subscriptions_destroy(mm);

check_mm(mm);

put_user_ns(mm->user_ns);

mm_pasid_drop(mm);

mm_destroy_cid(mm);

percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);

free_mm(mm);

}

EXPORT_SYMBOL_GPL(__mmdrop);

static void mmdrop_async_fn(struct work_struct *work)

{

struct mm_struct *mm;

mm = container_of(work, struct mm_struct, async_put_work);

__mmdrop(mm);

}

static void mmdrop_async(struct mm_struct *mm)

{

if (unlikely(atomic_dec_and_test(&mm->mm_count))) {

INIT_WORK(&mm->async_put_work, mmdrop_async_fn);

schedule_work(&mm->async_put_work);

}

}

static inline void free_signal_struct(struct signal_struct *sig)

{

taskstats_tgid_free(sig);

sched_autogroup_exit(sig);

/*

if (sig->oom_mm)

mmdrop_async(sig->oom_mm);

kmem_cache_free(signal_cachep, sig);

}

static inline void put_signal_struct(struct signal_struct *sig)

{

if (refcount_dec_and_test(&sig->sigcnt))

free_signal_struct(sig);

}

void __put_task_struct(struct task_struct *tsk)

{

WARN_ON(!tsk->exit_state);

WARN_ON(refcount_read(&tsk->usage));

WARN_ON(tsk == current);

sched_ext_free(tsk);

io_uring_free(tsk);

cgroup_free(tsk);

task_numa_free(tsk, true);

security_task_free(tsk);

exit_creds(tsk);

delayacct_tsk_free(tsk);

put_signal_struct(tsk->signal);

sched_core_free(tsk);

free_task(tsk);

}

EXPORT_SYMBOL_GPL(__put_task_struct);

void __put_task_struct_rcu_cb(struct rcu_head *rhp)

{

struct task_struct *task = container_of(rhp, struct task_struct, rcu);

__put_task_struct(task);

}

EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);

void __init __weak arch_task_cache_init(void) { }

static void __init set_max_threads(unsigned int max_threads_suggested)

{

u64 threads;

unsigned long nr_pages = memblock_estimated_nr_free_pages();

/*

if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)

threads = MAX_THREADS;

else

threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,

(u64) THREAD_SIZE * 8UL);

if (threads > max_threads_suggested)

threads = max_threads_suggested;

max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);

}

#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT

int arch_task_struct_size __read_mostly;

static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size)

{

/* Fetch thread_struct whitelist for the architecture. */

arch_thread_struct_whitelist(offset, size);

/*

if (unlikely(*size == 0))

*offset = 0;

else

*offset += offsetof(struct task_struct, thread);

}

void __init fork_init(void)

{

int i;

#ifndef ARCH_MIN_TASKALIGN

#define ARCH_MIN_TASKALIGN 0

int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);

unsigned long useroffset, usersize;

/* create a slab on which task_structs can be allocated */

task_struct_whitelist(&useroffset, &usersize);

task_struct_cachep = kmem_cache_create_usercopy("task_struct",

arch_task_struct_size, align,

SLAB_PANIC|SLAB_ACCOUNT,

useroffset, usersize, NULL);

/* do the arch specific task caches init */

arch_task_cache_init();

set_max_threads(MAX_THREADS);

init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;

init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;

init_task.signal->rlim[RLIMIT_SIGPENDING] =

init_task.signal->rlim[RLIMIT_NPROC];

for (i = 0; i < UCOUNT_COUNTS; i++)

init_user_ns.ucount_max[i] = max_threads/2;

set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY);

set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY);

set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);

set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY);

#ifdef CONFIG_VMAP_STACK

cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",

NULL, free_vm_stack_cache);

scs_init();

lockdep_init_task(&init_task);

uprobes_init();

}

int __weak arch_dup_task_struct(struct task_struct *dst,

struct task_struct *src)

{

*dst = *src;

return 0;

}

void set_task_stack_end_magic(struct task_struct *tsk)

{

unsigned long *stackend;

stackend = end_of_stack(tsk);

*stackend = STACK_END_MAGIC; /* for overflow detection */

}

static struct task_struct *dup_task_struct(struct task_struct *orig, int node)

{

struct task_struct *tsk;

int err;

if (node == NUMA_NO_NODE)

node = tsk_fork_get_node(orig);

tsk = alloc_task_struct_node(node);

if (!tsk)

return NULL;

err = arch_dup_task_struct(tsk, orig);

if (err)

goto free_tsk;

err = alloc_thread_stack_node(tsk, node);

if (err)

goto free_tsk;

#ifdef CONFIG_THREAD_INFO_IN_TASK

refcount_set(&tsk->stack_refcount, 1);

account_kernel_stack(tsk, 1);

err = scs_prepare(tsk, node);

if (err)

goto free_stack;

#ifdef CONFIG_SECCOMP

/*

tsk->seccomp.filter = NULL;

setup_thread_stack(tsk, orig);

clear_user_return_notifier(tsk);

clear_tsk_need_resched(tsk);

set_task_stack_end_magic(tsk);

clear_syscall_work_syscall_user_dispatch(tsk);

#ifdef CONFIG_STACKPROTECTOR

tsk->stack_canary = get_random_canary();

if (orig->cpus_ptr == &orig->cpus_mask)

tsk->cpus_ptr = &tsk->cpus_mask;

dup_user_cpus_ptr(tsk, orig, node);

/*

refcount_set(&tsk->rcu_users, 2);

/* One for the rcu users */

refcount_set(&tsk->usage, 1);

#ifdef CONFIG_BLK_DEV_IO_TRACE

tsk->btrace_seq = 0;

tsk->splice_pipe = NULL;

tsk->task_frag.page = NULL;

tsk->wake_q.next = NULL;

tsk->worker_private = NULL;

kcov_task_init(tsk);

kmsan_task_create(tsk);

kmap_local_fork(tsk);

#ifdef CONFIG_FAULT_INJECTION

tsk->fail_nth = 0;

#ifdef CONFIG_BLK_CGROUP

tsk->throttle_disk = NULL;

tsk->use_memdelay = 0;

#ifdef CONFIG_ARCH_HAS_CPU_PASID

tsk->pasid_activated = 0;

#ifdef CONFIG_MEMCG

tsk->active_memcg = NULL;

#ifdef CONFIG_X86_BUS_LOCK_DETECT

tsk->reported_split_lock = 0;

#ifdef CONFIG_SCHED_MM_CID

tsk->mm_cid = -1;

tsk->last_mm_cid = -1;

tsk->mm_cid_active = 0;

tsk->migrate_from_cpu = -1;

return tsk;

free_stack:

exit_task_stack_account(tsk);

free_thread_stack(tsk);

free_tsk:

free_task_struct(tsk);

return NULL;

}

__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);

static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;

static int __init coredump_filter_setup(char *s)

{

default_dump_filter =

(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &

MMF_DUMP_FILTER_MASK;

return 1;

}

__setup("coredump_filter=", coredump_filter_setup);

#include <linux/init_task.h>

static void mm_init_aio(struct mm_struct *mm)

{

#ifdef CONFIG_AIO

spin_lock_init(&mm->ioctx_lock);

mm->ioctx_table = NULL;

}

static __always_inline void mm_clear_owner(struct mm_struct *mm,

struct task_struct *p)

{

#ifdef CONFIG_MEMCG

if (mm->owner == p)

WRITE_ONCE(mm->owner, NULL);

}