Merge e11853f into 8f4430d

criu/cr-check.c

@@ -1382,6 +1382,14 @@ static int check_ipv6_freebind(void)
 	return 0;
 }
 
+static int check_pagemap_scan(void)
+{
+	if (!kdat.has_pagemap_scan)
+		return -1;
+
+	return 0;
+}
+
 static int (*chk_feature)(void);
 
 /*
@@ -1502,6 +1510,7 @@ int cr_check(void)
 		ret |= check_openat2();
 		ret |= check_ptrace_get_rseq_conf();
 		ret |= check_ipv6_freebind();
+		ret |= check_pagemap_scan();
 
 		if (kdat.lsm == LSMTYPE__APPARMOR)
 			ret |= check_apparmor_stacking();
@@ -1623,6 +1632,7 @@ static struct feature_list feature_list[] = {
 	{ "openat2", check_openat2 },
 	{ "get_rseq_conf", check_ptrace_get_rseq_conf },
 	{ "ipv6_freebind", check_ipv6_freebind },
+	{ "pagemap_scan", check_pagemap_scan },
 	{ NULL, NULL },
 };
 

criu/include/fault-injection.h

@@ -19,6 +19,7 @@ enum faults {
 	FI_HUGE_ANON_SHMEM_ID = 132,
 	FI_CANNOT_MAP_VDSO = 133,
 	FI_CORRUPT_EXTREGS = 134,
+	FI_DONT_USE_PAGEMAP_SCAN = 135,
 	FI_MAX,
 };
 

criu/include/kerndat.h

@@ -86,6 +86,7 @@ struct kerndat_s {
 	struct __ptrace_rseq_configuration libc_rseq_conf;
 	bool has_ipv6_freebind;
 	bool has_membarrier_get_registrations;
+	bool has_pagemap_scan;
 };
 
 extern struct kerndat_s kdat;

criu/include/mem.h

@@ -7,6 +7,7 @@
 #include "pid.h"
 #include "proc_parse.h"
 #include "inventory.pb-c.h"
+#include "pagemap-cache.h"
 
 struct parasite_ctl;
 struct vm_area_list;
@@ -47,5 +48,6 @@ int open_vmas(struct pstree_item *t);
 int prepare_vmas(struct pstree_item *t, struct task_restore_args *ta);
 int unmap_guard_pages(struct pstree_item *t);
 int prepare_mappings(struct pstree_item *t);
-bool should_dump_page(VmaEntry *vmae, u64 pme);
+
+u64 should_dump_page(pmc_t *pmc, VmaEntry *vmae, u64 vaddr, bool *softdirty);
 #endif /* __CR_MEM_H__ */

criu/include/pagemap-cache.h

@@ -1,10 +1,12 @@
 #ifndef __CR_PAGEMAP_H__
 #define __CR_PAGEMAP_H__
 
+#include <stdbool.h>
 #include <sys/types.h>
 #include "int.h"
 
 #include "common/list.h"
+#include "pagemap_scan.h"
 
 struct vma_area;
 
@@ -15,9 +17,15 @@ typedef struct {
 	unsigned long start;		  /* start of area */
 	unsigned long end;		  /* end of area */
 	const struct list_head *vma_head; /* list head of VMAs we're serving */
+	int fd;				  /* file to read PMs from */
+
 	u64 *map;			  /* local buffer */
 	size_t map_len;			  /* length of a buffer */
-	int fd;				  /* file to read PMs from */
+
+	struct page_region *regs; /* buffer for the PAGEMAP_SCAN ioctl */
+	size_t regs_len;	  /* actual length of regs */
+	size_t regs_max_len;	  /* maximum length of regs */
+	size_t regs_idx;	  /* current index in the regs array */
 } pmc_t;
 
 #define PMC_INIT \
@@ -26,7 +34,8 @@ typedef struct {
 	}
 
 extern int pmc_init(pmc_t *pmc, pid_t pid, const struct list_head *vma_head, size_t size);
-extern u64 *pmc_get_map(pmc_t *pmc, const struct vma_area *vma);
+extern int pmc_get_map(pmc_t *pmc, const struct vma_area *vma);
 extern void pmc_fini(pmc_t *pmc);
+extern int pmc_fill(pmc_t *pmc, u64 start, u64 end);
 
 #endif /* __CR_PAGEMAP_H__ */

criu/include/pagemap_scan.h

@@ -0,0 +1,68 @@
+#ifndef __CR_PAGEMAP_SCAN_H__
+#define __CR_PAGEMAP_SCAN_H__
+
+#ifndef PAGEMAP_SCAN
+#include <sys/types.h>
+#include "int.h"
+
+/* Bitmasks provided in pm_scan_args masks and reported in page_region.categories. */
+#define PAGE_IS_WPALLOWED  (1 << 0)
+#define PAGE_IS_WRITTEN	   (1 << 1)
+#define PAGE_IS_FILE	   (1 << 2)
+#define PAGE_IS_PRESENT	   (1 << 3)
+#define PAGE_IS_SWAPPED	   (1 << 4)
+#define PAGE_IS_PFNZERO	   (1 << 5)
+#define PAGE_IS_HUGE	   (1 << 6)
+#define PAGE_IS_SOFT_DIRTY (1 << 7)
+
+/*
+ * struct page_region - Page region with flags
+ * @start:      Start of the region
+ * @end:        End of the region (exclusive)
+ * @categories: PAGE_IS_* category bitmask for the region
+ */
+struct page_region {
+	u64 start;
+	u64 end;
+	u64 categories;
+};
+
+#define PAGEMAP_SCAN _IOWR('f', 16, struct pm_scan_arg)
+
+/* Flags for PAGEMAP_SCAN ioctl */
+#define PM_SCAN_WP_MATCHING   (1 << 0) /* Write protect the pages matched. */
+#define PM_SCAN_CHECK_WPASYNC (1 << 1) /* Abort the scan when a non-WP-enabled page is found. */
+
+/*
+ * struct pm_scan_arg - Pagemap ioctl argument
+ * @size:               Size of the structure
+ * @flags:              Flags for the IOCTL
+ * @start:              Starting address of the region
+ * @end:                Ending address of the region
+ * @walk_end            Address where the scan stopped (written by kernel).
+ *                      walk_end == end (address tags cleared) informs that the scan completed on entire range.
+ * @vec:                Address of page_region struct array for output
+ * @vec_len:            Length of the page_region struct array
+ * @max_pages:          Optional limit for number of returned pages (0 = disabled)
+ * @category_inverted:  PAGE_IS_* categories which values match if 0 instead of 1
+ * @category_mask:      Skip pages for which any category doesn't match
+ * @category_anyof_mask: Skip pages for which no category matches
+ * @return_mask:        PAGE_IS_* categories that are to be reported in `page_region`s returned
+ */
+struct pm_scan_arg {
+	u64 size;
+	u64 flags;
+	u64 start;
+	u64 end;
+	u64 walk_end;
+	u64 vec;
+	u64 vec_len;
+	u64 max_pages;
+	u64 category_inverted;
+	u64 category_mask;
+	u64 category_anyof_mask;
+	u64 return_mask;
+};
+#endif /* PAGEMAP_SCAN */
+
+#endif /* __CR_PAGEMAP_SCAN_H__ */

criu/include/shmem.h

@@ -4,13 +4,14 @@
 #include "int.h"
 #include "common/lock.h"
 #include "images/vma.pb-c.h"
+#include "pagemap-cache.h"
 
 struct vma_area;
 
 extern int collect_shmem(int pid, struct vma_area *vma);
 extern int collect_sysv_shmem(unsigned long shmid, unsigned long size);
 extern int cr_dump_shmem(void);
-extern int add_shmem_area(pid_t pid, VmaEntry *vma, u64 *map);
+extern int add_shmem_area(pid_t pid, VmaEntry *vma, pmc_t *pmc);
 extern int fixup_sysv_shmems(void);
 extern int dump_one_memfd_shmem(int fd, unsigned long shmid, unsigned long size);
 extern int dump_one_sysv_shmem(void *addr, unsigned long size, unsigned long shmid);

criu/kerndat.c

@@ -54,6 +54,7 @@
 #include "memfd.h"
 #include "mount-v2.h"
 #include "util-caps.h"
+#include "pagemap_scan.h"
 
 struct kerndat_s kdat = {};
 volatile int dummy_var;
@@ -74,6 +75,25 @@ static int check_pagemap(void)
 		return -1;
 	}
 
+	if (ioctl(fd, PAGEMAP_SCAN, NULL) == 0) {
+		pr_err("PAGEMAP_SCAN succeeded unexpectedly\n");
+		return -1;
+	} else {
+		switch (errno) {
+		case EFAULT:
+			pr_debug("PAGEMAP_SCAN is supported\n");
+			kdat.has_pagemap_scan = true;
+			break;
+		case EINVAL:
+		case ENOTTY:
+			pr_debug("PAGEMAP_SCAN isn't supported\n");
+			break;
+		default:
+			pr_perror("PAGEMAP_SCAN failed with unexpected errno");
+			return -1;
+		}
+	}
+
 	retry = 3;
 	while (retry--) {
 		++dummy_var;