1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001
From: Phillip Lougher <phillip@squashfs.org.uk>
Date: Tue, 10 Jun 2014 21:51:52 +0100
Subject: [PATCH] mksquashfs: fix phys mem calculation for 32-bit processes on
PAE/64-bit kernels
When adding the code to base default memory usage on physical memory
(by default use 25% of physical memory), I made an oversight. I assumed
the process would be able to address 25% of physical memory.
However, for 32-bit processes running on a PAE kernel or 64-bit kernel,
25% of physical memory can easily exceed the addressible memory for a
32-bit process, e.g. if a machine has 24 GB of physical memory, the
code would asume the process could easily use 6 GB.
A 32-bit process by definition can only address 4 GB (32-bit pointers).
But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB)
on PAE kernels, a 32-bit process may only be able to address 2 GB.
So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel,
the code assumes it can address much more memory than it really can, which
means it runs out of memory.
The fix is to impose a maximum default limit on 32-bit kernels, or
otherwise to never use a value more than 25% of the address space. If
we assume the maximum address space is 2 GB, then the maximum becomes
512 MB. But, given most kernels used the 1GB/3GB split, that may be
unduely conservative, and 25% of 3 GB (756 MB) may be better. This
patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB
values. It is also the fixed default value previously used by Mksquashfs.
This patch also alters the code which imposes a maximum size. Previously
it was believed limiting to the physical memory size was adequate. But
obviously this needs to be updated to take into account a 32-bit process
may only be able to address 2 GB. In the process I've also taken the
opportunity to limit all requests to no more than 75% of physical memory.
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
---
squashfs-tools/mksquashfs.c | 78 ++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 70 insertions(+), 8 deletions(-)
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
index 86f82bb..5370ecf 100644
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -304,7 +304,7 @@ void restorefs();
struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
unsigned short get_checksum_mem(char *buff, int bytes);
-int get_physical_memory();
+void check_usable_phys_mem(int total_mem);
void prep_exit()
@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
BAD_ERROR("Queue sizes rediculously too large\n");
total_mem += fwriteq;
- if(total_mem > get_physical_memory()) {
- ERROR("Total queue sizes larger than physical memory.\n");
- ERROR("Mksquashfs will exhaust physical memory and thrash.\n");
- BAD_ERROR("Queues too large\n");
- }
+ check_usable_phys_mem(total_mem);
/*
* convert from queue size in Mbytes to queue size in
@@ -4879,6 +4875,72 @@ int get_physical_memory()
}
+void check_usable_phys_mem(int total_mem)
+{
+ /*
+ * We want to allow users to use as much of their physical
+ * memory as they wish. However, for practical reasons there are
+ * limits which need to be imposed, to protect users from themselves
+ * and to prevent people from using Mksquashfs as a DOS attack by using
+ * all physical memory. Mksquashfs uses memory to cache data from disk
+ * to optimise performance. It is pointless to ask it to use more
+ * than 75% of physical memory, as this causes thrashing and it is thus
+ * self-defeating.
+ */
+ int mem = get_physical_memory();
+
+ mem = (mem >> 1) + (mem >> 2); /* 75% */
+
+ if(total_mem > mem) {
+ ERROR("Total memory requested is more than 75%% of physical "
+ "memory.\n");
+ ERROR("Mksquashfs uses memory to cache data from disk to "
+ "optimise performance.\n");
+ ERROR("It is pointless to ask it to use more than this amount "
+ "of memory, as this\n");
+ ERROR("causes thrashing and it is thus self-defeating.\n");
+ BAD_ERROR("Requested memory size too large\n");
+ }
+
+ if(sizeof(void *) == 4 && total_mem > 2048) {
+ /*
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
+ * then the 75% physical memory limit can still easily exceed
+ * the addressable memory by this process.
+ *
+ * Due to the typical kernel/user-space split (1GB/3GB, or
+ * 2GB/2GB), we have to conservatively assume the 32-bit
+ * processes can only address 2-3GB. So refuse if the user
+ * tries to allocate more than 2GB.
+ */
+ ERROR("Total memory requested may exceed maximum "
+ "addressable memory by this process\n");
+ BAD_ERROR("Requested memory size too large\n");
+ }
+}
+
+
+int get_default_phys_mem()
+{
+ int mem = get_physical_memory() / SQUASHFS_TAKE;
+
+ if(sizeof(void *) == 4 && mem > 640) {
+ /*
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
+ * the default memory usage can exceed the addressable
+ * memory by this process.
+ * Due to the typical kernel/user-space split (1GB/3GB, or
+ * 2GB/2GB), we have to conservatively assume the 32-bit
+ * processes can only address 2-3GB. So limit the default
+ * usage to 640M, which gives room for other data.
+ */
+ mem = 640;
+ }
+
+ return mem;
+}
+
+
void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
int *fwriteq)
{
@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
#define VERSION() \
- printf("mksquashfs version 4.3 (2014/05/12)\n");\
+ printf("mksquashfs version 4.3-git (2014/06/09)\n");\
printf("copyright (C) 2014 Phillip Lougher "\
"<phillip@squashfs.org.uk>\n\n"); \
printf("This program is free software; you can redistribute it and/or"\
@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[])
int fragq;
int bwriteq;
int fwriteq;
- int total_mem = get_physical_memory() / SQUASHFS_TAKE;
+ int total_mem = get_default_phys_mem();
int progress = TRUE;
int force_progress = FALSE;
struct file_buffer **fragment = NULL;
|
