Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "elem.h"
21 : #include "import.h"
22 : #include "search.h"
23 : #include "state.h"
24 : #include "support.h"
25 : #include "parity.h"
26 : #include "stream.h"
27 : #include "handle.h"
28 : #include "io.h"
29 : #include "thermal.h"
30 : #include "raid/raid.h"
31 : #include "raid/cpu.h"
32 :
33 : /**
34 : * Configure the multithread support.
35 : *
36 : * Multi thread for write could be either faster or slower, depending
37 : * on the specific conditions. With multithreads it's likely faster
38 : * writing to disk, but you'll need to access multiple times the same data,
39 : * being potentially slower.
40 : *
41 : * Multi thread for verify is instead always generally faster,
42 : * so we enable it if possible.
43 : */
44 : #if HAVE_THREAD
45 : /* #define HAVE_MT_WRITE 1 */
46 : #define HAVE_MT_VERIFY 1
47 : #endif
48 :
49 7 : const char* lev_name(unsigned l)
50 : {
51 7 : switch (l) {
52 2 : case 0 : return "Parity";
53 1 : case 1 : return "2-Parity";
54 1 : case 2 : return "3-Parity";
55 1 : case 3 : return "4-Parity";
56 1 : case 4 : return "5-Parity";
57 1 : case 5 : return "6-Parity";
58 : }
59 :
60 0 : return "invalid";
61 : }
62 :
63 288397 : const char* lev_config_name(unsigned l)
64 : {
65 288397 : switch (l) {
66 95382 : case 0 : return "parity";
67 76385 : case 1 : return "2-parity";
68 39712 : case 2 : return "3-parity";
69 25667 : case 3 : return "4-parity";
70 25642 : case 4 : return "5-parity";
71 25609 : case 5 : return "6-parity";
72 : }
73 :
74 0 : return "invalid";
75 : }
76 :
77 7437 : static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
78 : {
79 7437 : if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
80 681 : *level = 0;
81 681 : return 0;
82 : }
83 :
84 6756 : if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
85 247 : *level = 1;
86 247 : return 0;
87 : }
88 :
89 6509 : if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
90 213 : *level = 2;
91 213 : return 0;
92 : }
93 :
94 6296 : if (strcmp(s, "4-parity") == 0) {
95 210 : *level = 3;
96 210 : return 0;
97 : }
98 :
99 6086 : if (strcmp(s, "5-parity") == 0) {
100 207 : *level = 4;
101 207 : return 0;
102 : }
103 :
104 5879 : if (strcmp(s, "6-parity") == 0) {
105 203 : *level = 5;
106 203 : return 0;
107 : }
108 :
109 5676 : if (strcmp(s, "z-parity") == 0) {
110 3 : *level = 2;
111 3 : if (mode)
112 3 : *mode = RAID_MODE_VANDERMONDE;
113 3 : return 0;
114 : }
115 :
116 5673 : return -1;
117 : }
118 :
119 291 : const char* lev_raid_name(unsigned mode, unsigned n)
120 : {
121 291 : switch (n) {
122 45 : case 1 : return "par1";
123 31 : case 2 : return "par2";
124 6 : case 3 : if (mode == RAID_MODE_CAUCHY)
125 3 : return "par3";
126 : else
127 3 : return "parz";
128 3 : case 4 : return "par4";
129 4 : case 5 : return "par5";
130 202 : case 6 : return "par6";
131 : }
132 :
133 0 : return 0;
134 : }
135 :
136 293 : void state_init(struct snapraid_state* state)
137 : {
138 : unsigned l, s, i;
139 :
140 293 : memset(&state->opt, 0, sizeof(state->opt));
141 293 : state->filter_hidden = 0;
142 293 : state->autosave = 0;
143 293 : state->need_write = 0;
144 293 : state->written = 0;
145 293 : state->checked_read = 0;
146 293 : state->block_size = 256 * KIBI; /* default 256 KiB */
147 293 : state->raid_mode = RAID_MODE_CAUCHY;
148 293 : state->file_mode = ADVISE_DEFAULT;
149 2051 : for (l = 0; l < LEV_MAX; ++l) {
150 1758 : state->parity[l].split_mac = 0;
151 15822 : for (s = 0; s < SPLIT_MAX; ++s) {
152 14064 : state->parity[l].split_map[s].path[0] = 0;
153 14064 : state->parity[l].split_map[s].uuid[0] = 0;
154 14064 : state->parity[l].split_map[s].fstype[0] = 0;
155 14064 : state->parity[l].split_map[s].fslabel[0] = 0;
156 14064 : state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
157 14064 : state->parity[l].split_map[s].device = 0;
158 : }
159 1758 : state->parity[l].smartctl[0] = 0;
160 8790 : for (i = 0; i < SMART_IGNORE_MAX; ++i)
161 7032 : state->parity[l].smartignore[i] = 0;
162 1758 : state->parity[l].total_blocks = 0;
163 1758 : state->parity[l].free_blocks = 0;
164 1758 : state->parity[l].skip_access = 0;
165 1758 : state->parity[l].tick = 0;
166 1758 : state->parity[l].cached_blocks = 0;
167 1758 : state->parity[l].is_excluded_by_filter = 0;
168 : }
169 293 : state->tick_io = 0;
170 293 : state->tick_misc = 0;
171 293 : state->tick_sched = 0;
172 293 : state->tick_raid = 0;
173 293 : state->tick_hash = 0;
174 293 : state->tick_last = os_tick();
175 293 : state->share[0] = 0;
176 293 : state->pool[0] = 0;
177 293 : state->pool_device = 0;
178 293 : state->lockfile[0] = 0;
179 293 : state->level = 1; /* default is the lowest protection */
180 293 : state->clear_past_hash = 0;
181 293 : state->no_conf = 0;
182 1465 : for (i = 0; i < SMART_IGNORE_MAX; ++i)
183 1172 : state->smartignore[i] = 0;
184 293 : state->rehash_blocks = 0;
185 293 : state->bad_blocks = 0;
186 293 : state->unsynced_blocks = 0;
187 293 : state->unscrubbed_blocks = 0;
188 293 : state->thermal_stop_gathering = 0;
189 293 : state->thermal_ambient_temperature = 0;
190 293 : state->thermal_highest_temperature = 0;
191 293 : state->thermal_first = 0;
192 293 : state->thermal_latest = 0;
193 293 : state->thermal_temperature_limit = 0;
194 293 : state->thermal_cooldown_time = 0;
195 :
196 293 : tommy_list_init(&state->disklist);
197 293 : tommy_list_init(&state->extralist);
198 293 : tommy_list_init(&state->maplist);
199 293 : tommy_list_init(&state->contentlist);
200 293 : tommy_list_init(&state->filterlist);
201 293 : tommy_list_init(&state->importlist);
202 293 : tommy_list_init(&state->thermallist);
203 293 : tommy_hashdyn_init(&state->importset);
204 293 : tommy_hashdyn_init(&state->previmportset);
205 293 : tommy_hashdyn_init(&state->searchset);
206 293 : tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
207 293 : tommy_list_init(&state->bucketlist);
208 293 : }
209 :
210 282 : void state_done(struct snapraid_state* state)
211 : {
212 282 : tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
213 282 : tommy_list_foreach(&state->extralist, (tommy_foreach_func*)extra_free);
214 282 : tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
215 282 : tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
216 282 : tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
217 282 : tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
218 282 : tommy_list_foreach(&state->thermallist, (tommy_foreach_func*)thermal_free);
219 282 : tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
220 282 : tommy_hashdyn_done(&state->importset);
221 282 : tommy_hashdyn_done(&state->previmportset);
222 282 : tommy_hashdyn_done(&state->searchset);
223 282 : tommy_arrayblkof_done(&state->infoarr);
224 282 : tommy_list_foreach(&state->bucketlist, (tommy_foreach_func*)bucket_free);
225 282 : }
226 :
227 : /**
228 : * Check the configuration.
229 : */
230 291 : static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
231 : {
232 : tommy_node* i;
233 : unsigned l, s;
234 :
235 : /* check for parity level */
236 291 : if (state->raid_mode == RAID_MODE_VANDERMONDE) {
237 3 : if (state->level > 3) {
238 : /* LCOV_EXCL_START */
239 : log_fatal(EUSER, "Using z-parity limits you to a maximum of 3 parities.\n");
240 : exit(EXIT_FAILURE);
241 : /* LCOV_EXCL_STIO */
242 : }
243 : }
244 :
245 : for (l = 0; l < state->level; ++l) {
246 : if (state->parity[l].split_mac == 0) {
247 : /* LCOV_EXCL_START */
248 : log_fatal(EUSER, "Missing '%s' specification in '%s'\n", lev_config_name(l), path);
249 : exit(EXIT_FAILURE);
250 : /* LCOV_EXCL_STOP */
251 : }
252 : }
253 :
254 291 : if (tommy_list_empty(&state->contentlist)) {
255 : /* LCOV_EXCL_START */
256 : log_fatal(EUSER, "Missing 'content' specification in '%s'\n", path);
257 : exit(EXIT_FAILURE);
258 : /* LCOV_EXCL_STOP */
259 : }
260 :
261 : /* check for equal paths */
262 1957 : for (i = state->contentlist; i != 0; i = i->next) {
263 1666 : struct snapraid_content* content = i->data;
264 :
265 10690 : for (l = 0; l < state->level; ++l) {
266 44976 : for (s = 0; s < state->parity[l].split_mac; ++s) {
267 35952 : if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
268 : /* LCOV_EXCL_START */
269 : log_fatal(EUSER, "Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
270 : exit(EXIT_FAILURE);
271 : /* LCOV_EXCL_STOP */
272 : }
273 : }
274 : }
275 : }
276 :
277 : /* check device of data disks */
278 291 : if (!state->opt.skip_device && !state->opt.skip_disk_access) {
279 66 : for (i = state->disklist; i != 0; i = i->next) {
280 : tommy_node* j;
281 60 : struct snapraid_disk* disk = i->data;
282 :
283 : /* skip data disks that are not accessible */
284 60 : if (disk->skip_access)
285 0 : continue;
286 :
287 : #ifdef _WIN32
288 : if (disk->device == 0) {
289 : /* LCOV_EXCL_START */
290 : log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", disk->dir);
291 : log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
292 : log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
293 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
294 : exit(EXIT_FAILURE);
295 : /* LCOV_EXCL_STOP */
296 : }
297 : #endif
298 :
299 330 : for (j = i->next; j != 0; j = j->next) {
300 270 : struct snapraid_disk* other = j->data;
301 270 : if (disk->device == other->device) {
302 0 : if (state->opt.force_device) {
303 : /* note that we just ignore the issue */
304 : /* and we DON'T mark the disk to be skipped */
305 : /* because we want to use these disks */
306 0 : if (!state->opt.no_warnings)
307 0 : log_fatal(EUSER, "DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
308 : } else {
309 : /* LCOV_EXCL_START */
310 : log_fatal(ESOFT, "Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
311 : #ifdef _WIN32
312 : log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->device);
313 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
314 : log_fatal(ESOFT, "to change one of the disk serial.\n");
315 : #endif
316 : /* in "fix" we allow to continue anyway */
317 : if (strcmp(state->command, "fix") == 0) {
318 : log_fatal(ESOFT, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
319 : }
320 : exit(EXIT_FAILURE);
321 : /* LCOV_EXCL_STOP */
322 : }
323 : }
324 : }
325 :
326 : /* skip data disks that are not accessible */
327 60 : if (disk->skip_access)
328 0 : continue;
329 :
330 60 : if (!state->opt.skip_parity_access) {
331 180 : for (l = 0; l < state->level; ++l) {
332 240 : for (s = 0; s < state->parity[l].split_mac; ++s) {
333 120 : if (disk->device == state->parity[l].split_map[s].device) {
334 0 : if (state->opt.force_device) {
335 : /* note that we just ignore the issue */
336 : /* and we DON'T mark the disk to be skipped */
337 : /* because we want to use these disks */
338 0 : if (!state->opt.no_warnings)
339 0 : log_fatal(EUSER, "DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
340 : } else {
341 : /* LCOV_EXCL_START */
342 : log_fatal(ESOFT, "Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
343 : #ifdef _WIN32
344 : log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->device);
345 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
346 : log_fatal(ESOFT, "to change one of the disk serial.\n");
347 : #endif
348 : exit(EXIT_FAILURE);
349 : /* LCOV_EXCL_STOP */
350 : }
351 : }
352 : }
353 : }
354 : }
355 :
356 60 : if (state->pool[0] != 0 && disk->device == state->pool_device) {
357 : /* LCOV_EXCL_START */
358 : log_fatal(ESOFT, "Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
359 : #ifdef _WIN32
360 : log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->device);
361 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
362 : log_fatal(ESOFT, "to change one of the disk serial.\n");
363 : #endif
364 : exit(EXIT_FAILURE);
365 : /* LCOV_EXCL_STOP */
366 : }
367 : }
368 : }
369 :
370 : /* check device of parity disks */
371 291 : if (!state->opt.skip_device && !state->opt.skip_parity_access) {
372 18 : for (l = 0; l < state->level; ++l) {
373 24 : for (s = 0; s < state->parity[l].split_mac; ++s) {
374 : unsigned j, t;
375 :
376 : /* skip parity disks that are not accessible */
377 12 : if (state->parity[l].skip_access)
378 0 : continue;
379 :
380 : #ifdef _WIN32
381 : if (state->parity[l].split_map[s].device == 0) {
382 : /* LCOV_EXCL_START */
383 : log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
384 : log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
385 : log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
386 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
387 : exit(EXIT_FAILURE);
388 : /* LCOV_EXCL_STOP */
389 : }
390 : #endif
391 :
392 18 : for (j = l + 1; j < state->level; ++j) {
393 12 : for (t = 0; t < state->parity[j].split_mac; ++t) {
394 6 : if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
395 0 : if (state->opt.force_device) {
396 : /* note that we just ignore the issue */
397 : /* and we DON'T mark the disk to be skipped */
398 : /* because we want to use these disks */
399 0 : if (!state->opt.no_warnings)
400 0 : log_fatal(EUSER, "DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
401 : } else {
402 : /* LCOV_EXCL_START */
403 : log_fatal(ESOFT, "Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
404 : #ifdef _WIN32
405 : log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
406 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
407 : log_fatal(ESOFT, "to change one of the disk serial.\n");
408 : #endif
409 : /* in "fix" we allow to continue anyway */
410 : if (strcmp(state->command, "fix") == 0) {
411 : log_fatal(ESOFT, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
412 : }
413 : exit(EXIT_FAILURE);
414 : /* LCOV_EXCL_STOP */
415 : }
416 : }
417 : }
418 : }
419 :
420 12 : if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
421 : /* LCOV_EXCL_START */
422 : log_fatal(ESOFT, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
423 : #ifdef _WIN32
424 : log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
425 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
426 : log_fatal(ESOFT, "to change one of the disk serial.\n");
427 : #endif
428 : exit(EXIT_FAILURE);
429 : /* LCOV_EXCL_STOP */
430 : }
431 : }
432 : }
433 : }
434 :
435 : /* check device of pool disk */
436 : #ifdef _WIN32
437 : if (!state->opt.skip_device) {
438 : if (state->pool[0] != 0 && state->pool_device == 0) {
439 : /* LCOV_EXCL_START */
440 : log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", state->pool);
441 : log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
442 : log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
443 : log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
444 : exit(EXIT_FAILURE);
445 : /* LCOV_EXCL_STOP */
446 : }
447 : }
448 : #endif
449 :
450 : /* count the content files */
451 291 : if (!state->opt.skip_device && !state->opt.skip_content_access) {
452 : unsigned content_count;
453 :
454 6 : content_count = 0;
455 30 : for (i = state->contentlist; i != 0; i = i->next) {
456 : tommy_node* j;
457 24 : struct snapraid_content* content = i->data;
458 :
459 : /* check if there are others in the same disk */
460 60 : for (j = i->next; j != 0; j = j->next) {
461 36 : struct snapraid_content* other = j->data;
462 36 : if (content->device == other->device) {
463 0 : log_fatal(ESOFT, "WARNING! Content files on the same disk: '%s' and '%s'.\n", content->content, other->content);
464 0 : break;
465 : }
466 : }
467 24 : if (j != 0) {
468 : /* skip it */
469 0 : continue;
470 : }
471 :
472 24 : ++content_count;
473 : }
474 :
475 6 : if (content_count < state->level + 1) {
476 : /* LCOV_EXCL_START */
477 : log_fatal(EUSER, "You must have at least %d 'content' files in different disks.\n", state->level + 1);
478 : exit(EXIT_FAILURE);
479 : /* LCOV_EXCL_STOP */
480 : }
481 : }
482 :
483 : /* check for speed */
484 : #ifdef CONFIG_X86
485 291 : if (!raid_cpu_has_ssse3())
486 : #endif
487 0 : if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
488 0 : if (state->level == 3) {
489 0 : log_fatal(ESOFT, "WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
490 0 : log_fatal(ESOFT, "WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
491 0 : } else if (state->level > 3) {
492 0 : log_fatal(ESOFT, "WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
493 0 : log_fatal(ESOFT, "WARNING! It's recommended to reduce the parity levels to triple parity.\n");
494 : }
495 : }
496 :
497 : /* ensure that specified filter disks are valid ones */
498 302 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
499 : tommy_node* j;
500 11 : struct snapraid_filter* filter = i->data;
501 58 : for (j = state->disklist; j != 0; j = j->next) {
502 51 : struct snapraid_disk* disk = j->data;
503 51 : if (wnmatch(filter->pattern, disk->name) == 0)
504 4 : break;
505 : }
506 11 : if (j == 0) {
507 7 : for (j = state->extralist; j != 0; j = j->next) {
508 0 : struct snapraid_extra* extra = j->data;
509 0 : if (wnmatch(filter->pattern, extra->name) == 0)
510 0 : break;
511 : }
512 : }
513 11 : if (j == 0) {
514 : /* check matching with parity disks */
515 24 : for (l = 0; l < state->level; ++l)
516 24 : if (wnmatch(filter->pattern, lev_config_name(l)) == 0)
517 7 : break;
518 7 : if (l == state->level) {
519 : /* LCOV_EXCL_START */
520 : log_fatal(EUSER, "Option -d, --filter-disk %s doesn't match any data or parity or extra disk.\n", filter->pattern);
521 : exit(EXIT_FAILURE);
522 : /* LCOV_EXCL_STOP */
523 : }
524 : }
525 : }
526 291 : }
527 :
528 : /**
529 : * Validate the smartctl command.
530 : *
531 : * It must contains only one %s string, and not other % chars.
532 : */
533 390 : static int validate_smartctl(const char* custom)
534 : {
535 390 : const char* s = custom;
536 390 : int arg = 0;
537 :
538 2340 : while (*s) {
539 1950 : if (s[0] == '%' && s[1] == 's') {
540 195 : if (arg) {
541 : /* LCOV_EXCL_START */
542 : return -1;
543 : /* LCOV_EXCL_STOP */
544 : }
545 195 : arg = 1;
546 1755 : } else if (s[0] == '%') {
547 : /* LCOV_EXCL_START */
548 : return -1;
549 : /* LCOV_EXCL_STOP */
550 : }
551 :
552 1950 : ++s;
553 : }
554 :
555 390 : return 0;
556 : }
557 :
558 292 : void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
559 : {
560 : STREAM* f;
561 : unsigned line;
562 : tommy_node* i;
563 : unsigned l, s;
564 : char esc_buffer[ESC_MAX];
565 : char esc_buffer1[ESC_MAX];
566 :
567 : /* copy the options */
568 292 : state->opt = *opt;
569 :
570 : /* if unset, sort by physical order */
571 292 : if (!state->opt.force_order)
572 7 : state->opt.force_order = SORT_PHYSICAL;
573 :
574 : /* adjust file mode */
575 292 : if (state->opt.file_mode != ADVISE_DEFAULT) {
576 285 : state->file_mode = state->opt.file_mode;
577 : } else {
578 : /* default mode, if nothing is specified */
579 7 : state->file_mode = ADVISE_DISCARD;
580 : }
581 :
582 : /* store current command */
583 292 : state->command = command;
584 :
585 292 : log_tag("conf:file:%s\n", esc_tag(path, esc_buffer));
586 :
587 292 : f = sopen_read(path, 0);
588 292 : if (!f) {
589 : /* LCOV_EXCL_START */
590 : if (errno == ENOENT) {
591 : log_fatal(errno, "No configuration file found at '%s'\n", path);
592 : } else if (errno == EACCES) {
593 : log_fatal(errno, "You do not have rights to access the configuration file '%s'\n", path);
594 : } else {
595 : log_fatal(errno, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
596 : }
597 : exit(EXIT_FAILURE);
598 : /* LCOV_EXCL_STOP */
599 : }
600 :
601 292 : line = 1;
602 6898 : while (1) {
603 : char tag[PATH_MAX];
604 : char buffer[PATH_MAX];
605 : int ret;
606 : int c;
607 : unsigned level;
608 :
609 : /* skip initial spaces */
610 7190 : sgetspace(f);
611 :
612 : /* read the command */
613 7190 : ret = sgettok(f, tag, sizeof(tag));
614 7190 : if (ret < 0) {
615 : /* LCOV_EXCL_START */
616 : log_fatal(EUSER, "Error reading the configuration file '%s' at line %u\n", path, line);
617 : exit(EXIT_FAILURE);
618 : /* LCOV_EXCL_STOP */
619 : }
620 :
621 : /* skip spaces after the command */
622 7190 : sgetspace(f);
623 :
624 7190 : if (strcmp(tag, "blocksize") == 0
625 : /* block_size is the old format of the option */
626 6904 : || strcmp(tag, "block_size") == 0) {
627 :
628 292 : ret = sgetu32(f, &state->block_size);
629 292 : if (ret < 0) {
630 : /* LCOV_EXCL_START */
631 : log_fatal(EUSER, "Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
632 : exit(EXIT_FAILURE);
633 : /* LCOV_EXCL_STOP */
634 : }
635 292 : if (state->block_size < 1) {
636 : /* LCOV_EXCL_START */
637 : log_fatal(EUSER, "Too small 'blocksize' specification in '%s' at line %u\n", path, line);
638 : exit(EXIT_FAILURE);
639 : /* LCOV_EXCL_STOP */
640 : }
641 292 : if (state->block_size > 16 * KIBI) {
642 : /* LCOV_EXCL_START */
643 : log_fatal(EUSER, "Too big 'blocksize' specification in '%s' at line %u\n", path, line);
644 : exit(EXIT_FAILURE);
645 : /* LCOV_EXCL_STOP */
646 : }
647 : /* check if it's a power of 2 */
648 292 : if ((state->block_size & (state->block_size - 1)) != 0) {
649 : /* LCOV_EXCL_START */
650 : log_fatal(EUSER, "Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
651 : exit(EXIT_FAILURE);
652 : /* LCOV_EXCL_STOP */
653 : }
654 292 : state->block_size *= KIBI;
655 6898 : } else if (strcmp(tag, "hashsize") == 0
656 6853 : || strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
657 45 : ) {
658 : uint32_t hash_size;
659 :
660 45 : ret = sgetu32(f, &hash_size);
661 45 : if (ret < 0) {
662 : /* LCOV_EXCL_START */
663 : log_fatal(EUSER, "Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
664 : exit(EXIT_FAILURE);
665 : /* LCOV_EXCL_STOP */
666 : }
667 45 : if (hash_size < 2) {
668 : /* LCOV_EXCL_START */
669 : log_fatal(EUSER, "Too small 'hashsize' specification in '%s' at line %u\n", path, line);
670 : exit(EXIT_FAILURE);
671 : /* LCOV_EXCL_STOP */
672 : }
673 45 : if (hash_size > HASH_MAX) {
674 : /* LCOV_EXCL_START */
675 : log_fatal(EUSER, "Too big 'hashsize' specification in '%s' at line %u\n", path, line);
676 : exit(EXIT_FAILURE);
677 : /* LCOV_EXCL_STOP */
678 : }
679 : /* check if it's a power of 2 */
680 45 : if ((hash_size & (hash_size - 1)) != 0) {
681 : /* LCOV_EXCL_START */
682 : log_fatal(EUSER, "Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
683 : exit(EXIT_FAILURE);
684 : /* LCOV_EXCL_STOP */
685 : }
686 :
687 45 : BLOCK_HASH_SIZE = hash_size;
688 6853 : } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
689 : char device[PATH_MAX];
690 : char* split_map[SPLIT_MAX + 1];
691 : unsigned split_mac;
692 : char* slash;
693 : uint64_t dev;
694 : int skip_access;
695 :
696 1375 : if (state->parity[level].split_mac != 0) {
697 : /* LCOV_EXCL_START */
698 : log_fatal(EUSER, "Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
699 : exit(EXIT_FAILURE);
700 : /* LCOV_EXCL_STOP */
701 : }
702 :
703 1375 : ret = sgetlasttok(f, buffer, sizeof(buffer));
704 1375 : if (ret < 0) {
705 : /* LCOV_EXCL_START */
706 : log_fatal(EUSER, "Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
707 : exit(EXIT_FAILURE);
708 : /* LCOV_EXCL_STOP */
709 : }
710 :
711 1375 : if (!*buffer) {
712 : /* LCOV_EXCL_START */
713 : log_fatal(EUSER, "Empty '%s' specification in '%s' at line %u\n", tag, path, line);
714 : exit(EXIT_FAILURE);
715 : /* LCOV_EXCL_STOP */
716 : }
717 :
718 1375 : split_mac = strsplit(split_map, SPLIT_MAX + 1, buffer, ",");
719 :
720 1375 : if (split_mac > SPLIT_MAX) {
721 : /* LCOV_EXCL_START */
722 : log_fatal(EUSER, "Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
723 : exit(EXIT_FAILURE);
724 : /* LCOV_EXCL_STOP */
725 : }
726 :
727 1375 : skip_access = 0;
728 1375 : state->parity[level].split_mac = split_mac;
729 6835 : for (s = 0; s < split_mac; ++s) {
730 : char uuid[UUID_MAX];
731 5461 : pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
732 :
733 5461 : if (!state->opt.skip_parity_access) {
734 : struct stat st;
735 :
736 : /* get the device of the directory containing the parity file */
737 4749 : pathimport(device, sizeof(device), split_map[s]);
738 4749 : slash = strrchr(device, '/');
739 4749 : if (slash)
740 4749 : *slash = 0;
741 : else
742 0 : pathcpy(device, sizeof(device), ".");
743 :
744 4749 : if (stat(device, &st) == 0) {
745 4744 : dev = st.st_dev;
746 :
747 : /* read the uuid, if unsupported use an empty one */
748 4744 : if (devuuid(dev, split_map[s], uuid, sizeof(uuid)) != 0) {
749 0 : *uuid = 0;
750 : }
751 : } else {
752 : /* if the disk can be skipped */
753 5 : if (state->opt.force_device) {
754 : /* use a fake device, and mark the disk to be skipped */
755 4 : dev = 0;
756 4 : *uuid = 0;
757 4 : skip_access = 1;
758 4 : log_fatal(errno, "DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
759 : } else {
760 : /* LCOV_EXCL_START */
761 : log_fatal(errno, "Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
762 :
763 : /* in "fix" we allow to continue anyway */
764 : if (strcmp(state->command, "fix") == 0) {
765 : log_fatal(errno, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
766 : }
767 : exit(EXIT_FAILURE);
768 : /* LCOV_EXCL_STOP */
769 : }
770 : }
771 : } else {
772 : /* use a fake device */
773 712 : dev = 0;
774 712 : *uuid = 0;
775 : }
776 :
777 5460 : state->parity[level].split_map[s].device = dev;
778 5460 : pathcpy(state->parity[level].split_map[s].uuid, sizeof(state->parity[level].split_map[s].uuid), uuid);
779 : }
780 :
781 : /* store the global parity skip_access */
782 1374 : state->parity[level].skip_access = skip_access;
783 :
784 : /* adjust the level */
785 1374 : if (state->level < level + 1)
786 1082 : state->level = level + 1;
787 5478 : } else if (strcmp(tag, "share") == 0) {
788 45 : if (*state->share) {
789 : /* LCOV_EXCL_START */
790 : log_fatal(EUSER, "Multiple 'share' specification in '%s' at line %u\n", path, line);
791 : exit(EXIT_FAILURE);
792 : /* LCOV_EXCL_STOP */
793 : }
794 :
795 45 : ret = sgetlasttok(f, buffer, sizeof(buffer));
796 45 : if (ret < 0) {
797 : /* LCOV_EXCL_START */
798 : log_fatal(EUSER, "Invalid 'share' specification in '%s' at line %u\n", path, line);
799 : exit(EXIT_FAILURE);
800 : /* LCOV_EXCL_STOP */
801 : }
802 :
803 45 : if (!*buffer) {
804 : /* LCOV_EXCL_START */
805 : log_fatal(EUSER, "Empty 'share' specification in '%s' at line %u\n", path, line);
806 : exit(EXIT_FAILURE);
807 : /* LCOV_EXCL_STOP */
808 : }
809 :
810 45 : pathimport(state->share, sizeof(state->share), buffer);
811 5433 : } else if (strcmp(tag, "pool") == 0) {
812 : struct stat st;
813 :
814 51 : if (*state->pool) {
815 : /* LCOV_EXCL_START */
816 : log_fatal(EUSER, "Multiple 'pool' specification in '%s' at line %u\n", path, line);
817 : exit(EXIT_FAILURE);
818 : /* LCOV_EXCL_STOP */
819 : }
820 :
821 51 : ret = sgetlasttok(f, buffer, sizeof(buffer));
822 51 : if (ret < 0) {
823 : /* LCOV_EXCL_START */
824 : log_fatal(EUSER, "Invalid 'pool' specification in '%s' at line %u\n", path, line);
825 : exit(EXIT_FAILURE);
826 : /* LCOV_EXCL_STOP */
827 : }
828 :
829 51 : if (!*buffer) {
830 : /* LCOV_EXCL_START */
831 : log_fatal(EUSER, "Empty 'pool' specification in '%s' at line %u\n", path, line);
832 : exit(EXIT_FAILURE);
833 : /* LCOV_EXCL_STOP */
834 : }
835 :
836 51 : pathimport(state->pool, sizeof(state->pool), buffer);
837 :
838 : /* get the device of the directory containing the pool tree */
839 51 : if (stat(buffer, &st) != 0) {
840 : /* LCOV_EXCL_START */
841 : log_fatal(errno, "Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
842 : exit(EXIT_FAILURE);
843 : /* LCOV_EXCL_STOP */
844 : }
845 :
846 51 : state->pool_device = st.st_dev;
847 5382 : } else if (strcmp(tag, "content") == 0) {
848 : struct snapraid_content* content;
849 : char device[PATH_MAX];
850 : char* slash;
851 : struct stat st;
852 : uint64_t dev;
853 :
854 1666 : ret = sgetlasttok(f, buffer, sizeof(buffer));
855 1666 : if (ret < 0) {
856 : /* LCOV_EXCL_START */
857 : log_fatal(EUSER, "Invalid 'content' specification in '%s' at line %u\n", path, line);
858 : exit(EXIT_FAILURE);
859 : /* LCOV_EXCL_STOP */
860 : }
861 :
862 1666 : if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
863 : /* LCOV_EXCL_START */
864 : log_fatal(EUSER, "You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
865 : exit(EXIT_FAILURE);
866 : /* LCOV_EXCL_STOP */
867 : }
868 :
869 1666 : if (!*buffer) {
870 : /* LCOV_EXCL_START */
871 : log_fatal(EUSER, "Empty 'content' specification in '%s' at line %u\n", path, line);
872 : exit(EXIT_FAILURE);
873 : /* LCOV_EXCL_STOP */
874 : }
875 :
876 : /* check if the content file is already specified */
877 6190 : for (i = state->contentlist; i != 0; i = i->next) {
878 4524 : content = i->data;
879 4524 : if (pathcmp(content->content, buffer) == 0)
880 0 : break;
881 : }
882 1666 : if (i) {
883 : /* LCOV_EXCL_START */
884 : log_fatal(EUSER, "Duplicate 'content' specification in '%s' at line %u\n", path, line);
885 : exit(EXIT_FAILURE);
886 : /* LCOV_EXCL_STOP */
887 : }
888 :
889 : /* get the device of the directory containing the content file */
890 1666 : pathimport(device, sizeof(device), buffer);
891 1666 : slash = strrchr(device, '/');
892 1666 : if (slash)
893 1666 : *(slash + 1) = 0;
894 : else
895 0 : pathcpy(device, sizeof(device), ".");
896 1666 : if (stat(device, &st) == 0) {
897 1665 : dev = st.st_dev;
898 : } else {
899 1 : if (state->opt.skip_content_access) {
900 : /* use a fake device */
901 1 : dev = 0;
902 1 : log_fatal(errno, "WARNING! Skipping inaccessible content file '%s'...\n", buffer);
903 : } else {
904 : /* LCOV_EXCL_START */
905 : log_fatal(errno, "Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
906 : exit(EXIT_FAILURE);
907 : /* LCOV_EXCL_STOP */
908 : }
909 : }
910 :
911 : /* set the lock file at the first accessible content file */
912 1666 : if (state->lockfile[0] == 0 && dev != 0) {
913 291 : pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
914 291 : pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
915 : }
916 :
917 1666 : content = content_alloc(buffer, dev);
918 :
919 1666 : tommy_list_insert_tail(&state->contentlist, &content->node, content);
920 5481 : } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
921 : /* "disk" is the deprecated name up to SnapRAID 9.x */
922 : char dir[PATH_MAX];
923 : char device[PATH_MAX];
924 : char uuid[UUID_MAX];
925 : struct snapraid_disk* disk;
926 : uint64_t dev;
927 : int skip_access;
928 :
929 1765 : ret = sgettok(f, buffer, sizeof(buffer));
930 1765 : if (ret < 0) {
931 : /* LCOV_EXCL_START */
932 : log_fatal(EUSER, "Invalid 'data' name specification in '%s' at line %u\n", path, line);
933 : exit(EXIT_FAILURE);
934 : /* LCOV_EXCL_STOP */
935 : }
936 :
937 1765 : if (!*buffer) {
938 : /* LCOV_EXCL_START */
939 : log_fatal(EUSER, "Empty 'data' name specification in '%s' at line %u\n", path, line);
940 : exit(EXIT_FAILURE);
941 : /* LCOV_EXCL_STOP */
942 : }
943 :
944 1765 : sgetspace(f);
945 :
946 1765 : ret = sgetlasttok(f, dir, sizeof(dir));
947 1765 : if (ret < 0) {
948 : /* LCOV_EXCL_START */
949 : log_fatal(EUSER, "Invalid 'data' dir specification in '%s' at line %u\n", path, line);
950 : exit(EXIT_FAILURE);
951 : /* LCOV_EXCL_STOP */
952 : }
953 :
954 1765 : if (!*dir) {
955 : /* LCOV_EXCL_START */
956 : log_fatal(EUSER, "Empty 'data' dir specification in '%s' at line %u\n", path, line);
957 : exit(EXIT_FAILURE);
958 : /* LCOV_EXCL_STOP */
959 : }
960 :
961 : /* get the device of the dir */
962 1765 : pathimport(device, sizeof(device), dir);
963 :
964 : /* check if the disk name already exists */
965 6285 : for (i = state->disklist; i != 0; i = i->next) {
966 4520 : disk = i->data;
967 4520 : if (strcmp(disk->name, buffer) == 0)
968 0 : break;
969 : }
970 1765 : if (i) {
971 : /* LCOV_EXCL_START */
972 : log_fatal(EUSER, "Duplicate 'data' name '%s' at line %u\n", buffer, line);
973 : exit(EXIT_FAILURE);
974 : /* LCOV_EXCL_STOP */
975 : }
976 :
977 : /* if the disk has to be present */
978 1765 : skip_access = 0;
979 1765 : if (!state->opt.skip_disk_access) {
980 : struct stat st;
981 :
982 1609 : if (stat(device, &st) == 0) {
983 1608 : dev = st.st_dev;
984 :
985 : /* read the uuid, if unsupported use an empty one */
986 1608 : if (devuuid(dev, dir, uuid, sizeof(uuid)) != 0) {
987 0 : *uuid = 0;
988 : }
989 :
990 : /* fake a different UUID when testing */
991 1608 : if (state->opt.fake_uuid) {
992 2 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
993 2 : --state->opt.fake_uuid;
994 : }
995 : } else {
996 : /* if the disk can be skipped */
997 1 : if (state->opt.force_device) {
998 : /* use a fake device, and mark the disk to be skipped */
999 1 : dev = 0;
1000 1 : *uuid = 0;
1001 1 : skip_access = 1;
1002 1 : log_fatal(errno, "DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
1003 : } else {
1004 : /* LCOV_EXCL_START */
1005 : log_fatal(errno, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
1006 :
1007 : /* in "fix" we allow to continue anyway */
1008 : if (strcmp(state->command, "fix") == 0) {
1009 : log_fatal(errno, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
1010 : }
1011 : exit(EXIT_FAILURE);
1012 : /* LCOV_EXCL_STOP */
1013 : }
1014 : }
1015 : } else {
1016 : /* use a fake device */
1017 156 : dev = 0;
1018 156 : *uuid = 0;
1019 : }
1020 :
1021 1765 : disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
1022 :
1023 1765 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1024 1951 : } else if (strcmp(tag, "extra") == 0) {
1025 : char dir[PATH_MAX];
1026 : char device[PATH_MAX];
1027 : char uuid[UUID_MAX];
1028 : struct snapraid_extra* extra;
1029 : uint64_t dev;
1030 :
1031 12 : ret = sgettok(f, buffer, sizeof(buffer));
1032 12 : if (ret < 0) {
1033 : /* LCOV_EXCL_START */
1034 : log_fatal(EUSER, "Invalid 'extra' name specification in '%s' at line %u\n", path, line);
1035 : exit(EXIT_FAILURE);
1036 : /* LCOV_EXCL_STOP */
1037 : }
1038 :
1039 12 : if (!*buffer) {
1040 : /* LCOV_EXCL_START */
1041 : log_fatal(EUSER, "Empty 'extra' name specification in '%s' at line %u\n", path, line);
1042 : exit(EXIT_FAILURE);
1043 : /* LCOV_EXCL_STOP */
1044 : }
1045 :
1046 12 : sgetspace(f);
1047 :
1048 12 : ret = sgetlasttok(f, dir, sizeof(dir));
1049 12 : if (ret < 0) {
1050 : /* LCOV_EXCL_START */
1051 : log_fatal(EUSER, "Invalid 'extra' dir specification in '%s' at line %u\n", path, line);
1052 : exit(EXIT_FAILURE);
1053 : /* LCOV_EXCL_STOP */
1054 : }
1055 :
1056 12 : if (!*dir) {
1057 : /* LCOV_EXCL_START */
1058 : log_fatal(EUSER, "Empty 'extra' dir specification in '%s' at line %u\n", path, line);
1059 : exit(EXIT_FAILURE);
1060 : /* LCOV_EXCL_STOP */
1061 : }
1062 :
1063 : /* get the device of the dir */
1064 12 : pathimport(device, sizeof(device), dir);
1065 :
1066 : /* check if the disk name already exists */
1067 18 : for (i = state->extralist; i != 0; i = i->next) {
1068 6 : extra = i->data;
1069 6 : if (strcmp(extra->name, buffer) == 0)
1070 0 : break;
1071 : }
1072 12 : if (i) {
1073 : /* LCOV_EXCL_START */
1074 : log_fatal(EUSER, "Duplicate 'extra' name '%s' at line %u\n", buffer, line);
1075 : exit(EXIT_FAILURE);
1076 : /* LCOV_EXCL_STOP */
1077 : }
1078 :
1079 : /* if the disk has to be present */
1080 12 : if (!state->opt.skip_disk_access) {
1081 : struct stat st;
1082 :
1083 12 : if (stat(device, &st) == 0) {
1084 12 : dev = st.st_dev;
1085 :
1086 : /* read the uuid, if unsupported use an empty one */
1087 12 : if (devuuid(dev, dir, uuid, sizeof(uuid)) != 0) {
1088 0 : *uuid = 0;
1089 : }
1090 :
1091 : /* fake a different UUID when testing */
1092 12 : if (state->opt.fake_uuid) {
1093 0 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
1094 0 : --state->opt.fake_uuid;
1095 : }
1096 : } else {
1097 : /* use a fake device, and mark the disk to be skipped */
1098 0 : dev = 0;
1099 0 : *uuid = 0;
1100 : }
1101 : } else {
1102 : /* use a fake device */
1103 0 : dev = 0;
1104 0 : *uuid = 0;
1105 : }
1106 :
1107 12 : extra = extra_alloc(buffer, dir, dev, uuid);
1108 :
1109 12 : tommy_list_insert_tail(&state->extralist, &extra->node, extra);
1110 1939 : } else if (strcmp(tag, "smartctl") == 0) {
1111 : char custom[PATH_MAX];
1112 :
1113 390 : ret = sgettok(f, buffer, sizeof(buffer));
1114 390 : if (ret < 0) {
1115 : /* LCOV_EXCL_START */
1116 : log_fatal(EUSER, "Invalid 'smartctl' name in '%s' at line %u\n", path, line);
1117 : exit(EXIT_FAILURE);
1118 : /* LCOV_EXCL_STOP */
1119 : }
1120 :
1121 390 : if (!*buffer) {
1122 : /* LCOV_EXCL_START */
1123 : log_fatal(EUSER, "Empty 'smartctl' name in '%s' at line %u\n", path, line);
1124 : exit(EXIT_FAILURE);
1125 : /* LCOV_EXCL_STOP */
1126 : }
1127 :
1128 390 : sgetspace(f);
1129 :
1130 390 : ret = sgetlasttok(f, custom, sizeof(custom));
1131 390 : if (ret < 0) {
1132 : /* LCOV_EXCL_START */
1133 : log_fatal(EUSER, "Invalid 'smartctl' option in '%s' at line %u\n", path, line);
1134 : exit(EXIT_FAILURE);
1135 : /* LCOV_EXCL_STOP */
1136 : }
1137 :
1138 390 : if (!*custom) {
1139 : /* LCOV_EXCL_START */
1140 : log_fatal(EUSER, "Empty 'smartctl' option in '%s' at line %u\n", path, line);
1141 : exit(EXIT_FAILURE);
1142 : /* LCOV_EXCL_STOP */
1143 : }
1144 :
1145 390 : if (validate_smartctl(custom) != 0) {
1146 : /* LCOV_EXCL_START */
1147 : log_fatal(EUSER, "Invalid 'smartctl' option in '%s' at line %u\n", path, line);
1148 : exit(EXIT_FAILURE);
1149 : /* LCOV_EXCL_STOP */
1150 : }
1151 :
1152 : /* search for parity */
1153 390 : if (lev_config_scan(buffer, &level, 0) == 0) {
1154 195 : if (state->parity[level].smartctl[0] != 0) {
1155 : /* LCOV_EXCL_START */
1156 : log_fatal(EUSER, "Duplicate parity 'smartctl' '%s' at line %u\n", buffer, line);
1157 : exit(EXIT_FAILURE);
1158 : /* LCOV_EXCL_STOP */
1159 : }
1160 :
1161 195 : pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1162 : } else {
1163 : /* search the disk */
1164 195 : struct snapraid_disk* found_disk = 0;
1165 195 : for (i = state->disklist; i != 0; i = i->next) {
1166 195 : struct snapraid_disk* disk = i->data;
1167 195 : if (strcmp(disk->name, buffer) == 0) {
1168 195 : found_disk = disk;
1169 195 : break;
1170 : }
1171 : }
1172 195 : struct snapraid_extra* found_extra = 0;
1173 195 : for (i = state->extralist; i != 0; i = i->next) {
1174 0 : struct snapraid_extra* extra = i->data;
1175 0 : if (strcmp(extra->name, buffer) == 0) {
1176 0 : found_extra = extra;
1177 0 : break;
1178 : }
1179 : }
1180 195 : if (found_disk) {
1181 195 : if (found_disk->smartctl[0] != 0) {
1182 : /* LCOV_EXCL_START */
1183 : log_fatal(EUSER, "Duplicate 'smartctl' name '%s' at line %u\n", buffer, line);
1184 : exit(EXIT_FAILURE);
1185 : /* LCOV_EXCL_STOP */
1186 : }
1187 :
1188 195 : pathcpy(found_disk->smartctl, sizeof(found_disk->smartctl), custom);
1189 0 : } else if (found_extra) {
1190 0 : if (found_extra->smartctl[0] != 0) {
1191 : /* LCOV_EXCL_START */
1192 : log_fatal(EUSER, "Duplicate 'smartctl' name '%s' at line %u\n", buffer, line);
1193 : exit(EXIT_FAILURE);
1194 : /* LCOV_EXCL_STOP */
1195 : }
1196 :
1197 0 : pathcpy(found_extra->smartctl, sizeof(found_extra->smartctl), custom);
1198 : } else {
1199 : /* LCOV_EXCL_START */
1200 : log_fatal(EUSER, "Unknown 'smartctl' name '%s' at line %u\n", buffer, line);
1201 : exit(EXIT_FAILURE);
1202 : /* LCOV_EXCL_STOP */
1203 : }
1204 : }
1205 1549 : } else if (strcmp(tag, "smartignore") == 0) {
1206 : int* smart;
1207 :
1208 388 : ret = sgettok(f, buffer, sizeof(buffer));
1209 388 : if (ret < 0) {
1210 : /* LCOV_EXCL_START */
1211 : log_fatal(EUSER, "Invalid 'smartignore' name in '%s' at line %u\n", path, line);
1212 : exit(EXIT_FAILURE);
1213 : /* LCOV_EXCL_STOP */
1214 : }
1215 :
1216 388 : if (!*buffer) {
1217 : /* LCOV_EXCL_START */
1218 : log_fatal(EUSER, "Empty 'smartignore' name in '%s' at line %u\n", path, line);
1219 : exit(EXIT_FAILURE);
1220 : /* LCOV_EXCL_STOP */
1221 : }
1222 :
1223 388 : sgetspace(f);
1224 :
1225 388 : if (strcmp(buffer, "*") == 0) {
1226 194 : smart = state->smartignore;
1227 194 : } else if (lev_config_scan(buffer, &level, 0) == 0) { /* search for parity */
1228 194 : smart = state->parity[level].smartignore;
1229 : } else {
1230 : /* search the disk */
1231 0 : struct snapraid_disk* found_disk = 0;
1232 0 : for (i = state->disklist; i != 0; i = i->next) {
1233 0 : struct snapraid_disk* disk = i->data;
1234 0 : if (strcmp(disk->name, buffer) == 0) {
1235 0 : found_disk = disk;
1236 0 : break;
1237 : }
1238 : }
1239 0 : struct snapraid_extra* found_extra = 0;
1240 0 : for (i = state->extralist; i != 0; i = i->next) {
1241 0 : struct snapraid_extra* extra = i->data;
1242 0 : if (strcmp(extra->name, buffer) == 0) {
1243 0 : found_extra = extra;
1244 0 : break;
1245 : }
1246 : }
1247 0 : if (found_disk) {
1248 0 : smart = found_disk->smartignore;
1249 0 : } else if (found_extra) {
1250 0 : smart = found_extra->smartignore;
1251 : } else {
1252 : /* LCOV_EXCL_START */
1253 : log_fatal(EUSER, "Unknown 'smartignore' name '%s' at line %u\n", buffer, line);
1254 : exit(EXIT_FAILURE);
1255 : /* LCOV_EXCL_STOP */
1256 : }
1257 : }
1258 :
1259 388 : int si = 0;
1260 :
1261 388 : while (1) {
1262 : char* e;
1263 :
1264 776 : ret = sgettok(f, buffer, sizeof(buffer));
1265 776 : if (ret < 0) {
1266 : /* LCOV_EXCL_START */
1267 : log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1268 : exit(EXIT_FAILURE);
1269 : /* LCOV_EXCL_STOP */
1270 : }
1271 :
1272 776 : if (!*buffer)
1273 388 : break;
1274 :
1275 388 : if (si == SMART_IGNORE_MAX) {
1276 : /* LCOV_EXCL_START */
1277 : log_fatal(EUSER, "Too many 'smartignore' specification in '%s' at line %u\n", path, line);
1278 : exit(EXIT_FAILURE);
1279 : /* LCOV_EXCL_STOP */
1280 : }
1281 :
1282 388 : smart[si] = strtoul(buffer, &e, 0);
1283 :
1284 388 : if (!e || *e) {
1285 : /* LCOV_EXCL_START */
1286 : log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1287 : exit(EXIT_FAILURE);
1288 : /* LCOV_EXCL_STOP */
1289 : }
1290 :
1291 388 : ++si;
1292 :
1293 388 : sgetspace(f);
1294 : }
1295 :
1296 388 : if (si == 0) {
1297 : /* LCOV_EXCL_START */
1298 : log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1299 : exit(EXIT_FAILURE);
1300 : /* LCOV_EXCL_STOP */
1301 : }
1302 1161 : } else if (strcmp(tag, "temp_limit") == 0) {
1303 : unsigned temp;
1304 :
1305 7 : ret = sgetu32(f, &temp);
1306 7 : if (ret < 0) {
1307 : /* LCOV_EXCL_START */
1308 : log_fatal(EUSER, "Invalid 'temp_limit' specification in '%s' at line %u\n", path, line);
1309 : exit(EXIT_FAILURE);
1310 : /* LCOV_EXCL_STOP */
1311 : }
1312 7 : if (temp < 40 || temp > 70) {
1313 : /* LCOV_EXCL_START */
1314 : log_fatal(EUSER, "Invalid 'temp_limit' temperature specification in '%s' at line %u. It must be between 40 and 70\n", path, line);
1315 : exit(EXIT_FAILURE);
1316 : /* LCOV_EXCL_STOP */
1317 : }
1318 :
1319 7 : state->thermal_temperature_limit = temp;
1320 1154 : } else if (strcmp(tag, "temp_sleep") == 0) {
1321 : unsigned time;
1322 :
1323 7 : ret = sgetu32(f, &time);
1324 7 : if (ret < 0) {
1325 : /* LCOV_EXCL_START */
1326 : log_fatal(EUSER, "Invalid 'temp_sleep' specification in '%s' at line %u\n", path, line);
1327 : exit(EXIT_FAILURE);
1328 : /* LCOV_EXCL_STOP */
1329 : }
1330 7 : if (time < 3 || time > 120) {
1331 : /* LCOV_EXCL_START */
1332 : log_fatal(EUSER, "Invalid 'temp_sleep' temperature specification in '%s' at line %u. It must be between 3 and 120\n", path, line);
1333 : exit(EXIT_FAILURE);
1334 : /* LCOV_EXCL_STOP */
1335 : }
1336 :
1337 7 : state->thermal_cooldown_time = time * 60;
1338 1147 : } else if (strcmp(tag, "nohidden") == 0) {
1339 0 : state->filter_hidden = 1;
1340 1147 : } else if (strcmp(tag, "exclude") == 0) {
1341 : struct snapraid_filter* filter;
1342 :
1343 315 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1344 315 : if (ret < 0) {
1345 : /* LCOV_EXCL_START */
1346 : log_fatal(EUSER, "Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1347 : exit(EXIT_FAILURE);
1348 : /* LCOV_EXCL_STOP */
1349 : }
1350 :
1351 315 : if (!*buffer) {
1352 : /* LCOV_EXCL_START */
1353 : log_fatal(EUSER, "Empty 'exclude' specification in '%s' at line %u\n", path, line);
1354 : exit(EXIT_FAILURE);
1355 : /* LCOV_EXCL_STOP */
1356 : }
1357 :
1358 315 : filter = filter_alloc_file(-1, "", buffer);
1359 315 : if (!filter) {
1360 : /* LCOV_EXCL_START */
1361 : log_fatal(EUSER, "Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1362 : exit(EXIT_FAILURE);
1363 : /* LCOV_EXCL_STOP */
1364 : }
1365 315 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1366 832 : } else if (strcmp(tag, "include") == 0) {
1367 : struct snapraid_filter* filter;
1368 :
1369 285 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1370 285 : if (ret < 0) {
1371 : /* LCOV_EXCL_START */
1372 : log_fatal(EUSER, "Invalid 'include' specification in '%s' at line %u\n", path, line);
1373 : exit(EXIT_FAILURE);
1374 : /* LCOV_EXCL_STOP */
1375 : }
1376 :
1377 285 : if (!*buffer) {
1378 : /* LCOV_EXCL_START */
1379 : log_fatal(EUSER, "Empty 'include' specification in '%s' at line %u\n", path, line);
1380 : exit(EXIT_FAILURE);
1381 : /* LCOV_EXCL_STOP */
1382 : }
1383 :
1384 285 : filter = filter_alloc_file(1, "", buffer);
1385 285 : if (!filter) {
1386 : /* LCOV_EXCL_START */
1387 : log_fatal(EUSER, "Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1388 : exit(EXIT_FAILURE);
1389 : /* LCOV_EXCL_STOP */
1390 : }
1391 285 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1392 547 : } else if (strcmp(tag, "autosave") == 0) {
1393 : char* e;
1394 :
1395 51 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1396 51 : if (ret < 0) {
1397 : /* LCOV_EXCL_START */
1398 : log_fatal(EUSER, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1399 : exit(EXIT_FAILURE);
1400 : /* LCOV_EXCL_STOP */
1401 : }
1402 :
1403 51 : if (!*buffer) {
1404 : /* LCOV_EXCL_START */
1405 : log_fatal(EUSER, "Empty 'autosave' specification in '%s' at line %u\n", path, line);
1406 : exit(EXIT_FAILURE);
1407 : /* LCOV_EXCL_STOP */
1408 : }
1409 :
1410 51 : state->autosave = strtoul(buffer, &e, 0);
1411 :
1412 51 : if (!e || *e) {
1413 : /* LCOV_EXCL_START */
1414 : log_fatal(EUSER, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1415 : exit(EXIT_FAILURE);
1416 : /* LCOV_EXCL_STOP */
1417 : }
1418 :
1419 : /* convert to GB */
1420 51 : state->autosave *= GIGA;
1421 496 : } else if (tag[0] == 0) {
1422 : /* allow empty lines */
1423 105 : } else if (tag[0] == '#') {
1424 105 : ret = sgetline(f, buffer, sizeof(buffer));
1425 105 : if (ret < 0) {
1426 : /* LCOV_EXCL_START */
1427 : log_fatal(EUSER, "Invalid comment in '%s' at line %u\n", path, line);
1428 : exit(EXIT_FAILURE);
1429 : /* LCOV_EXCL_STOP */
1430 : }
1431 : } else {
1432 : /* LCOV_EXCL_START */
1433 : log_fatal(EUSER, "Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1434 : exit(EXIT_FAILURE);
1435 : /* LCOV_EXCL_STOP */
1436 : }
1437 :
1438 : /* skip final spaces */
1439 7189 : sgetspace(f);
1440 :
1441 : /* next line */
1442 7189 : c = sgeteol(f);
1443 7189 : if (c == EOF) {
1444 291 : break;
1445 : }
1446 6898 : if (c != '\n') {
1447 : /* LCOV_EXCL_START */
1448 : log_fatal(EUSER, "Extra data in '%s' at line %u\n", path, line);
1449 : exit(EXIT_FAILURE);
1450 : /* LCOV_EXCL_STOP */
1451 : }
1452 6898 : ++line;
1453 : }
1454 :
1455 291 : if (serror(f)) {
1456 : /* LCOV_EXCL_START */
1457 : log_fatal(errno, "Error reading the configuration file '%s' at line %u\n", path, line);
1458 : exit(EXIT_FAILURE);
1459 : /* LCOV_EXCL_STOP */
1460 : }
1461 :
1462 291 : sclose(f);
1463 :
1464 291 : state_config_check(state, path, filterlist_disk);
1465 :
1466 : /* select the default hash */
1467 291 : if (state->opt.force_murmur3) {
1468 1 : state->besthash = HASH_MURMUR3;
1469 290 : } else if (state->opt.force_spooky2) {
1470 1 : state->besthash = HASH_SPOOKY2;
1471 : } else {
1472 289 : state->besthash = membesthash();
1473 : }
1474 :
1475 : /* by default use the best hash */
1476 291 : state->hash = state->besthash;
1477 :
1478 : /* by default use a random hash seed */
1479 291 : if (randomize(state->hashseed, HASH_MAX) != 0) {
1480 : /* LCOV_EXCL_START */
1481 : log_fatal(errno, "Failed to retrieve random values.\n");
1482 : exit(EXIT_FAILURE);
1483 : /* LCOV_EXCL_STOP */
1484 : }
1485 :
1486 : /* no previous hash by default */
1487 291 : state->prevhash = HASH_UNDEFINED;
1488 :
1489 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1490 :
1491 291 : log_tag("blocksize:%u\n", state->block_size);
1492 2056 : for (i = state->disklist; i != 0; i = i->next) {
1493 1765 : struct snapraid_disk* disk = i->data;
1494 1765 : log_tag("data:%s:%s:%s\n", esc_tag(disk->name, esc_buffer), esc_tag(disk->dir, esc_buffer1), disk->uuid);
1495 : }
1496 303 : for (i = state->extralist; i != 0; i = i->next) {
1497 12 : struct snapraid_extra* extra = i->data;
1498 12 : log_tag("extra:%s:%s:%s\n", esc_tag(extra->name, esc_buffer), esc_tag(extra->dir, esc_buffer1), extra->uuid);
1499 : }
1500 :
1501 291 : log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1502 1660 : for (l = 0; l < state->level; ++l) {
1503 1369 : log_tag("%s:%s:%s\n", lev_config_name(l), esc_tag(state->parity[l].split_map[0].path, esc_buffer), state->parity[l].split_map[0].uuid);
1504 5440 : for (s = 1; s < state->parity[l].split_mac; ++s) {
1505 4071 : log_tag("%s/%u:%s:%s\n", lev_config_name(l), s, esc_tag(state->parity[l].split_map[s].path, esc_buffer), state->parity[l].split_map[s].uuid);
1506 : }
1507 : }
1508 291 : if (state->pool[0] != 0)
1509 51 : log_tag("pool:%s\n", esc_tag(state->pool, esc_buffer));
1510 291 : if (state->share[0] != 0)
1511 45 : log_tag("share:%s\n", esc_tag(state->share, esc_buffer));
1512 291 : if (state->autosave != 0)
1513 51 : log_tag("autosave:%" PRIu64 "\n", state->autosave);
1514 891 : for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1515 : char out[PATH_MAX];
1516 600 : struct snapraid_filter* filter = i->data;
1517 600 : log_tag("filter:%s\n", esc_tag(filter_type(filter, out, sizeof(out)), esc_buffer));
1518 : }
1519 291 : if (state->filter_hidden)
1520 0 : log_tag("filter:nohidden:\n");
1521 291 : log_flush();
1522 291 : }
1523 :
1524 : /**
1525 : * Find a disk by name.
1526 : */
1527 7748 : static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1528 : {
1529 : tommy_node* i;
1530 :
1531 27039 : for (i = state->disklist; i != 0; i = i->next) {
1532 27031 : struct snapraid_disk* disk = i->data;
1533 27031 : if (strcmp(disk->name, name) == 0)
1534 7740 : return disk;
1535 : }
1536 :
1537 8 : if (state->no_conf) {
1538 : /* without a configuration file, add disks automatically */
1539 : struct snapraid_disk* disk;
1540 :
1541 6 : disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1542 :
1543 6 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1544 :
1545 6 : return disk;
1546 : }
1547 :
1548 2 : return 0;
1549 : }
1550 :
1551 : /**
1552 : * Find a disk by UUID.
1553 : */
1554 2 : static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1555 : {
1556 : tommy_node* i;
1557 2 : struct snapraid_disk* found = 0;
1558 :
1559 : /* special test case to find the first matching UUID */
1560 : /* when testing UUID are all equal or not supported */
1561 : /* and we should handle this case specifically */
1562 2 : if (state->opt.match_first_uuid)
1563 2 : return state->disklist->data;
1564 :
1565 : /* LCOV_EXCL_START */
1566 : /* never find an empty uuid */
1567 : if (!*uuid)
1568 : return 0;
1569 :
1570 : for (i = state->disklist; i != 0; i = i->next) {
1571 : struct snapraid_disk* disk = i->data;
1572 : if (strcmp(disk->uuid, uuid) == 0) {
1573 : /* never match duplicate UUID */
1574 : if (found)
1575 : return 0;
1576 : found = disk;
1577 : }
1578 : }
1579 :
1580 : return found;
1581 : /* LCOV_EXCL_STOP */
1582 : }
1583 :
1584 : /**
1585 : * Update the disk mapping if required.
1586 : */
1587 279 : static void state_map(struct snapraid_state* state)
1588 : {
1589 : unsigned hole;
1590 : tommy_node* i;
1591 : unsigned uuid_mismatch;
1592 : unsigned diskcount;
1593 : unsigned l, s;
1594 :
1595 : /* remove all the mapping without a disk */
1596 : /* this happens when a disk is removed from the configuration file */
1597 : /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1598 : /* when saving the content file, but we keep this code to import older content files. */
1599 1790 : for (i = state->maplist; i != 0; ) {
1600 1511 : struct snapraid_map* map = i->data;
1601 : struct snapraid_disk* disk;
1602 :
1603 1511 : disk = find_disk_by_name(state, map->name);
1604 :
1605 : /* go to the next mapping before removing */
1606 1511 : i = i->next;
1607 :
1608 1511 : if (disk == 0) {
1609 : /* disk not found, remove the mapping */
1610 0 : tommy_list_remove_existing(&state->maplist, &map->node);
1611 0 : map_free(map);
1612 : }
1613 : }
1614 :
1615 : /* maps each unmapped disk present in the configuration file in the first available hole */
1616 : /* this happens when you add disks for the first time in the configuration file */
1617 279 : hole = 0; /* first position to try */
1618 1948 : for (i = state->disklist; i != 0; i = i->next) {
1619 1669 : struct snapraid_disk* disk = i->data;
1620 : struct snapraid_map* map;
1621 : tommy_node* j;
1622 :
1623 : /* check if the disk is already mapped */
1624 5829 : for (j = state->maplist; j != 0; j = j->next) {
1625 5671 : map = j->data;
1626 5671 : if (strcmp(disk->name, map->name) == 0) {
1627 : /* mapping found */
1628 1511 : break;
1629 : }
1630 : }
1631 1669 : if (j != 0) {
1632 : /* mapping is present, then copy the free blocks into to disk */
1633 1511 : disk->total_blocks = map->total_blocks;
1634 1511 : disk->free_blocks = map->free_blocks;
1635 1511 : continue;
1636 : }
1637 :
1638 : /* mapping not found, search for an hole */
1639 : while (1) {
1640 1100 : for (j = state->maplist; j != 0; j = j->next) {
1641 942 : map = j->data;
1642 942 : if (map->position == hole) {
1643 : /* position already used */
1644 158 : break;
1645 : }
1646 : }
1647 316 : if (j == 0) {
1648 : /* hole found */
1649 158 : break;
1650 : }
1651 :
1652 : /* try with the next one */
1653 158 : ++hole;
1654 : }
1655 :
1656 : /* insert the new mapping */
1657 158 : map = map_alloc(disk->name, hole, 0, 0, "");
1658 :
1659 158 : tommy_list_insert_tail(&state->maplist, &map->node, map);
1660 : }
1661 :
1662 : /* without configuration don't check for number of data disks or uuid changes */
1663 279 : if (state->no_conf)
1664 1 : return;
1665 :
1666 : /* counter for the number of UUID mismatches */
1667 278 : uuid_mismatch = 0;
1668 :
1669 : /* check if mapping match the disk uuid */
1670 278 : if (!state->opt.skip_disk_access) {
1671 1759 : for (i = state->maplist; i != 0; i = i->next) {
1672 1507 : struct snapraid_map* map = i->data;
1673 : struct snapraid_disk* disk;
1674 :
1675 1507 : disk = find_disk_by_name(state, map->name);
1676 1507 : if (disk == 0) {
1677 : /* LCOV_EXCL_START */
1678 : log_fatal(EINTERNAL, "Internal inconsistency: Mapping not found for '%s'\n", map->name);
1679 : os_abort();
1680 : /* LCOV_EXCL_STOP */
1681 : }
1682 :
1683 1507 : if (disk->has_unsupported_uuid) {
1684 : /* if uuid is not available, skip this one */
1685 1 : continue;
1686 : }
1687 :
1688 : /* if the uuid is changed */
1689 1506 : if (strcmp(disk->uuid, map->uuid) != 0) {
1690 : /* mark the disk as with an UUID change */
1691 113 : disk->has_different_uuid = 1;
1692 :
1693 : /* if the previous uuid is available */
1694 113 : if (map->uuid[0] != 0) {
1695 : /* count the number of uuid change */
1696 4 : ++uuid_mismatch;
1697 4 : log_fatal(ESOFT, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1698 : } else {
1699 : /* no message here, because having a disk without */
1700 : /* UUID is the normal state of an empty disk */
1701 109 : disk->had_empty_uuid = 1;
1702 : }
1703 :
1704 : /* update the uuid in the mapping, */
1705 113 : pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1706 :
1707 : /* write the new state with the new uuid */
1708 113 : state->need_write = 1;
1709 : }
1710 : }
1711 : }
1712 :
1713 : /* check the parity uuid */
1714 278 : if (!state->opt.skip_parity_access) {
1715 1377 : for (l = 0; l < state->level; ++l) {
1716 5685 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1717 : char uuid[UUID_MAX];
1718 : int ret;
1719 :
1720 4548 : ret = devuuid(state->parity[l].split_map[s].device, state->parity[l].split_map[s].path, uuid, sizeof(uuid));
1721 :
1722 4548 : if (ret != 0) {
1723 : /* uuid not available, just ignore */
1724 4 : continue;
1725 : }
1726 :
1727 : /* if the uuid is changed */
1728 4544 : if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1729 : /* if the previous uuid is available */
1730 0 : if (state->parity[l].split_map[s].uuid[0] != 0) {
1731 : /* count the number of uuid change */
1732 0 : ++uuid_mismatch;
1733 0 : log_fatal(ESOFT, "UUID change for parity '%s/%u' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1734 : }
1735 :
1736 : /* update the uuid */
1737 0 : pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1738 :
1739 : /* write the new state with the new uuid */
1740 0 : state->need_write = 1;
1741 : }
1742 : }
1743 : }
1744 : }
1745 :
1746 278 : if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1747 : /* LCOV_EXCL_START */
1748 : log_fatal(ESOFT, "Too many disks have changed UUIDs since the last 'sync'.\n");
1749 : log_fatal(ESOFT, "If this happened because you actually replaced them,\n");
1750 : log_fatal(ESOFT, "you can still '%s', using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1751 : log_fatal(ESOFT, "Alternatively, you may have misconfigured the disk mount points,\n");
1752 : log_fatal(ESOFT, "and you need to restore the mount points to the state of the latest sync.\n");
1753 : exit(EXIT_FAILURE);
1754 : /* LCOV_EXCL_STOP */
1755 : }
1756 :
1757 : /* count the number of data disks, including holes left after removing some */
1758 278 : diskcount = 0;
1759 1941 : for (i = state->maplist; i != 0; i = i->next) {
1760 1663 : struct snapraid_map* map = i->data;
1761 :
1762 1663 : if (map->position + 1 > diskcount)
1763 1475 : diskcount = map->position + 1;
1764 : }
1765 :
1766 : /* ensure to don't go over the limit of the RAID engine */
1767 278 : if (diskcount > RAID_DATA_MAX) {
1768 : /* LCOV_EXCL_START */
1769 : log_fatal(ESOFT, "Too many data disks. Maximum allowed is %u.\n", RAID_DATA_MAX);
1770 : exit(EXIT_FAILURE);
1771 : /* LCOV_EXCL_STOP */
1772 : }
1773 :
1774 : /* now count the real number of data disks, excluding holes left after removing some */
1775 278 : diskcount = tommy_list_count(&state->maplist);
1776 :
1777 : /* recommend number of parities */
1778 278 : if (!state->opt.no_warnings) {
1779 : /* LCOV_EXCL_START */
1780 : if (diskcount >= 36 && state->level < 6) {
1781 : log_error(EUSER, "WARNING! For %u disks, it's recommended to use six parity levels.\n", diskcount);
1782 : } else if (diskcount >= 29 && state->level < 5) {
1783 : log_error(EUSER, "WARNING! For %u disks, it's recommended to use five parity levels.\n", diskcount);
1784 : } else if (diskcount >= 22 && state->level < 4) {
1785 : log_error(EUSER, "WARNING! For %u disks, it's recommended to use four parity levels.\n", diskcount);
1786 : } else if (diskcount >= 15 && state->level < 3) {
1787 : log_error(EUSER, "WARNING! For %u disks, it's recommended to use three parity levels.\n", diskcount);
1788 : } else if (diskcount >= 5 && state->level < 2) {
1789 : log_error(EUSER, "WARNING! For %u disks, it's recommended to use two parity levels.\n", diskcount);
1790 : }
1791 : /* LCOV_EXCL_STOP */
1792 : }
1793 : }
1794 :
1795 213 : void state_refresh(struct snapraid_state* state)
1796 : {
1797 : char esc_buffer[ESC_MAX];
1798 : tommy_node* i;
1799 : unsigned l, s;
1800 213 : uint64_t bs = (uint64_t)state->block_size;
1801 :
1802 : /* for all disks */
1803 1488 : for (i = state->maplist; i != 0; i = i->next) {
1804 1275 : struct snapraid_map* map = i->data;
1805 : struct snapraid_disk* disk;
1806 : uint64_t total_space;
1807 : uint64_t free_space;
1808 : int ret;
1809 :
1810 1275 : disk = find_disk_by_name(state, map->name);
1811 1275 : if (disk == 0) {
1812 : /* LCOV_EXCL_START */
1813 : log_fatal(EINTERNAL, "Internal inconsistency: Mapping not found for '%s'\n", map->name);
1814 : os_abort();
1815 : /* LCOV_EXCL_STOP */
1816 : }
1817 :
1818 1275 : ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space, disk->fstype, sizeof(disk->fstype), disk->fslabel, sizeof(disk->fslabel));
1819 1275 : if (ret != 0) {
1820 : /* LCOV_EXCL_START */
1821 : log_fatal(errno, "Error accessing disk '%s' to retrieve file-system info. %s.\n", disk->dir, strerror(errno));
1822 : exit(EXIT_FAILURE);
1823 : /* LCOV_EXCL_STOP */
1824 : }
1825 :
1826 : /* set the new free blocks */
1827 1275 : map->total_blocks = total_space / bs;
1828 1275 : map->free_blocks = free_space / bs;
1829 :
1830 : /* also update the disk info */
1831 1275 : disk->total_blocks = map->total_blocks;
1832 1275 : disk->free_blocks = map->free_blocks;
1833 :
1834 1275 : log_tag("fsinfo_data_split:%s:%" PRIu64 ":%" PRIu64 ":%s:%s\n", esc_tag(disk->name, esc_buffer), disk->total_blocks * bs, disk->free_blocks * bs, disk->fstype, disk->fslabel);
1835 1275 : log_tag("fsinfo_data:%s:%" PRIu64 ":%" PRIu64 "\n", esc_tag(disk->name, esc_buffer), disk->total_blocks * bs, disk->free_blocks * bs);
1836 : }
1837 :
1838 : /* for all parities */
1839 1386 : for (l = 0; l < state->level; ++l) {
1840 : /* set the new free blocks */
1841 1173 : state->parity[l].total_blocks = 0;
1842 1173 : state->parity[l].free_blocks = 0;
1843 :
1844 5865 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1845 4692 : struct snapraid_split* split = &state->parity[l].split_map[s];
1846 : uint64_t total_space;
1847 : uint64_t free_space;
1848 : int ret;
1849 :
1850 4692 : ret = fsinfo(split->path, 0, 0, &total_space, &free_space, split->fstype, sizeof(split->fstype), split->fslabel, sizeof(split->fslabel));
1851 4692 : if (ret != 0) {
1852 : /* LCOV_EXCL_START */
1853 : log_fatal(errno, "Error accessing file '%s' to retrieve file-system info. %s.\n", split->path, strerror(errno));
1854 : exit(EXIT_FAILURE);
1855 : /* LCOV_EXCL_STOP */
1856 : }
1857 :
1858 : #ifdef WIN32
1859 : /*
1860 : * Take into account the space we have to leave free
1861 : * to avoid the waring about low space
1862 : */
1863 : if (free_space >= WINDOWS_SPACEHOLDER_SIZE)
1864 : free_space -= WINDOWS_SPACEHOLDER_SIZE;
1865 : else
1866 : free_space = 0;
1867 : #endif
1868 :
1869 : /* add the new free blocks */
1870 4692 : uint64_t split_total_blocks = total_space / bs;
1871 4692 : uint64_t split_free_blocks = free_space / bs;
1872 :
1873 4692 : state->parity[l].total_blocks += split_total_blocks;
1874 4692 : state->parity[l].free_blocks += split_free_blocks;
1875 :
1876 4692 : if (s == 0)
1877 1173 : log_tag("fsinfo_parity_split:%s:%" PRIu64 ":%" PRIu64 ":%s:%s\n", lev_config_name(l), split_total_blocks * bs, split_free_blocks * bs, split->fstype, split->fslabel);
1878 : else
1879 3519 : log_tag("fsinfo_parity_split:%s/%u:%" PRIu64 ":%" PRIu64 ":%s:%s\n", lev_config_name(l), s, split_total_blocks * bs, split_free_blocks * bs, split->fstype, split->fslabel);
1880 : }
1881 :
1882 1173 : log_tag("fsinfo_parity:%s:%" PRIu64 ":%" PRIu64 "\n", lev_config_name(l), state->parity[l].total_blocks * bs, state->parity[l].free_blocks * bs);
1883 : }
1884 :
1885 : /* for all extra disks */
1886 213 : for (i = state->extralist; i != 0; i = i->next) {
1887 0 : struct snapraid_extra* extra = i->data;
1888 : uint64_t total_space;
1889 : uint64_t free_space;
1890 : int ret;
1891 :
1892 0 : ret = fsinfo(extra->dir, 0, 0, &total_space, &free_space, extra->fstype, sizeof(extra->fstype), extra->fslabel, sizeof(extra->fslabel));
1893 0 : if (ret != 0) {
1894 : /* LCOV_EXCL_START */
1895 : log_fatal(errno, "Error accessing disk '%s' to retrieve file-system info. %s.\n", extra->dir, strerror(errno));
1896 : exit(EXIT_FAILURE);
1897 : /* LCOV_EXCL_STOP */
1898 : }
1899 :
1900 0 : log_tag("fsinfo_extra:%s:%" PRIu64 ":%" PRIu64 "\n", esc_tag(extra->name, esc_buffer), total_space, free_space);
1901 : }
1902 :
1903 :
1904 : /* note what we don't set need_write = 1, because we don't want */
1905 : /* to update the content file only for the free space info. */
1906 213 : }
1907 :
1908 : /**
1909 : * Check the content.
1910 : */
1911 272 : static void state_content_check(struct snapraid_state* state, const char* path)
1912 : {
1913 : tommy_node* i;
1914 :
1915 : /* check that any map has different name and position */
1916 1899 : for (i = state->maplist; i != 0; i = i->next) {
1917 1627 : struct snapraid_map* map = i->data;
1918 : tommy_node* j;
1919 5682 : for (j = i->next; j != 0; j = j->next) {
1920 4055 : struct snapraid_map* other = j->data;
1921 4055 : if (strcmp(map->name, other->name) == 0) {
1922 : /* LCOV_EXCL_START */
1923 : log_fatal(ECONTENT, "Conflicting 'map' disk specification in '%s'\n", path);
1924 : exit(EXIT_FAILURE);
1925 : /* LCOV_EXCL_STOP */
1926 : }
1927 4055 : if (map->position == other->position) {
1928 : /* LCOV_EXCL_START */
1929 : log_fatal(ECONTENT, "Conflicting 'map' index specification in '%s'\n", path);
1930 : exit(EXIT_FAILURE);
1931 : /* LCOV_EXCL_STOP */
1932 : }
1933 : }
1934 : }
1935 272 : }
1936 :
1937 : /**
1938 : * Check if the position is REQUIRED, or we can completely clear it from the state.
1939 : *
1940 : * Note that position with only DELETED blocks are discharged.
1941 : */
1942 824833 : static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1943 : {
1944 : tommy_node* i;
1945 :
1946 : /* check for each disk */
1947 902945 : for (i = state->disklist; i != 0; i = i->next) {
1948 901067 : struct snapraid_disk* disk = i->data;
1949 901067 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1950 :
1951 : /* if we have at least one file, the position is needed */
1952 901067 : if (block_has_file(block))
1953 822955 : return 1;
1954 : }
1955 :
1956 1878 : return 0;
1957 : }
1958 :
1959 : /**
1960 : * Check if the info block is REQUIREQ.
1961 : *
1962 : * This is used to ensure that we keep the last check used for scrubbing.
1963 : * and that we add it when importing old context files.
1964 : *
1965 : * Note that you can have position without info blocks, for example
1966 : * if all the blocks are not synced.
1967 : *
1968 : * Note also that not requiring an info block, doesn't mean that if present it
1969 : * can be discarded.
1970 : */
1971 1660597 : static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1972 : {
1973 : tommy_node* i;
1974 :
1975 : /* check for each disk */
1976 2187239 : for (i = state->disklist; i != 0; i = i->next) {
1977 2133357 : struct snapraid_disk* disk = i->data;
1978 2133357 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1979 :
1980 : /* if we have at least one synced file, the info is required */
1981 2133357 : if (block_state_get(block) == BLOCK_STATE_BLK)
1982 1606715 : return 1;
1983 : }
1984 :
1985 53882 : return 0;
1986 : }
1987 :
1988 1878 : static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1989 : {
1990 : tommy_node* i;
1991 :
1992 : /* check for each disk if block is really used */
1993 13146 : for (i = state->disklist; i != 0; i = i->next) {
1994 11268 : struct snapraid_disk* disk = i->data;
1995 11268 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1996 :
1997 : /* if the block is deleted */
1998 11268 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1999 : /* set it to empty */
2000 939 : fs_deallocate(disk, pos);
2001 : }
2002 : }
2003 1878 : }
2004 :
2005 : /**
2006 : * Check if a block position in a disk is deleted.
2007 : */
2008 4942032 : static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
2009 : {
2010 4942032 : struct snapraid_block* block = fs_par2block_find(disk, pos);
2011 :
2012 4942032 : return block_state_get(block) == BLOCK_STATE_DELETED;
2013 : }
2014 :
2015 : /**
2016 : * Check if a block parity needs sync
2017 : *
2018 : * This is the same logic used in "status" to detect an incomplete "sync",
2019 : * that ignores invalid block, if they are not used by a file in any disk.
2020 : *
2021 : * This means that DELETED blocks won't necessarily imply an invalid parity.
2022 : */
2023 2485430 : static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos)
2024 : {
2025 2485430 : int one_invalid = 0;
2026 2485430 : int one_valid = 0;
2027 16110343 : for (tommy_node* i = state->disklist; i != 0; i = i->next) {
2028 13849954 : struct snapraid_disk* disk = i->data;
2029 13849954 : struct snapraid_block* block = fs_par2block_find(disk, pos);
2030 :
2031 13849954 : if (block_has_file(block))
2032 13456345 : one_valid = 1;
2033 13849954 : if (block_has_invalid_parity(block))
2034 226229 : one_invalid = 1;
2035 :
2036 13849954 : if (one_invalid && one_valid)
2037 225041 : return 1;
2038 : }
2039 :
2040 2260389 : return 0;
2041 : }
2042 :
2043 : /**
2044 : * Flush the file checking the final CRC.
2045 : * We exploit the fact that the CRC is always stored in the last 4 bytes.
2046 : */
2047 1 : static void decoding_error(const char* path, STREAM* f)
2048 : {
2049 : unsigned char buf[4];
2050 : uint32_t crc_stored;
2051 : uint32_t crc_computed;
2052 :
2053 1 : if (seof(f)) {
2054 : /* LCOV_EXCL_START */
2055 : log_fatal(ECONTENT, "Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2056 : log_fatal(ECONTENT, "This content file is truncated. Please use an alternate copy.\n");
2057 : exit(EXIT_FAILURE);
2058 : /* LCOV_EXCL_STOP */
2059 : }
2060 :
2061 1 : if (serror(f)) {
2062 : /* LCOV_EXCL_START */
2063 : log_fatal(errno, "Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2064 : exit(EXIT_FAILURE);
2065 : /* LCOV_EXCL_STOP */
2066 : }
2067 :
2068 1 : log_fatal(ECONTENT, "Error decoding '%s' at offset %" PRIi64 "\n", path, stell(f));
2069 :
2070 1 : if (sdeplete(f, buf) != 0) {
2071 : /* LCOV_EXCL_START */
2072 : log_fatal(errno, "Failed to flush content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2073 : exit(EXIT_FAILURE);
2074 : /* LCOV_EXCL_STOP */
2075 : }
2076 :
2077 : /* get the stored crc from the last four bytes */
2078 1 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
2079 :
2080 : /* get the computed crc */
2081 1 : crc_computed = scrc(f);
2082 :
2083 : /* adjust the stored crc to include itself */
2084 1 : crc_stored = crc32c(crc_stored, buf, 4);
2085 :
2086 1 : if (crc_computed != crc_stored) {
2087 1 : log_fatal(ECONTENT, "CRC mismatch in '%s'\n", path);
2088 1 : log_fatal(ECONTENT, "This content file is damaged! Please use an alternate copy.\n");
2089 1 : exit(EXIT_FAILURE);
2090 : } else {
2091 0 : log_fatal(ECONTENT, "The CRC of the file is correct!\n");
2092 : }
2093 0 : }
2094 :
2095 273 : static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
2096 : {
2097 : char esc_buffer[ESC_MAX];
2098 : block_off_t blockmax;
2099 : unsigned count_file;
2100 : unsigned count_hardlink;
2101 : unsigned count_symlink;
2102 : unsigned count_dir;
2103 : unsigned count_bad;
2104 : unsigned count_rehash;
2105 : unsigned count_unsynced;
2106 : unsigned count_unscrubbed;
2107 : int crc_checked;
2108 : char buffer[PATH_MAX];
2109 : int ret;
2110 : tommy_array disk_mapping;
2111 : uint32_t mapping_max;
2112 : tommy_hashdyn bucket_hash;
2113 :
2114 273 : blockmax = 0;
2115 273 : count_file = 0;
2116 273 : count_hardlink = 0;
2117 273 : count_symlink = 0;
2118 273 : count_dir = 0;
2119 273 : count_bad = 0;
2120 273 : count_rehash = 0;
2121 273 : count_unsynced = 0;
2122 273 : count_unscrubbed = 0;
2123 273 : crc_checked = 0;
2124 273 : mapping_max = 0;
2125 273 : tommy_array_init(&disk_mapping);
2126 273 : tommy_hashdyn_init(&bucket_hash);
2127 :
2128 : /* mark all disks as single threads */
2129 1900 : for (tommy_node* i = state->disklist; i != 0; i = i->next) {
2130 1627 : struct snapraid_disk* disk = i->data;
2131 1627 : disk->single_thread = 1;
2132 : }
2133 :
2134 273 : ret = sread(f, buffer, 12);
2135 273 : if (ret < 0) {
2136 : /* LCOV_EXCL_START */
2137 : decoding_error(path, f);
2138 : log_fatal(ECONTENT, "Invalid header!\n");
2139 : os_abort();
2140 : /* LCOV_EXCL_STOP */
2141 : }
2142 :
2143 : /*
2144 : * File format versions:
2145 : * - SNAPCNT1/SnapRAID 4.0 First version.
2146 : * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
2147 : * The previous 'm' entry is now deprecated, but supported for importing.
2148 : * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
2149 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
2150 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
2151 : * The previous 'P' entry is now deprecated, but supported for importing.
2152 : */
2153 273 : if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
2154 273 : && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
2155 273 : && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
2156 : ) {
2157 : /* LCOV_EXCL_START */
2158 : if (memcmp(buffer, "SNAPCNT", 7) != 0) {
2159 : decoding_error(path, f);
2160 : log_fatal(ECONTENT, "Invalid header!\n");
2161 : os_abort();
2162 : } else {
2163 : log_fatal(ECONTENT, "The content file '%s' was generated with a newer version of SnapRAID!\n", path);
2164 : exit(EXIT_FAILURE);
2165 : }
2166 : /* LCOV_EXCL_STOP */
2167 : }
2168 :
2169 3978501 : while (1) {
2170 : int c;
2171 :
2172 : /* read the command */
2173 3978774 : c = sgetc(f);
2174 3978774 : if (c == EOF) {
2175 272 : break;
2176 : }
2177 :
2178 3978502 : if (c == 'f') {
2179 : /* file */
2180 : char sub[PATH_MAX];
2181 : uint64_t v_size;
2182 : uint64_t v_mtime_sec;
2183 : uint32_t v_mtime_nsec;
2184 : uint64_t v_inode;
2185 : uint32_t v_idx;
2186 : struct snapraid_file* file;
2187 : struct snapraid_disk* disk;
2188 : uint32_t mapping;
2189 :
2190 3847070 : ret = sgetb32(f, &mapping);
2191 3847070 : if (ret < 0 || mapping >= mapping_max) {
2192 : /* LCOV_EXCL_START */
2193 : decoding_error(path, f);
2194 : log_fatal(EINTERNAL, "Internal inconsistency: File mapping index out of range\n");
2195 : os_abort();
2196 : /* LCOV_EXCL_STOP */
2197 : }
2198 3847070 : disk = tommy_array_get(&disk_mapping, mapping);
2199 :
2200 3847070 : ret = sgetb64(f, &v_size);
2201 3847070 : if (ret < 0) {
2202 : /* LCOV_EXCL_START */
2203 : decoding_error(path, f);
2204 : os_abort();
2205 : /* LCOV_EXCL_STOP */
2206 : }
2207 :
2208 3847070 : if (state->block_size == 0) {
2209 : /* LCOV_EXCL_START */
2210 : decoding_error(path, f);
2211 : log_fatal(EINTERNAL, "Internal inconsistency: Zero blocksize\n");
2212 : exit(EXIT_FAILURE);
2213 : /* LCOV_EXCL_STOP */
2214 : }
2215 :
2216 : /* check for impossible file size to avoid to crash for a too big allocation */
2217 3847070 : if (v_size / state->block_size > blockmax) {
2218 : /* LCOV_EXCL_START */
2219 : decoding_error(path, f);
2220 : log_fatal(EINTERNAL, "Internal inconsistency: File size too big!\n");
2221 : os_abort();
2222 : /* LCOV_EXCL_STOP */
2223 : }
2224 :
2225 3847070 : ret = sgetb64(f, &v_mtime_sec);
2226 3847070 : if (ret < 0) {
2227 : /* LCOV_EXCL_START */
2228 : decoding_error(path, f);
2229 : os_abort();
2230 : /* LCOV_EXCL_STOP */
2231 : }
2232 :
2233 3847070 : ret = sgetb32(f, &v_mtime_nsec);
2234 3847070 : if (ret < 0) {
2235 : /* LCOV_EXCL_START */
2236 : decoding_error(path, f);
2237 : os_abort();
2238 : /* LCOV_EXCL_STOP */
2239 : }
2240 :
2241 : /* STAT_NSEC_INVALID is encoded as 0 */
2242 3847070 : if (v_mtime_nsec == 0)
2243 0 : v_mtime_nsec = STAT_NSEC_INVALID;
2244 : else
2245 3847070 : --v_mtime_nsec;
2246 :
2247 3847070 : ret = sgetb64(f, &v_inode);
2248 3847070 : if (ret < 0) {
2249 : /* LCOV_EXCL_START */
2250 : decoding_error(path, f);
2251 : os_abort();
2252 : /* LCOV_EXCL_STOP */
2253 : }
2254 :
2255 3847070 : ret = sgetbs(f, sub, sizeof(sub));
2256 3847070 : if (ret < 0) {
2257 : /* LCOV_EXCL_START */
2258 : decoding_error(path, f);
2259 : os_abort();
2260 : /* LCOV_EXCL_STOP */
2261 : }
2262 3847070 : if (!*sub) {
2263 : /* LCOV_EXCL_START */
2264 : decoding_error(path, f);
2265 : log_fatal(EINTERNAL, "Internal inconsistency: Null file!\n");
2266 : os_abort();
2267 : /* LCOV_EXCL_STOP */
2268 : }
2269 :
2270 : /* allocate the file */
2271 3847070 : file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
2272 :
2273 : /* insert the file in the file containers */
2274 3847070 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
2275 3847070 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
2276 3847070 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
2277 3847070 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
2278 :
2279 : /* read all the blocks */
2280 3847070 : v_idx = 0;
2281 7716279 : while (v_idx < file->blockmax) {
2282 : block_off_t v_pos;
2283 : uint32_t v_count;
2284 :
2285 : /* get the "subcommand */
2286 3869210 : c = sgetc(f);
2287 :
2288 3869210 : ret = sgetb32(f, &v_pos);
2289 3869210 : if (ret < 0) {
2290 : /* LCOV_EXCL_START */
2291 : decoding_error(path, f);
2292 : os_abort();
2293 : /* LCOV_EXCL_STOP */
2294 : }
2295 :
2296 3869210 : ret = sgetb32(f, &v_count);
2297 3869210 : if (ret < 0) {
2298 : /* LCOV_EXCL_START */
2299 : decoding_error(path, f);
2300 : os_abort();
2301 : /* LCOV_EXCL_STOP */
2302 : }
2303 :
2304 3869210 : if (v_idx + v_count > file->blockmax) {
2305 : /* LCOV_EXCL_START */
2306 : decoding_error(path, f);
2307 : log_fatal(EINTERNAL, "Internal inconsistency: Block number out of range\n");
2308 : os_abort();
2309 : /* LCOV_EXCL_STOP */
2310 : }
2311 :
2312 3869209 : if (v_pos + v_count > blockmax) {
2313 : /* LCOV_EXCL_START */
2314 : decoding_error(path, f);
2315 : log_fatal(EINTERNAL, "Internal inconsistency: Block size %u/%u!\n", blockmax, v_pos + v_count);
2316 : os_abort();
2317 : /* LCOV_EXCL_START */
2318 : }
2319 :
2320 : /* fill the blocks in the run */
2321 : while (v_count) {
2322 : struct snapraid_block* block = fs_file2block_get(file, v_idx);
2323 :
2324 : switch (c) {
2325 : case 'b' :
2326 : block_state_set(block, BLOCK_STATE_BLK);
2327 : break;
2328 : case 'n' :
2329 : /* deprecated NEW blocks are converted to CHG ones */
2330 : block_state_set(block, BLOCK_STATE_CHG);
2331 : break;
2332 : case 'g' :
2333 : block_state_set(block, BLOCK_STATE_CHG);
2334 : break;
2335 : case 'p' :
2336 : block_state_set(block, BLOCK_STATE_REP);
2337 : break;
2338 : default :
2339 : /* LCOV_EXCL_START */
2340 : decoding_error(path, f);
2341 : log_fatal(ECONTENT, "Invalid block type!\n");
2342 : os_abort();
2343 : /* LCOV_EXCL_STOP */
2344 : }
2345 :
2346 : /* read the hash only for 'blk/chg/rep', and not for 'new' */
2347 9486340 : if (c != 'n') {
2348 9486340 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2349 9486340 : if (ret < 0) {
2350 : /* LCOV_EXCL_START */
2351 : decoding_error(path, f);
2352 : os_abort();
2353 : /* LCOV_EXCL_STOP */
2354 : }
2355 : } else {
2356 : /* set the ZERO hash for deprecated NEW blocks */
2357 0 : hash_zero_set(block->hash);
2358 : }
2359 :
2360 : /* if the block contains a hash of past data */
2361 : /* and we are clearing such indeterminate hashes */
2362 9486340 : if (state->clear_past_hash
2363 2821726 : && block_has_past_hash(block)
2364 : ) {
2365 : /* set the hash value to INVALID */
2366 27935 : hash_invalid_set(block->hash);
2367 : }
2368 :
2369 : /* if we are disabling the copy optimization */
2370 : /* we want also to clear any already previously stored information */
2371 : /* in other sync commands */
2372 : /* note that this is required only in sync, and we detect */
2373 : /* this using the clear_past_hash flag */
2374 9486340 : if (state->clear_past_hash
2375 2821726 : && state->opt.force_nocopy
2376 14195 : && block_state_get(block) == BLOCK_STATE_REP
2377 : ) {
2378 : /* set the hash value to INVALID */
2379 1 : hash_invalid_set(block->hash);
2380 : /* convert from REP to CHG block */
2381 1 : block_state_set(block, BLOCK_STATE_CHG);
2382 : }
2383 :
2384 : /* set the parity association */
2385 9486340 : fs_allocate(disk, v_pos, file, v_idx);
2386 :
2387 : /* go to the next block */
2388 9486340 : ++v_idx;
2389 9486340 : ++v_pos;
2390 9486340 : --v_count;
2391 : }
2392 : }
2393 :
2394 : /* stat */
2395 3847069 : ++count_file;
2396 131432 : } else if (c == 'i') {
2397 : /* "inf" command */
2398 : snapraid_info info;
2399 : uint32_t v_pos;
2400 : uint64_t v_oldest;
2401 :
2402 272 : ret = sgetb64(f, &v_oldest);
2403 272 : if (ret < 0) {
2404 : /* LCOV_EXCL_START */
2405 : decoding_error(path, f);
2406 : os_abort();
2407 : /* LCOV_EXCL_STOP */
2408 : }
2409 :
2410 272 : v_pos = 0;
2411 42636 : while (v_pos < blockmax) {
2412 : int bad;
2413 : int rehash;
2414 : int justsynced;
2415 : uint64_t t64;
2416 : uint32_t flag;
2417 : uint32_t v_count;
2418 :
2419 42364 : ret = sgetb32(f, &v_count);
2420 42364 : if (ret < 0) {
2421 : /* LCOV_EXCL_START */
2422 : decoding_error(path, f);
2423 : os_abort();
2424 : /* LCOV_EXCL_STOP */
2425 : }
2426 :
2427 42364 : if (v_pos + v_count > blockmax) {
2428 : /* LCOV_EXCL_START */
2429 : decoding_error(path, f);
2430 : log_fatal(EINTERNAL, "Internal inconsistency: Info size %u/%u!\n", blockmax, v_pos + v_count);
2431 : os_abort();
2432 : /* LCOV_EXCL_STOP */
2433 : }
2434 :
2435 42364 : ret = sgetb32(f, &flag);
2436 42364 : if (ret < 0) {
2437 : /* LCOV_EXCL_START */
2438 : decoding_error(path, f);
2439 : os_abort();
2440 : /* LCOV_EXCL_STOP */
2441 : }
2442 :
2443 : /* if there is an info */
2444 42364 : if ((flag & 1) != 0) {
2445 : /* read the time */
2446 42347 : ret = sgetb64(f, &t64);
2447 42347 : if (ret < 0) {
2448 : /* LCOV_EXCL_START */
2449 : decoding_error(path, f);
2450 : os_abort();
2451 : /* LCOV_EXCL_STOP */
2452 : }
2453 :
2454 : /* analyze the flags */
2455 42347 : bad = (flag & 2) != 0;
2456 42347 : rehash = (flag & 4) != 0;
2457 42347 : justsynced = (flag & 8) != 0;
2458 :
2459 42347 : if (bad)
2460 5301 : count_bad += v_count;
2461 42347 : if (rehash)
2462 14266 : count_rehash += v_count;
2463 42347 : if (justsynced)
2464 20983 : count_unscrubbed += v_count;
2465 :
2466 42347 : if (rehash && state->prevhash == HASH_UNDEFINED) {
2467 : /* LCOV_EXCL_START */
2468 : decoding_error(path, f);
2469 : log_fatal(EINTERNAL, "Internal inconsistency: Missing previous checksum!\n");
2470 : os_abort();
2471 : /* LCOV_EXCL_STOP */
2472 : }
2473 :
2474 42347 : info = info_make(t64 + v_oldest, bad, rehash, justsynced);
2475 :
2476 42347 : bucket_insert(&bucket_hash, t64 + v_oldest, v_count, justsynced);
2477 : } else {
2478 17 : info = 0;
2479 : }
2480 :
2481 1702961 : while (v_count) {
2482 : /* insert the info in the array */
2483 1660597 : info_set(&state->infoarr, v_pos, info);
2484 :
2485 : /* ensure that an info is present only for used positions */
2486 1660597 : if (fs_info_is_required(state, v_pos)) {
2487 1606715 : if (!info) {
2488 : /* LCOV_EXCL_START */
2489 : decoding_error(path, f);
2490 : log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n");
2491 : os_abort();
2492 : /* LCOV_EXCL_STOP */
2493 : }
2494 : } else {
2495 : /* extra info are accepted for backward compatibility */
2496 : /* they are discarded at the first write */
2497 : }
2498 :
2499 1660597 : if (fs_is_block_unsynced(state, v_pos))
2500 112705 : ++count_unsynced;
2501 :
2502 : /* go to next block */
2503 1660597 : ++v_pos;
2504 1660597 : --v_count;
2505 : }
2506 : }
2507 131160 : } else if (c == 'h') {
2508 : /* hole */
2509 : uint32_t v_pos;
2510 : struct snapraid_disk* disk;
2511 : uint32_t mapping;
2512 :
2513 1511 : ret = sgetb32(f, &mapping);
2514 1511 : if (ret < 0 || mapping >= mapping_max) {
2515 : /* LCOV_EXCL_START */
2516 : decoding_error(path, f);
2517 : log_fatal(EINTERNAL, "Internal inconsistency: Hole mapping index out of range\n");
2518 : os_abort();
2519 : /* LCOV_EXCL_STOP */
2520 : }
2521 1511 : disk = tommy_array_get(&disk_mapping, mapping);
2522 :
2523 1511 : v_pos = 0;
2524 3088 : while (v_pos < blockmax) {
2525 : uint32_t v_idx;
2526 : uint32_t v_count;
2527 : struct snapraid_file* deleted;
2528 :
2529 1577 : ret = sgetb32(f, &v_count);
2530 1577 : if (ret < 0) {
2531 : /* LCOV_EXCL_START */
2532 : decoding_error(path, f);
2533 : os_abort();
2534 : /* LCOV_EXCL_STOP */
2535 : }
2536 :
2537 1577 : if (v_pos + v_count > blockmax) {
2538 : /* LCOV_EXCL_START */
2539 : decoding_error(path, f);
2540 : log_fatal(EINTERNAL, "Internal inconsistency: Hole size %u/%u!\n", blockmax, v_pos + v_count);
2541 : os_abort();
2542 : /* LCOV_EXCL_STOP */
2543 : }
2544 :
2545 : /* get the sub-command */
2546 1577 : c = sgetc(f);
2547 :
2548 1577 : switch (c) {
2549 35 : case 'o' :
2550 : /* if it's a run of deleted blocks */
2551 :
2552 : /* allocate a fake deleted file */
2553 35 : deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2554 :
2555 : /* mark the file as deleted */
2556 35 : file_flag_set(deleted, FILE_IS_DELETED);
2557 :
2558 : /* insert it in the list of deleted files */
2559 35 : tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2560 :
2561 : /* process all blocks */
2562 35 : v_idx = 0;
2563 28167 : while (v_count) {
2564 28132 : struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2565 :
2566 : /* set the block as deleted */
2567 28132 : block_state_set(block, BLOCK_STATE_DELETED);
2568 :
2569 : /* read the hash */
2570 28132 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2571 28132 : if (ret < 0) {
2572 : /* LCOV_EXCL_START */
2573 : decoding_error(path, f);
2574 : os_abort();
2575 : /* LCOV_EXCL_STOP */
2576 : }
2577 :
2578 : /* if we are clearing indeterminate hashes */
2579 28132 : if (state->clear_past_hash) {
2580 : /* set the hash value to INVALID */
2581 9514 : hash_invalid_set(block->hash);
2582 : }
2583 :
2584 : /* insert the block in the block array */
2585 28132 : fs_allocate(disk, v_pos, deleted, v_idx);
2586 :
2587 : /* go to next block */
2588 28132 : ++v_pos;
2589 28132 : ++v_idx;
2590 28132 : --v_count;
2591 : }
2592 35 : break;
2593 1542 : case 'O' :
2594 : /* go to the next run */
2595 1542 : v_pos += v_count;
2596 1542 : break;
2597 0 : default :
2598 : /* LCOV_EXCL_START */
2599 : decoding_error(path, f);
2600 : log_fatal(ECONTENT, "Invalid hole type!\n");
2601 : os_abort();
2602 : /* LCOV_EXCL_STOP */
2603 : }
2604 : }
2605 129649 : } else if (c == 's') {
2606 : /* symlink */
2607 : char sub[PATH_MAX];
2608 : char linkto[PATH_MAX];
2609 : struct snapraid_link* slink;
2610 : struct snapraid_disk* disk;
2611 : uint32_t mapping;
2612 :
2613 123738 : ret = sgetb32(f, &mapping);
2614 123738 : if (ret < 0 || mapping >= mapping_max) {
2615 : /* LCOV_EXCL_START */
2616 : decoding_error(path, f);
2617 : log_fatal(EINTERNAL, "Internal inconsistency: Symlink mapping index out of range\n");
2618 : os_abort();
2619 : /* LCOV_EXCL_STOP */
2620 : }
2621 123738 : disk = tommy_array_get(&disk_mapping, mapping);
2622 :
2623 123738 : ret = sgetbs(f, sub, sizeof(sub));
2624 123738 : if (ret < 0) {
2625 : /* LCOV_EXCL_START */
2626 : decoding_error(path, f);
2627 : os_abort();
2628 : /* LCOV_EXCL_STOP */
2629 : }
2630 :
2631 123738 : if (!*sub) {
2632 : /* LCOV_EXCL_START */
2633 : decoding_error(path, f);
2634 : log_fatal(EINTERNAL, "Internal inconsistency: Null symlink!\n");
2635 : os_abort();
2636 : /* LCOV_EXCL_STOP */
2637 : }
2638 :
2639 123738 : ret = sgetbs(f, linkto, sizeof(linkto));
2640 123738 : if (ret < 0) {
2641 : /* LCOV_EXCL_START */
2642 : decoding_error(path, f);
2643 : os_abort();
2644 : /* LCOV_EXCL_STOP */
2645 : }
2646 :
2647 : /* allocate the link as symbolic link */
2648 123738 : slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2649 :
2650 : /* insert the link in the link containers */
2651 123738 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2652 123738 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2653 :
2654 : /* stat */
2655 123738 : ++count_symlink;
2656 5911 : } else if (c == 'a') {
2657 : /* hardlink */
2658 : char sub[PATH_MAX];
2659 : char linkto[PATH_MAX];
2660 : struct snapraid_link* slink;
2661 : struct snapraid_disk* disk;
2662 : uint32_t mapping;
2663 :
2664 651 : ret = sgetb32(f, &mapping);
2665 651 : if (ret < 0 || mapping >= mapping_max) {
2666 : /* LCOV_EXCL_START */
2667 : decoding_error(path, f);
2668 : log_fatal(EINTERNAL, "Internal inconsistency: Hardlink mapping index out of range!\n");
2669 : os_abort();
2670 : /* LCOV_EXCL_STOP */
2671 : }
2672 651 : disk = tommy_array_get(&disk_mapping, mapping);
2673 :
2674 651 : ret = sgetbs(f, sub, sizeof(sub));
2675 651 : if (ret < 0) {
2676 : /* LCOV_EXCL_START */
2677 : decoding_error(path, f);
2678 : os_abort();
2679 : /* LCOV_EXCL_STOP */
2680 : }
2681 :
2682 651 : if (!*sub) {
2683 : /* LCOV_EXCL_START */
2684 : decoding_error(path, f);
2685 : log_fatal(EINTERNAL, "Internal inconsistency: Null hardlink!\n");
2686 : os_abort();
2687 : /* LCOV_EXCL_STOP */
2688 : }
2689 :
2690 651 : ret = sgetbs(f, linkto, sizeof(linkto));
2691 651 : if (ret < 0) {
2692 : /* LCOV_EXCL_START */
2693 : decoding_error(path, f);
2694 : os_abort();
2695 : /* LCOV_EXCL_STOP */
2696 : }
2697 :
2698 651 : if (!*linkto) {
2699 : /* LCOV_EXCL_START */
2700 : decoding_error(path, f);
2701 : log_fatal(EINTERNAL, "Internal inconsistency: Empty hardlink '%s'!\n", sub);
2702 : os_abort();
2703 : /* LCOV_EXCL_STOP */
2704 : }
2705 :
2706 : /* allocate the link as hard link */
2707 651 : slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2708 :
2709 : /* insert the link in the link containers */
2710 651 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2711 651 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2712 :
2713 : /* stat */
2714 651 : ++count_hardlink;
2715 5260 : } else if (c == 'r') {
2716 : /* dir */
2717 : char sub[PATH_MAX];
2718 : struct snapraid_dir* dir;
2719 : struct snapraid_disk* disk;
2720 : uint32_t mapping;
2721 :
2722 862 : ret = sgetb32(f, &mapping);
2723 862 : if (ret < 0 || mapping >= mapping_max) {
2724 : /* LCOV_EXCL_START */
2725 : decoding_error(path, f);
2726 : log_fatal(EINTERNAL, "Internal inconsistency: Dir mapping index ouf of range!\n");
2727 : os_abort();
2728 : /* LCOV_EXCL_STOP */
2729 : }
2730 862 : disk = tommy_array_get(&disk_mapping, mapping);
2731 :
2732 862 : ret = sgetbs(f, sub, sizeof(sub));
2733 862 : if (ret < 0) {
2734 : /* LCOV_EXCL_START */
2735 : decoding_error(path, f);
2736 : os_abort();
2737 : /* LCOV_EXCL_STOP */
2738 : }
2739 :
2740 862 : if (!*sub) {
2741 : /* LCOV_EXCL_START */
2742 : decoding_error(path, f);
2743 : log_fatal(EINTERNAL, "Internal inconsistency: Null dir!\n");
2744 : os_abort();
2745 : /* LCOV_EXCL_STOP */
2746 : }
2747 :
2748 : /* allocate the dir */
2749 862 : dir = dir_alloc(sub);
2750 :
2751 : /* insert the dir in the dir containers */
2752 862 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2753 862 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2754 :
2755 : /* stat */
2756 862 : ++count_dir;
2757 4398 : } else if (c == 'c') {
2758 : /* get the subcommand */
2759 273 : c = sgetc(f);
2760 :
2761 273 : switch (c) {
2762 158 : case 'u' :
2763 158 : state->hash = HASH_MURMUR3;
2764 158 : break;
2765 115 : case 'k' :
2766 115 : state->hash = HASH_SPOOKY2;
2767 115 : break;
2768 0 : case 'm' :
2769 0 : state->hash = HASH_METRO;
2770 0 : break;
2771 0 : default :
2772 : /* LCOV_EXCL_START */
2773 : decoding_error(path, f);
2774 : log_fatal(ECONTENT, "Invalid checksum!\n");
2775 : os_abort();
2776 : /* LCOV_EXCL_STOP */
2777 : }
2778 :
2779 : /* read the seed */
2780 273 : ret = sread(f, state->hashseed, HASH_MAX);
2781 273 : if (ret < 0) {
2782 : /* LCOV_EXCL_START */
2783 : decoding_error(path, f);
2784 : os_abort();
2785 : /* LCOV_EXCL_STOP */
2786 : }
2787 4125 : } else if (c == 'C') {
2788 : /* get the sub-command */
2789 4 : c = sgetc(f);
2790 :
2791 4 : switch (c) {
2792 4 : case 'u' :
2793 4 : state->prevhash = HASH_MURMUR3;
2794 4 : break;
2795 0 : case 'k' :
2796 0 : state->prevhash = HASH_SPOOKY2;
2797 0 : break;
2798 0 : case 'm' :
2799 0 : state->prevhash = HASH_METRO;
2800 0 : break;
2801 0 : default :
2802 : /* LCOV_EXCL_START */
2803 : decoding_error(path, f);
2804 : log_fatal(ECONTENT, "Invalid checksum!\n");
2805 : os_abort();
2806 : /* LCOV_EXCL_STOP */
2807 : }
2808 :
2809 : /* read the seed */
2810 4 : ret = sread(f, state->prevhashseed, HASH_MAX);
2811 4 : if (ret < 0) {
2812 : /* LCOV_EXCL_START */
2813 : decoding_error(path, f);
2814 : os_abort();
2815 : /* LCOV_EXCL_STOP */
2816 : }
2817 4121 : } else if (c == 'z') {
2818 : uint32_t block_size;
2819 :
2820 273 : ret = sgetb32(f, &block_size);
2821 273 : if (ret < 0) {
2822 : /* LCOV_EXCL_START */
2823 : decoding_error(path, f);
2824 : os_abort();
2825 : /* LCOV_EXCL_STOP */
2826 : }
2827 :
2828 273 : if (block_size == 0) {
2829 : /* LCOV_EXCL_START */
2830 : decoding_error(path, f);
2831 : log_fatal(ECONTENT, "Zero 'blocksize' specification in the content file!\n");
2832 : exit(EXIT_FAILURE);
2833 : /* LCOV_EXCL_STOP */
2834 : }
2835 :
2836 : /* without configuration, auto assign the block size */
2837 273 : if (state->no_conf) {
2838 1 : state->block_size = block_size;
2839 : }
2840 :
2841 273 : if (block_size != state->block_size) {
2842 : /* LCOV_EXCL_START */
2843 : decoding_error(path, f);
2844 : log_fatal(EUSER, "Mismatching 'blocksize' specification in the content file!\n");
2845 : log_fatal(EUSER, "Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2846 : exit(EXIT_FAILURE);
2847 : /* LCOV_EXCL_STOP */
2848 : }
2849 3848 : } else if (c == 'y') {
2850 : uint32_t hash_size;
2851 :
2852 273 : ret = sgetb32(f, &hash_size);
2853 273 : if (ret < 0) {
2854 : /* LCOV_EXCL_START */
2855 : decoding_error(path, f);
2856 : os_abort();
2857 : /* LCOV_EXCL_STOP */
2858 : }
2859 :
2860 273 : if (hash_size < 2 || hash_size > HASH_MAX) {
2861 : /* LCOV_EXCL_START */
2862 : decoding_error(path, f);
2863 : log_fatal(ECONTENT, "Invalid 'hashsize' specification in the content file!\n");
2864 : exit(EXIT_FAILURE);
2865 : /* LCOV_EXCL_STOP */
2866 : }
2867 :
2868 : /* without configuration, auto assign the block size */
2869 273 : if (state->no_conf) {
2870 1 : BLOCK_HASH_SIZE = hash_size;
2871 : }
2872 :
2873 273 : if ((int)hash_size != BLOCK_HASH_SIZE) {
2874 : /* LCOV_EXCL_START */
2875 : decoding_error(path, f);
2876 : log_fatal(EUSER, "Mismatching 'hashsize' specification in the content file!\n");
2877 : log_fatal(EUSER, "Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2878 : exit(EXIT_FAILURE);
2879 : /* LCOV_EXCL_STOP */
2880 : }
2881 3575 : } else if (c == 'x') {
2882 273 : ret = sgetb32(f, &blockmax);
2883 273 : if (ret < 0) {
2884 : /* LCOV_EXCL_START */
2885 : decoding_error(path, f);
2886 : os_abort();
2887 : /* LCOV_EXCL_STOP */
2888 : }
2889 3302 : } else if (c == 'm' || c == 'M') {
2890 : struct snapraid_map* map;
2891 : char uuid[UUID_MAX];
2892 : uint32_t v_pos;
2893 : uint32_t v_total_blocks;
2894 : uint32_t v_free_blocks;
2895 : struct snapraid_disk* disk;
2896 :
2897 1517 : ret = sgetbs(f, buffer, sizeof(buffer));
2898 1517 : if (ret < 0) {
2899 : /* LCOV_EXCL_START */
2900 : decoding_error(path, f);
2901 : os_abort();
2902 : /* LCOV_EXCL_STOP */
2903 : }
2904 :
2905 1517 : ret = sgetb32(f, &v_pos);
2906 1517 : if (ret < 0) {
2907 : /* LCOV_EXCL_START */
2908 : decoding_error(path, f);
2909 : os_abort();
2910 : /* LCOV_EXCL_STOP */
2911 : }
2912 :
2913 : /* from SnapRAID 7.0 the 'M' command includes the free space */
2914 1517 : if (c == 'M') {
2915 1517 : ret = sgetb32(f, &v_total_blocks);
2916 1517 : if (ret < 0) {
2917 : /* LCOV_EXCL_START */
2918 : decoding_error(path, f);
2919 : os_abort();
2920 : /* LCOV_EXCL_STOP */
2921 : }
2922 :
2923 1517 : ret = sgetb32(f, &v_free_blocks);
2924 1517 : if (ret < 0) {
2925 : /* LCOV_EXCL_START */
2926 : decoding_error(path, f);
2927 : os_abort();
2928 : /* LCOV_EXCL_STOP */
2929 : }
2930 :
2931 1517 : log_tag("content_data:%s:%" PRIi64 ":%" PRIi64 "\n",
2932 : esc_tag(buffer, esc_buffer),
2933 1517 : v_total_blocks * (uint64_t)state->block_size,
2934 1517 : v_free_blocks * (uint64_t)state->block_size);
2935 : } else {
2936 0 : v_total_blocks = 0;
2937 0 : v_free_blocks = 0;
2938 : }
2939 :
2940 : /* read the uuid */
2941 1517 : ret = sgetbs(f, uuid, sizeof(uuid));
2942 1517 : if (ret < 0) {
2943 : /* LCOV_EXCL_START */
2944 : decoding_error(path, f);
2945 : os_abort();
2946 : /* LCOV_EXCL_STOP */
2947 : }
2948 :
2949 1517 : log_tag("content_data_split:%s:%s\n",
2950 : esc_tag(buffer, esc_buffer),
2951 : uuid);
2952 :
2953 : /* find the disk */
2954 1517 : disk = find_disk_by_name(state, buffer);
2955 1517 : if (!disk) {
2956 : /* search by UUID if renamed */
2957 2 : disk = find_disk_by_uuid(state, uuid);
2958 2 : if (disk) {
2959 2 : log_fatal(EUSER, "WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2960 :
2961 : /* write the new state with the new name */
2962 2 : state->need_write = 1;
2963 : }
2964 : }
2965 1517 : if (!disk) {
2966 : /* LCOV_EXCL_START */
2967 : decoding_error(path, f);
2968 : log_fatal(EUSER, "Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2969 : log_fatal(EUSER, "If you have removed it from the configuration file, please restore it\n");
2970 : /* if it's a command without UUID, it cannot autorename using UUID */
2971 : if (state->opt.skip_disk_access)
2972 : log_fatal(EUSER, "If you have renamed it, run 'sync' to update the new name\n");
2973 : exit(EXIT_FAILURE);
2974 : /* LCOV_EXCL_STOP */
2975 : }
2976 :
2977 1517 : map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2978 :
2979 1517 : tommy_list_insert_tail(&state->maplist, &map->node, map);
2980 :
2981 : /* insert in the mapping vector */
2982 1517 : tommy_array_grow(&disk_mapping, mapping_max + 1);
2983 1517 : tommy_array_set(&disk_mapping, mapping_max, disk);
2984 1517 : ++mapping_max;
2985 1785 : } else if (c == 'P') {
2986 : /* from SnapRAID 7.0 the 'P' command includes the free space */
2987 : /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2988 : char v_uuid[UUID_MAX];
2989 : uint32_t v_level;
2990 : uint32_t v_total_blocks;
2991 : uint32_t v_free_blocks;
2992 :
2993 0 : ret = sgetb32(f, &v_level);
2994 0 : if (ret < 0) {
2995 : /* LCOV_EXCL_START */
2996 : decoding_error(path, f);
2997 : os_abort();
2998 : /* LCOV_EXCL_STOP */
2999 : }
3000 :
3001 0 : ret = sgetb32(f, &v_total_blocks);
3002 0 : if (ret < 0) {
3003 : /* LCOV_EXCL_START */
3004 : decoding_error(path, f);
3005 : os_abort();
3006 : /* LCOV_EXCL_STOP */
3007 : }
3008 :
3009 0 : ret = sgetb32(f, &v_free_blocks);
3010 0 : if (ret < 0) {
3011 : /* LCOV_EXCL_START */
3012 : decoding_error(path, f);
3013 : os_abort();
3014 : /* LCOV_EXCL_STOP */
3015 : }
3016 :
3017 0 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
3018 0 : if (ret < 0) {
3019 : /* LCOV_EXCL_START */
3020 : decoding_error(path, f);
3021 : os_abort();
3022 : /* LCOV_EXCL_STOP */
3023 : }
3024 :
3025 0 : if (v_level >= LEV_MAX) {
3026 : /* LCOV_EXCL_START */
3027 : decoding_error(path, f);
3028 : log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level);
3029 : exit(EXIT_FAILURE);
3030 : /* LCOV_EXCL_STOP */
3031 : }
3032 :
3033 0 : log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
3034 : lev_config_name(v_level),
3035 0 : v_total_blocks * (uint64_t)state->block_size,
3036 0 : v_free_blocks * (uint64_t)state->block_size);
3037 :
3038 : /* auto configure if configuration is missing */
3039 0 : if (state->no_conf) {
3040 0 : if (v_level >= state->level)
3041 0 : state->level = v_level + 1;
3042 : }
3043 :
3044 : /* if we use this parity entry */
3045 0 : if (v_level < state->level) {
3046 : /* if the configuration has more splits, keep them */
3047 0 : if (state->parity[v_level].split_mac < 1)
3048 0 : state->parity[v_level].split_mac = 1;
3049 : /* set the parity info */
3050 0 : pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
3051 0 : state->parity[v_level].total_blocks = v_total_blocks;
3052 0 : state->parity[v_level].free_blocks = v_free_blocks;
3053 :
3054 0 : log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
3055 : lev_config_name(v_level), /* P command always has a single split */
3056 0 : state->parity[v_level].split_map[0].uuid,
3057 0 : esc_tag(state->parity[v_level].split_map[0].path, esc_buffer),
3058 : state->parity[v_level].split_map[0].size);
3059 : }
3060 1785 : } else if (c == 'Q') {
3061 : /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
3062 : uint32_t v_level;
3063 : uint32_t v_total_blocks;
3064 : uint32_t v_free_blocks;
3065 : uint32_t v_split_mac;
3066 : unsigned s;
3067 :
3068 1513 : ret = sgetb32(f, &v_level);
3069 1513 : if (ret < 0) {
3070 : /* LCOV_EXCL_START */
3071 : decoding_error(path, f);
3072 : os_abort();
3073 : /* LCOV_EXCL_STOP */
3074 : }
3075 :
3076 1513 : ret = sgetb32(f, &v_total_blocks);
3077 1513 : if (ret < 0) {
3078 : /* LCOV_EXCL_START */
3079 : decoding_error(path, f);
3080 : os_abort();
3081 : /* LCOV_EXCL_STOP */
3082 : }
3083 :
3084 1513 : ret = sgetb32(f, &v_free_blocks);
3085 1513 : if (ret < 0) {
3086 : /* LCOV_EXCL_START */
3087 : decoding_error(path, f);
3088 : os_abort();
3089 : /* LCOV_EXCL_STOP */
3090 : }
3091 :
3092 1513 : ret = sgetb32(f, &v_split_mac);
3093 1513 : if (ret < 0) {
3094 : /* LCOV_EXCL_START */
3095 : decoding_error(path, f);
3096 : os_abort();
3097 : /* LCOV_EXCL_STOP */
3098 : }
3099 :
3100 1513 : if (v_level >= LEV_MAX) {
3101 : /* LCOV_EXCL_START */
3102 : decoding_error(path, f);
3103 : log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level);
3104 : exit(EXIT_FAILURE);
3105 : /* LCOV_EXCL_STOP */
3106 : }
3107 :
3108 : /* auto configure if configuration is missing */
3109 1513 : if (state->no_conf) {
3110 6 : if (v_level >= state->level)
3111 5 : state->level = v_level + 1;
3112 6 : if (state->parity[v_level].split_mac < v_split_mac)
3113 6 : state->parity[v_level].split_mac = v_split_mac;
3114 : }
3115 :
3116 : /* if we use this parity entry */
3117 1513 : if (v_level < state->level) {
3118 : /* set the parity info */
3119 1245 : state->parity[v_level].total_blocks = v_total_blocks;
3120 1245 : state->parity[v_level].free_blocks = v_free_blocks;
3121 : }
3122 :
3123 1513 : log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
3124 : lev_config_name(v_level),
3125 1513 : v_total_blocks * (uint64_t)state->block_size,
3126 1513 : v_free_blocks * (uint64_t)state->block_size);
3127 :
3128 7565 : for (s = 0; s < v_split_mac; ++s) {
3129 : char v_path[PATH_MAX];
3130 : char v_uuid[UUID_MAX];
3131 : uint64_t v_size;
3132 :
3133 6052 : ret = sgetbs(f, v_path, sizeof(v_path));
3134 6052 : if (ret < 0) {
3135 : /* LCOV_EXCL_START */
3136 : decoding_error(path, f);
3137 : os_abort();
3138 : /* LCOV_EXCL_STOP */
3139 : }
3140 :
3141 6052 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
3142 6052 : if (ret < 0) {
3143 : /* LCOV_EXCL_START */
3144 : decoding_error(path, f);
3145 : os_abort();
3146 : /* LCOV_EXCL_STOP */
3147 : }
3148 :
3149 6052 : ret = sgetb64(f, &v_size);
3150 6052 : if (ret < 0) {
3151 : /* LCOV_EXCL_START */
3152 : decoding_error(path, f);
3153 : os_abort();
3154 : /* LCOV_EXCL_STOP */
3155 : }
3156 :
3157 : /* if we use this parity entry */
3158 6052 : if (v_level < state->level) {
3159 : /* if this split was removed from the configuration */
3160 4980 : if (s >= state->parity[v_level].split_mac) {
3161 : /* if the file is used, we really need it */
3162 0 : if (v_size != 0) {
3163 : /* LCOV_EXCL_START */
3164 : decoding_error(path, f);
3165 : log_fatal(EUSER, "Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
3166 : log_fatal(EUSER, "If you have removed it from the configuration file, please restore it\n");
3167 : exit(EXIT_FAILURE);
3168 : /* LCOV_EXCL_STOP */
3169 : }
3170 :
3171 : /* otherwise we can drop it */
3172 0 : log_fatal(EUSER, "WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
3173 : } else {
3174 : char parity_name[64];
3175 :
3176 : /* we copy the path only if without configuration file */
3177 4980 : if (state->no_conf)
3178 24 : pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
3179 :
3180 : /* set the split info */
3181 4980 : pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
3182 4980 : state->parity[v_level].split_map[s].size = v_size;
3183 :
3184 4980 : if (s == 0)
3185 1245 : pathcpy(parity_name, sizeof(parity_name), lev_config_name(v_level));
3186 : else
3187 3735 : pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(v_level), s);
3188 9960 : log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
3189 : parity_name,
3190 4980 : state->parity[v_level].split_map[s].uuid,
3191 4980 : esc_tag(state->parity[v_level].split_map[s].path, esc_buffer),
3192 : state->parity[v_level].split_map[s].size);
3193 : }
3194 : }
3195 : }
3196 272 : } else if (c == 'N') {
3197 : uint32_t crc_stored;
3198 : uint32_t crc_computed;
3199 :
3200 : /* get the crc before reading it from the file */
3201 272 : crc_computed = scrc(f);
3202 :
3203 272 : ret = sgetble32(f, &crc_stored);
3204 272 : if (ret < 0) {
3205 : /* LCOV_EXCL_START */
3206 : /* here don't call decoding_error() because it's too late to get the crc */
3207 : log_fatal(errno, "Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
3208 : log_fatal(errno, "This content file is damaged! Use an alternate copy.\n");
3209 : exit(EXIT_FAILURE);
3210 : /* LCOV_EXCL_STOP */
3211 : }
3212 :
3213 272 : if (crc_stored != crc_computed) {
3214 : /* LCOV_EXCL_START */
3215 : /* here don't call decoding_error() because it's too late to get the crc */
3216 : log_fatal(ECONTENT, "CRC mismatch in '%s'\n", path);
3217 : log_fatal(ECONTENT, "The content file is damaged! Please use an alternate copy.\n");
3218 : exit(EXIT_FAILURE);
3219 : /* LCOV_EXCL_STOP */
3220 : }
3221 :
3222 272 : crc_checked = 1;
3223 : } else {
3224 : /* LCOV_EXCL_START */
3225 : decoding_error(path, f);
3226 : log_fatal(ECONTENT, "Invalid command '%c'!\n", (char)c);
3227 : os_abort();
3228 : /* LCOV_EXCL_STOP */
3229 : }
3230 : }
3231 :
3232 : /* mark all disks as multi threads */
3233 1899 : for (tommy_node* i = state->disklist; i != 0; i = i->next) {
3234 1627 : struct snapraid_disk* disk = i->data;
3235 1627 : disk->single_thread = 0;
3236 : }
3237 :
3238 272 : tommy_array_done(&disk_mapping);
3239 :
3240 272 : if (serror(f)) {
3241 : /* LCOV_EXCL_START */
3242 : log_fatal(errno, "Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
3243 : exit(EXIT_FAILURE);
3244 : /* LCOV_EXCL_STOP */
3245 : }
3246 :
3247 272 : if (!crc_checked) {
3248 : /* LCOV_EXCL_START */
3249 : log_fatal(ECONTENT, "Reached the end of '%s' without finding the expected CRC\n", path);
3250 : log_fatal(ECONTENT, "This content file is truncated or damaged! Use an alternate copy.\n");
3251 : exit(EXIT_FAILURE);
3252 : /* LCOV_EXCL_STOP */
3253 : }
3254 :
3255 : /* check the file-system on all disks */
3256 272 : state_fscheck(state, "after read");
3257 :
3258 : /* check that the stored parity size matches the loaded state */
3259 272 : if (blockmax != parity_allocated_size(state)) {
3260 : /* LCOV_EXCL_START */
3261 : log_fatal(EINTERNAL, "Internal inconsistency: Parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
3262 : if (state->opt.skip_content_check) {
3263 : log_fatal(ECONTENT, "Overriding.\n");
3264 : blockmax = parity_allocated_size(state);
3265 : } else {
3266 : exit(EXIT_FAILURE);
3267 : }
3268 : /* LCOV_EXCL_STOP */
3269 : }
3270 :
3271 272 : msg_verbose("%8u files\n", count_file);
3272 272 : msg_verbose("%8u hardlinks\n", count_hardlink);
3273 272 : msg_verbose("%8u symlinks\n", count_symlink);
3274 272 : msg_verbose("%8u empty dirs\n", count_dir);
3275 :
3276 272 : log_tag("content_info:file:%u\n", count_file);
3277 272 : log_tag("content_info:hardlink:%u\n", count_hardlink);
3278 272 : log_tag("content_info:symlink:%u\n", count_symlink);
3279 272 : log_tag("content_info:dir_empty:%u\n", count_dir);
3280 :
3281 272 : log_tag("content_info:block:%u\n", blockmax);
3282 272 : log_tag("content_info:block_bad:%u\n", count_bad);
3283 272 : log_tag("content_info:block_rehash:%u\n", count_rehash);
3284 272 : log_tag("content_info:block_unsynced:%u\n", count_unsynced);
3285 272 : log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
3286 :
3287 : /* store the blocks counters */
3288 272 : state->rehash_blocks = count_rehash;
3289 272 : state->bad_blocks = count_bad;
3290 272 : state->unsynced_blocks = count_unsynced;
3291 272 : state->unscrubbed_blocks = count_unscrubbed;
3292 :
3293 : /* store the bucket info list */
3294 272 : bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
3295 272 : tommy_hashdyn_done(&bucket_hash);
3296 272 : }
3297 :
3298 : struct state_write_thread_context {
3299 : struct snapraid_state* state;
3300 : #if HAVE_MT_WRITE
3301 : thread_id_t thread;
3302 : #endif
3303 : /* input */
3304 : block_off_t blockmax;
3305 : time_t info_oldest;
3306 : time_t info_now;
3307 : int info_has_rehash;
3308 : STREAM* f;
3309 : int first;
3310 :
3311 : /* output (required to postpone the output to the terminal after the latest write) */
3312 : uint32_t crc;
3313 : unsigned count_file;
3314 : unsigned count_hardlink;
3315 : unsigned count_symlink;
3316 : unsigned count_dir;
3317 : unsigned count_bad;
3318 : unsigned count_rehash;
3319 : unsigned count_unsynced;
3320 : unsigned count_unscrubbed;
3321 : };
3322 :
3323 161 : static void* state_write_thread(void* arg)
3324 : {
3325 : char esc_buffer[ESC_MAX];
3326 161 : struct state_write_thread_context* context = arg;
3327 161 : struct snapraid_state* state = context->state;
3328 161 : block_off_t blockmax = context->blockmax;
3329 161 : time_t info_oldest = context->info_oldest;
3330 161 : time_t info_now = context->info_now;
3331 161 : int info_has_rehash = context->info_has_rehash;
3332 161 : STREAM* f = context->f;
3333 : uint32_t crc;
3334 : uint64_t t64;
3335 : unsigned count_file;
3336 : unsigned count_hardlink;
3337 : unsigned count_symlink;
3338 : unsigned count_dir;
3339 : unsigned count_bad;
3340 : unsigned count_rehash;
3341 : unsigned count_unsynced;
3342 : unsigned count_unscrubbed;
3343 : tommy_node* i;
3344 : block_off_t idx;
3345 : block_off_t begin;
3346 : unsigned l, s;
3347 : tommy_hashdyn bucket_hash;
3348 :
3349 161 : count_file = 0;
3350 161 : count_hardlink = 0;
3351 161 : count_symlink = 0;
3352 161 : count_dir = 0;
3353 161 : count_bad = 0;
3354 161 : count_rehash = 0;
3355 161 : count_unsynced = 0;
3356 161 : count_unscrubbed = 0;
3357 161 : tommy_hashdyn_init(&bucket_hash);
3358 :
3359 : /* force version 3 a we want to always store the parity size */
3360 : /* write header */
3361 161 : swrite("SNAPCNT3\n\3\0\0", 12, f);
3362 :
3363 : /* write block size and block max */
3364 161 : sputc('z', f);
3365 161 : sputb32(state->block_size, f);
3366 161 : sputc('x', f);
3367 161 : sputb32(blockmax, f);
3368 :
3369 : /* hash size */
3370 161 : sputc('y', f);
3371 161 : sputb32(BLOCK_HASH_SIZE, f);
3372 :
3373 161 : if (serror(f)) {
3374 : /* LCOV_EXCL_START */
3375 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3376 : goto bail;
3377 : /* LCOV_EXCL_STOP */
3378 : }
3379 :
3380 161 : sputc('c', f);
3381 161 : if (state->hash == HASH_MURMUR3) {
3382 106 : sputc('u', f);
3383 55 : } else if (state->hash == HASH_SPOOKY2) {
3384 55 : sputc('k', f);
3385 0 : } else if (state->hash == HASH_METRO) {
3386 0 : sputc('m', f);
3387 : } else {
3388 : /* LCOV_EXCL_START */
3389 : log_fatal(EINTERNAL, "Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
3390 : goto bail;
3391 : /* LCOV_EXCL_STOP */
3392 : }
3393 161 : swrite(state->hashseed, HASH_MAX, f);
3394 161 : if (serror(f)) {
3395 : /* LCOV_EXCL_START */
3396 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3397 : goto bail;
3398 : /* LCOV_EXCL_STOP */
3399 : }
3400 :
3401 : /* previous hash only present */
3402 161 : if (state->prevhash != HASH_UNDEFINED) {
3403 : /* if at least one rehash tag found, we have to save the previous hash */
3404 3 : if (info_has_rehash) {
3405 2 : sputc('C', f);
3406 2 : if (state->prevhash == HASH_MURMUR3) {
3407 2 : sputc('u', f);
3408 0 : } else if (state->prevhash == HASH_SPOOKY2) {
3409 0 : sputc('k', f);
3410 0 : } else if (state->prevhash == HASH_METRO) {
3411 0 : sputc('m', f);
3412 : } else {
3413 : /* LCOV_EXCL_START */
3414 : log_fatal(EINTERNAL, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
3415 : goto bail;
3416 : /* LCOV_EXCL_STOP */
3417 : }
3418 2 : swrite(state->prevhashseed, HASH_MAX, f);
3419 2 : if (serror(f)) {
3420 : /* LCOV_EXCL_START */
3421 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3422 : goto bail;
3423 : /* LCOV_EXCL_STOP */
3424 : }
3425 : }
3426 : }
3427 :
3428 : /* for each map */
3429 1127 : for (i = state->maplist; i != 0; i = i->next) {
3430 966 : struct snapraid_map* map = i->data;
3431 : struct snapraid_disk* disk;
3432 :
3433 : /* find the disk for this mapping */
3434 966 : disk = find_disk_by_name(state, map->name);
3435 966 : if (!disk) {
3436 : /* LCOV_EXCL_START */
3437 : log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name);
3438 : goto bail;
3439 : /* LCOV_EXCL_STOP */
3440 : }
3441 :
3442 : /* save the mapping only if disk is mapped */
3443 966 : if (disk->mapping_idx != -1) {
3444 874 : sputc('M', f);
3445 874 : sputbs(map->name, f);
3446 874 : sputb32(map->position, f);
3447 874 : sputb32(map->total_blocks, f);
3448 874 : sputb32(map->free_blocks, f);
3449 874 : sputbs(map->uuid, f);
3450 874 : if (context->first) {
3451 874 : log_tag("content_data:%s:%" PRIi64 ":%" PRIi64 "\n",
3452 874 : esc_tag(map->name, esc_buffer),
3453 874 : map->total_blocks * (uint64_t)state->block_size,
3454 874 : map->free_blocks * (uint64_t)state->block_size);
3455 874 : log_tag("content_data_split:%s:%s\n",
3456 874 : esc_tag(map->name, esc_buffer),
3457 874 : map->uuid);
3458 : }
3459 874 : if (serror(f)) {
3460 : /* LCOV_EXCL_START */
3461 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3462 : goto bail;
3463 : /* LCOV_EXCL_STOP */
3464 : }
3465 : }
3466 : }
3467 :
3468 : /* for each parity */
3469 1072 : for (l = 0; l < state->level; ++l) {
3470 911 : sputc('Q', f);
3471 911 : sputb32(l, f);
3472 911 : sputb32(state->parity[l].total_blocks, f);
3473 911 : sputb32(state->parity[l].free_blocks, f);
3474 911 : if (context->first) {
3475 911 : log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
3476 : lev_config_name(l),
3477 911 : state->parity[l].total_blocks * (uint64_t)state->block_size,
3478 911 : state->parity[l].free_blocks * (uint64_t)state->block_size);
3479 : }
3480 911 : sputb32(state->parity[l].split_mac, f);
3481 4555 : for (s = 0; s < state->parity[l].split_mac; ++s) {
3482 : char parity_name[64];
3483 3644 : sputbs(state->parity[l].split_map[s].path, f);
3484 3644 : sputbs(state->parity[l].split_map[s].uuid, f);
3485 3644 : sputb64(state->parity[l].split_map[s].size, f);
3486 3644 : if (s == 0)
3487 911 : pathcpy(parity_name, sizeof(parity_name), lev_config_name(l));
3488 : else
3489 2733 : pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(l), s);
3490 3644 : if (context->first) {
3491 3644 : log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
3492 : parity_name,
3493 3644 : state->parity[l].split_map[s].uuid,
3494 3644 : state->parity[l].split_map[s].path,
3495 : state->parity[l].split_map[s].size);
3496 : }
3497 : }
3498 911 : if (serror(f)) {
3499 : /* LCOV_EXCL_START */
3500 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3501 : goto bail;
3502 : /* LCOV_EXCL_STOP */
3503 : }
3504 : }
3505 :
3506 : /* for each disk */
3507 1127 : for (i = state->disklist; i != 0; i = i->next) {
3508 : tommy_node* j;
3509 966 : struct snapraid_disk* disk = i->data;
3510 :
3511 : /* if the disk is not mapped, skip it */
3512 966 : if (disk->mapping_idx < 0)
3513 92 : continue;
3514 :
3515 : /* for each file */
3516 1904060 : for (j = disk->filelist; j != 0; j = j->next) {
3517 1903186 : struct snapraid_file* file = j->data;
3518 : uint64_t size;
3519 : uint64_t mtime_sec;
3520 : int32_t mtime_nsec;
3521 : uint64_t inode;
3522 :
3523 1903186 : size = file->size;
3524 1903186 : mtime_sec = file->mtime_sec;
3525 1903186 : mtime_nsec = file->mtime_nsec;
3526 1903186 : inode = file->inode;
3527 :
3528 1903186 : sputc('f', f);
3529 1903186 : sputb32(disk->mapping_idx, f);
3530 1903186 : sputb64(size, f);
3531 1903186 : sputb64(mtime_sec, f);
3532 : /* encode STAT_NSEC_INVALID as 0 */
3533 1903186 : if (mtime_nsec == STAT_NSEC_INVALID)
3534 0 : sputb32(0, f);
3535 : else
3536 1903186 : sputb32(mtime_nsec + 1, f);
3537 1903186 : sputb64(inode, f);
3538 1903186 : sputbs(file->sub, f);
3539 1903186 : if (serror(f)) {
3540 : /* LCOV_EXCL_START */
3541 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3542 : goto bail;
3543 : /* LCOV_EXCL_STOP */
3544 : }
3545 :
3546 : /* for all the blocks of the file */
3547 1903186 : begin = 0;
3548 3817097 : while (begin < file->blockmax) {
3549 1913911 : unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3550 1913911 : block_off_t v_pos = fs_file2par_get(disk, file, begin);
3551 : uint32_t v_count;
3552 :
3553 : block_off_t end;
3554 :
3555 : /* find the end of run of blocks */
3556 1913911 : end = begin + 1;
3557 4702588 : while (end < file->blockmax) {
3558 2802073 : if (v_state != block_state_get(fs_file2block_get(file, end)))
3559 14 : break;
3560 2802059 : if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3561 13382 : break;
3562 2788677 : ++end;
3563 : }
3564 :
3565 1913911 : switch (v_state) {
3566 1813738 : case BLOCK_STATE_BLK :
3567 1813738 : sputc('b', f);
3568 1813738 : break;
3569 65899 : case BLOCK_STATE_CHG :
3570 65899 : sputc('g', f);
3571 65899 : break;
3572 34274 : case BLOCK_STATE_REP :
3573 34274 : sputc('p', f);
3574 34274 : break;
3575 0 : default :
3576 : /* LCOV_EXCL_START */
3577 : log_fatal(EINTERNAL, "Internal inconsistency: State for block %u state %u\n", v_pos, v_state);
3578 : goto bail;
3579 : /* LCOV_EXCL_STOP */
3580 : }
3581 :
3582 1913911 : sputb32(v_pos, f);
3583 :
3584 1913911 : v_count = end - begin;
3585 1913911 : sputb32(v_count, f);
3586 :
3587 : /* write hashes */
3588 6616499 : for (idx = begin; idx < end; ++idx) {
3589 4702588 : struct snapraid_block* block = fs_file2block_get(file, idx);
3590 :
3591 4702588 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3592 : }
3593 :
3594 1913911 : if (serror(f)) {
3595 : /* LCOV_EXCL_START */
3596 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3597 : goto bail;
3598 : /* LCOV_EXCL_STOP */
3599 : }
3600 :
3601 : /* next begin position */
3602 1913911 : begin = end;
3603 : }
3604 :
3605 1903186 : ++count_file;
3606 : }
3607 :
3608 : /* for each link */
3609 63138 : for (j = disk->linklist; j != 0; j = j->next) {
3610 62264 : struct snapraid_link* slink = j->data;
3611 :
3612 62264 : switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3613 354 : case FILE_IS_HARDLINK :
3614 354 : sputc('a', f);
3615 354 : ++count_hardlink;
3616 354 : break;
3617 61910 : case FILE_IS_SYMLINK :
3618 61910 : sputc('s', f);
3619 61910 : ++count_symlink;
3620 61910 : break;
3621 : }
3622 :
3623 62264 : sputb32(disk->mapping_idx, f);
3624 62264 : sputbs(slink->sub, f);
3625 62264 : sputbs(slink->linkto, f);
3626 62264 : if (serror(f)) {
3627 : /* LCOV_EXCL_START */
3628 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3629 : goto bail;
3630 : /* LCOV_EXCL_STOP */
3631 : }
3632 : }
3633 :
3634 : /* for each dir */
3635 1345 : for (j = disk->dirlist; j != 0; j = j->next) {
3636 471 : struct snapraid_dir* dir = j->data;
3637 :
3638 471 : sputc('r', f);
3639 471 : sputb32(disk->mapping_idx, f);
3640 471 : sputbs(dir->sub, f);
3641 471 : if (serror(f)) {
3642 : /* LCOV_EXCL_START */
3643 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3644 : goto bail;
3645 : /* LCOV_EXCL_STOP */
3646 : }
3647 :
3648 471 : ++count_dir;
3649 : }
3650 :
3651 : /* deleted blocks of the disk */
3652 874 : sputc('h', f);
3653 874 : sputb32(disk->mapping_idx, f);
3654 874 : if (serror(f)) {
3655 : /* LCOV_EXCL_START */
3656 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3657 : goto bail;
3658 : /* LCOV_EXCL_STOP */
3659 : }
3660 874 : begin = 0;
3661 1829 : while (begin < blockmax) {
3662 : int is_deleted;
3663 : block_off_t end;
3664 :
3665 955 : is_deleted = fs_is_block_deleted(disk, begin);
3666 :
3667 : /* find the end of run of blocks */
3668 955 : end = begin + 1;
3669 955 : while (end < blockmax
3670 4941951 : && is_deleted == fs_is_block_deleted(disk, end)
3671 : ) {
3672 4940996 : ++end;
3673 : }
3674 :
3675 955 : sputb32(end - begin, f);
3676 :
3677 955 : if (is_deleted) {
3678 : /* write the run of deleted blocks with hash */
3679 51 : sputc('o', f);
3680 :
3681 : /* write all the hash */
3682 17793 : while (begin < end) {
3683 17742 : struct snapraid_block* block = fs_par2block_get(disk, begin);
3684 :
3685 17742 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3686 :
3687 17742 : ++begin;
3688 : }
3689 : } else {
3690 : /* write the run of blocks without hash */
3691 : /* they can be either used or empty blocks */
3692 904 : sputc('O', f);
3693 :
3694 : /* next begin position */
3695 904 : begin = end;
3696 : }
3697 :
3698 955 : if (serror(f)) {
3699 : /* LCOV_EXCL_START */
3700 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3701 : goto bail;
3702 : /* LCOV_EXCL_STOP */
3703 : }
3704 : }
3705 : }
3706 :
3707 : /* write the info for each block */
3708 161 : sputc('i', f);
3709 : /* ensure to write a 64 bit time */
3710 161 : t64 = info_oldest;
3711 161 : sputb64(t64, f);
3712 161 : begin = 0;
3713 14648 : while (begin < blockmax) {
3714 : snapraid_info info;
3715 : block_off_t end;
3716 : block_off_t count;
3717 : time_t t;
3718 : unsigned flag;
3719 :
3720 14487 : info = info_get(&state->infoarr, begin);
3721 :
3722 : /* avoid this slow operation if not needed */
3723 14487 : if (context->first && fs_is_block_unsynced(state, begin))
3724 122 : ++count_unsynced;
3725 :
3726 : /* find the end of run of blocks */
3727 14487 : end = begin + 1;
3728 14487 : while (end < blockmax
3729 824833 : && info == info_get(&state->infoarr, end)
3730 : ) {
3731 810346 : if (context->first && fs_is_block_unsynced(state, end))
3732 112214 : ++count_unsynced;
3733 810346 : ++end;
3734 : }
3735 :
3736 14487 : count = end - begin;
3737 14487 : sputb32(count, f);
3738 :
3739 : /* if there is info */
3740 14487 : if (info) {
3741 : /* other flags */
3742 14459 : flag = 1; /* info is present */
3743 14459 : if (info_get_bad(info)) {
3744 1333 : flag |= 2;
3745 1333 : if (context->first)
3746 1333 : count_bad += count;
3747 : }
3748 14459 : if (info_get_rehash(info)) {
3749 4992 : flag |= 4;
3750 4992 : if (context->first)
3751 4992 : count_rehash += count;
3752 : }
3753 14459 : if (info_get_justsynced(info)) {
3754 7745 : flag |= 8;
3755 7745 : if (context->first)
3756 7745 : count_unscrubbed += count;
3757 : }
3758 14459 : sputb32(flag, f);
3759 :
3760 14459 : t = info_get_time(info);
3761 :
3762 14459 : if (context->first)
3763 14459 : bucket_insert(&bucket_hash, t, count, flag & 8);
3764 :
3765 : /* truncate any time that is in the future */
3766 14459 : if (t > info_now)
3767 0 : t = info_now;
3768 :
3769 : /* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
3770 14459 : if (t < info_oldest)
3771 0 : t = 0;
3772 : else
3773 14459 : t -= info_oldest;
3774 :
3775 : /* ensure to write a 64 bit time */
3776 14459 : t64 = t;
3777 14459 : sputb64(t64, f);
3778 : } else {
3779 : /* write a special 0 flag to mark missing info */
3780 28 : sputb32(0, f);
3781 : }
3782 :
3783 14487 : if (serror(f)) {
3784 : /* LCOV_EXCL_START */
3785 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3786 : goto bail;
3787 : /* LCOV_EXCL_STOP */
3788 : }
3789 :
3790 : /* next begin position */
3791 14487 : begin = end;
3792 : }
3793 :
3794 161 : sputc('N', f);
3795 :
3796 : /* flush data written to the disk */
3797 161 : if (sflush(f)) {
3798 : /* LCOV_EXCL_START */
3799 : log_fatal(errno, "Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3800 : goto bail;
3801 : /* LCOV_EXCL_STOP */
3802 : }
3803 :
3804 : /* get the file crc */
3805 161 : crc = scrc(f);
3806 :
3807 : /* compare the crc of the data written to file */
3808 : /* with the one of the data written to the stream */
3809 161 : if (crc != scrc_stream(f)) {
3810 : /* LCOV_EXCL_START */
3811 : log_fatal(ECONTENT, "CRC mismatch while writing the content stream.\n");
3812 : log_fatal(ECONTENT, "DANGER! Your RAM memory is faulty! DO NOT PROCEED UNTIL FIXED!\n");
3813 : log_fatal(ECONTENT, "Try running a memory test like http://www.memtest86.com/\n");
3814 : goto bail;
3815 : /* LCOV_EXCL_STOP */
3816 : }
3817 :
3818 161 : sputble32(crc, f);
3819 161 : if (serror(f)) {
3820 : /* LCOV_EXCL_START */
3821 : log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3822 : goto bail;
3823 : /* LCOV_EXCL_STOP */
3824 : }
3825 :
3826 : /* set output variables */
3827 161 : context->crc = crc;
3828 161 : context->count_file = count_file;
3829 161 : context->count_hardlink = count_hardlink;
3830 161 : context->count_symlink = count_symlink;
3831 161 : context->count_dir = count_dir;
3832 161 : context->count_bad = count_bad;
3833 161 : context->count_rehash = count_rehash;
3834 161 : context->count_unsynced = count_unsynced;
3835 161 : context->count_unscrubbed = count_unscrubbed;
3836 :
3837 : /* store the bucket info list */
3838 161 : if (context->first)
3839 161 : bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
3840 :
3841 161 : tommy_hashdyn_done(&bucket_hash);
3842 161 : return 0;
3843 :
3844 0 : bail:
3845 0 : tommy_hashdyn_done(&bucket_hash);
3846 0 : return context;
3847 : }
3848 :
3849 161 : static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3850 : {
3851 : #if HAVE_MT_WRITE
3852 : int fail;
3853 : int first;
3854 : #else
3855 : STREAM* f;
3856 : unsigned count_content;
3857 : unsigned k;
3858 : struct state_write_thread_context* context;
3859 : void* retval;
3860 : #endif
3861 : tommy_node* i;
3862 : block_off_t blockmax;
3863 : time_t info_oldest;
3864 : time_t info_now;
3865 : int info_has_rehash;
3866 : int mapping_idx;
3867 : block_off_t idx;
3868 : uint32_t crc;
3869 : unsigned count_file;
3870 : unsigned count_hardlink;
3871 : unsigned count_symlink;
3872 : unsigned count_dir;
3873 : unsigned count_bad;
3874 : unsigned count_rehash;
3875 : unsigned count_unsynced;
3876 : unsigned count_unscrubbed;
3877 :
3878 : /* blocks of all array */
3879 161 : blockmax = parity_allocated_size(state);
3880 :
3881 : /* check the file-system on all disks */
3882 161 : state_fscheck(state, "before write");
3883 :
3884 : /* clear the info for unused blocks */
3885 : /* and get some other info */
3886 161 : info_oldest = 0; /* oldest time in info */
3887 161 : info_now = time(0); /* get the present time */
3888 161 : info_has_rehash = 0; /* if there is a rehash info */
3889 824994 : for (idx = 0; idx < blockmax; ++idx) {
3890 : /* if the position is used */
3891 824833 : if (fs_position_is_required(state, idx)) {
3892 822955 : snapraid_info info = info_get(&state->infoarr, idx);
3893 :
3894 : /* only if there is some info to store */
3895 822955 : if (info) {
3896 776314 : time_t info_time = info_get_time(info);
3897 :
3898 776314 : if (!info_oldest || info_time < info_oldest)
3899 160 : info_oldest = info_time;
3900 :
3901 776314 : if (info_get_rehash(info))
3902 13912 : info_has_rehash = 1;
3903 : }
3904 : } else {
3905 : /* clear any previous info */
3906 1878 : info_set(&state->infoarr, idx, 0);
3907 :
3908 : /* and clear any deleted blocks */
3909 1878 : fs_position_clear_deleted(state, idx);
3910 : }
3911 : }
3912 :
3913 : /* map disks */
3914 161 : mapping_idx = 0;
3915 1127 : for (i = state->maplist; i != 0; i = i->next) {
3916 966 : struct snapraid_map* map = i->data;
3917 : struct snapraid_disk* disk;
3918 :
3919 : /* find the disk for this mapping */
3920 966 : disk = find_disk_by_name(state, map->name);
3921 966 : if (!disk) {
3922 : /* LCOV_EXCL_START */
3923 : log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name);
3924 : os_abort();
3925 : /* LCOV_EXCL_STOP */
3926 : }
3927 :
3928 : /* save the mapping only for not empty disks */
3929 966 : if (!fs_is_empty(disk, blockmax)) {
3930 : /* assign the mapping index used to identify disks */
3931 874 : disk->mapping_idx = mapping_idx;
3932 874 : ++mapping_idx;
3933 : } else {
3934 : /* mark the disk as without mapping */
3935 92 : disk->mapping_idx = -1;
3936 : }
3937 : }
3938 :
3939 : #if HAVE_MT_WRITE
3940 : /* start all writing threads */
3941 : first = 1;
3942 : i = tommy_list_head(&state->contentlist);
3943 : while (i) {
3944 : struct snapraid_content* content = i->data;
3945 : struct state_write_thread_context* context;
3946 : char tmp[PATH_MAX];
3947 : char esc_buffer[ESC_MAX];
3948 : STREAM* f;
3949 :
3950 : msg_progress("Saving state to %s...\n", content->content);
3951 : log_tag("content_write:%s\n", esc_tag(content->content, esc_buffer));
3952 :
3953 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3954 :
3955 : /* ensure to delete a previous stale file */
3956 : if (remove(tmp) != 0) {
3957 : if (errno != ENOENT) {
3958 : /* LCOV_EXCL_START */
3959 : log_fatal(errno, "Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
3960 : exit(EXIT_FAILURE);
3961 : /* LCOV_EXCL_STOP */
3962 : }
3963 : }
3964 :
3965 : f = sopen_write(tmp);
3966 : if (f == 0) {
3967 : /* LCOV_EXCL_START */
3968 : log_fatal(errno, "Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3969 : exit(EXIT_FAILURE);
3970 : /* LCOV_EXCL_STOP */
3971 : }
3972 :
3973 : /* allocate the thread context */
3974 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3975 : content->context = context;
3976 :
3977 : /* initialize */
3978 : context->state = state;
3979 : context->blockmax = blockmax;
3980 : context->info_oldest = info_oldest;
3981 : context->info_now = info_now;
3982 : context->info_has_rehash = info_has_rehash;
3983 : context->f = f;
3984 : context->first = first;
3985 : first = 0;
3986 :
3987 : thread_create(&context->thread, state_write_thread, context);
3988 :
3989 : i = i->next;
3990 : }
3991 :
3992 : /* join all thread */
3993 : fail = 0;
3994 : first = 1;
3995 : crc = 0;
3996 : count_file = 0;
3997 : count_hardlink = 0;
3998 : count_symlink = 0;
3999 : count_dir = 0;
4000 : count_bad = 0;
4001 : count_rehash = 0;
4002 : count_unsynced = 0;
4003 : count_unscrubbed = 0;
4004 : i = tommy_list_head(&state->contentlist);
4005 : while (i) {
4006 : struct snapraid_content* content = i->data;
4007 : struct state_write_thread_context* context = content->context;
4008 : void* retval;
4009 :
4010 : thread_join(context->thread, &retval);
4011 :
4012 : if (retval) {
4013 : /* LCOV_EXCL_START */
4014 : fail = 1;
4015 : /* LCOV_EXCL_STOP */
4016 : } else {
4017 : STREAM* f = context->f;
4018 :
4019 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
4020 : /* than even in a system crash event we have one valid copy of the file. */
4021 : if (sflush(f) != 0) {
4022 : /* LCOV_EXCL_START */
4023 : log_fatal(errno, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
4024 : exit(EXIT_FAILURE);
4025 : /* LCOV_EXCL_STOP */
4026 : }
4027 :
4028 : #if HAVE_FSYNC
4029 : if (ssync(f) != 0) {
4030 : /* LCOV_EXCL_START */
4031 : log_fatal(errno, "Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
4032 : exit(EXIT_FAILURE);
4033 : /* LCOV_EXCL_STOP */
4034 : }
4035 : #endif
4036 :
4037 : if (sclose(f) != 0) {
4038 : /* LCOV_EXCL_START */
4039 : log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
4040 : exit(EXIT_FAILURE);
4041 : /* LCOV_EXCL_STOP */
4042 : }
4043 :
4044 : if (first) {
4045 : first = 0;
4046 : crc = context->crc;
4047 : count_file = context->count_file;
4048 : count_hardlink = context->count_hardlink;
4049 : count_symlink = context->count_symlink;
4050 : count_dir = context->count_dir;
4051 : cound_bad = context->count_bad;
4052 : count_rehash = context->count_rehash;
4053 : count_unsynced = context->count_unsynced;
4054 : count_unscrubbed = context->count_unscrubbed;
4055 : } else {
4056 : if (crc != context->crc) {
4057 : /* LCOV_EXCL_START */
4058 : log_fatal(ECONTENT, "Different CRCs writing content streams.\n");
4059 : log_fatal(ECONTENT, "DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
4060 : log_fatal(ECONTENT, "Try running a memory test like http://www.memtest86.com/\n");
4061 : exit(EXIT_FAILURE);
4062 : /* LCOV_EXCL_STOP */
4063 : }
4064 : }
4065 : }
4066 :
4067 : free(context);
4068 :
4069 : i = i->next;
4070 : }
4071 :
4072 : /* abort on failure */
4073 : if (fail) {
4074 : /* LCOV_EXCL_START */
4075 : exit(EXIT_FAILURE);
4076 : /* LCOV_EXCL_STOP */
4077 : }
4078 : #else
4079 : /* count the content files */
4080 161 : count_content = 0;
4081 161 : i = tommy_list_head(&state->contentlist);
4082 1233 : while (i) {
4083 1072 : struct snapraid_content* content = i->data;
4084 1072 : msg_progress("Saving state to %s...\n", content->content);
4085 1072 : ++count_content;
4086 1072 : i = i->next;
4087 : }
4088 :
4089 : /* open all the content files */
4090 161 : f = sopen_multi_write(count_content, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
4091 161 : if (!f) {
4092 : /* LCOV_EXCL_START */
4093 : log_fatal(errno, "Error opening the content files.\n");
4094 : exit(EXIT_FAILURE);
4095 : /* LCOV_EXCL_STOP */
4096 : }
4097 :
4098 161 : k = 0;
4099 161 : i = tommy_list_head(&state->contentlist);
4100 1233 : while (i) {
4101 1072 : struct snapraid_content* content = i->data;
4102 : char tmp[PATH_MAX];
4103 1072 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4104 :
4105 : /* ensure to delete a previous stale file */
4106 1072 : if (remove(tmp) != 0) {
4107 1072 : if (errno != ENOENT) {
4108 : /* LCOV_EXCL_START */
4109 : log_fatal(errno, "Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
4110 : exit(EXIT_FAILURE);
4111 : /* LCOV_EXCL_STOP */
4112 : }
4113 : }
4114 :
4115 1072 : if (sopen_multi_file(f, k, tmp) != 0) {
4116 : /* LCOV_EXCL_START */
4117 : log_fatal(errno, "Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
4118 : exit(EXIT_FAILURE);
4119 : /* LCOV_EXCL_STOP */
4120 : }
4121 :
4122 1072 : ++k;
4123 1072 : i = i->next;
4124 : }
4125 :
4126 : /* allocate the thread context */
4127 161 : context = malloc_nofail(sizeof(struct state_write_thread_context));
4128 :
4129 : /* initialize */
4130 161 : context->state = state;
4131 161 : context->blockmax = blockmax;
4132 161 : context->info_oldest = info_oldest;
4133 161 : context->info_now = info_now;
4134 161 : context->info_has_rehash = info_has_rehash;
4135 161 : context->f = f;
4136 161 : context->first = 1;
4137 :
4138 161 : retval = state_write_thread(context);
4139 :
4140 : /* abort on failure */
4141 161 : if (retval) {
4142 : /* LCOV_EXCL_START */
4143 : exit(EXIT_FAILURE);
4144 : /* LCOV_EXCL_STOP */
4145 : }
4146 :
4147 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
4148 : /* than even in a system crash event we have one valid copy of the file. */
4149 161 : if (sflush(f) != 0) {
4150 : /* LCOV_EXCL_START */
4151 : log_fatal(errno, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
4152 : exit(EXIT_FAILURE);
4153 : /* LCOV_EXCL_STOP */
4154 : }
4155 :
4156 : #if HAVE_FSYNC
4157 161 : if (ssync(f) != 0) {
4158 : /* LCOV_EXCL_START */
4159 : log_fatal(errno, "Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
4160 : exit(EXIT_FAILURE);
4161 : /* LCOV_EXCL_STOP */
4162 : }
4163 : #endif
4164 :
4165 161 : if (sclose(f) != 0) {
4166 : /* LCOV_EXCL_START */
4167 : log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
4168 : exit(EXIT_FAILURE);
4169 : /* LCOV_EXCL_STOP */
4170 : }
4171 :
4172 161 : crc = context->crc;
4173 161 : count_file = context->count_file;
4174 161 : count_hardlink = context->count_hardlink;
4175 161 : count_symlink = context->count_symlink;
4176 161 : count_dir = context->count_dir;
4177 161 : count_bad = context->count_bad;
4178 161 : count_rehash = context->count_rehash;
4179 161 : count_unsynced = context->count_unsynced;
4180 161 : count_unscrubbed = context->count_unscrubbed;
4181 :
4182 161 : free(context);
4183 : #endif
4184 :
4185 161 : msg_verbose("%8u files\n", count_file);
4186 161 : msg_verbose("%8u hardlinks\n", count_hardlink);
4187 161 : msg_verbose("%8u symlinks\n", count_symlink);
4188 161 : msg_verbose("%8u empty dirs\n", count_dir);
4189 :
4190 161 : log_tag("content_info:file:%u\n", count_file);
4191 161 : log_tag("content_info:hardlink:%u\n", count_hardlink);
4192 161 : log_tag("content_info:symlink:%u\n", count_symlink);
4193 161 : log_tag("content_info:dir_empty:%u\n", count_dir);
4194 :
4195 161 : log_tag("content_info:block:%u\n", blockmax);
4196 161 : log_tag("content_info:block_bad:%u\n", count_bad);
4197 161 : log_tag("content_info:block_rehash:%u\n", count_rehash);
4198 161 : log_tag("content_info:block_unsynced:%u\n", count_unsynced);
4199 161 : log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
4200 :
4201 : /* store the blocks counters */
4202 161 : state->rehash_blocks = count_rehash;
4203 161 : state->bad_blocks = count_bad;
4204 161 : state->unsynced_blocks = count_unsynced;
4205 161 : state->unscrubbed_blocks = count_unscrubbed;
4206 :
4207 161 : *out_crc = crc;
4208 161 : }
4209 :
4210 280 : void state_read(struct snapraid_state* state)
4211 : {
4212 : STREAM* f;
4213 : char path[PATH_MAX];
4214 : struct stat st;
4215 : tommy_node* node;
4216 : int ret;
4217 : int c;
4218 :
4219 : /* iterate over all the available content files and load the first one present */
4220 280 : f = 0;
4221 280 : node = tommy_list_head(&state->contentlist);
4222 330 : while (node) {
4223 323 : struct snapraid_content* content = node->data;
4224 323 : pathcpy(path, sizeof(path), content->content);
4225 :
4226 323 : if (!state->no_conf) {
4227 : char esc_buffer[ESC_MAX];
4228 322 : log_tag("content:%s\n", esc_tag(path, esc_buffer));
4229 322 : log_flush();
4230 : }
4231 323 : msg_progress("Loading state from %s...\n", path);
4232 :
4233 323 : f = sopen_read(path, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
4234 323 : if (f != 0) {
4235 : /* if opened stop the search */
4236 273 : break;
4237 : } else {
4238 : /* if it's real error of an existing file, abort */
4239 50 : if (errno != ENOENT) {
4240 : /* LCOV_EXCL_START */
4241 : log_fatal(errno, "Error opening the content file '%s'. %s.\n", path, strerror(errno));
4242 : exit(EXIT_FAILURE);
4243 : /* LCOV_EXCL_STOP */
4244 : }
4245 :
4246 : /* otherwise continue */
4247 50 : if (node->next) {
4248 43 : log_fatal(errno, "WARNING! Content file '%s' not found, attempting with another copy...\n", path);
4249 :
4250 : /* ensure to rewrite all the content files */
4251 43 : state->need_write = 1;
4252 : }
4253 : }
4254 :
4255 : /* next content file */
4256 50 : node = node->next;
4257 : }
4258 :
4259 : /* if not found, assume empty */
4260 280 : if (!f) {
4261 7 : log_fatal(EUSER, "No content file found. Assuming empty.\n");
4262 :
4263 : /* create the initial mapping */
4264 7 : state_map(state);
4265 7 : return;
4266 : }
4267 :
4268 : /* get the stat of the content file */
4269 273 : ret = fstat(shandle(f), &st);
4270 273 : if (ret != 0) {
4271 : /* LCOV_EXCL_START */
4272 : log_fatal(errno, "Error stating the content file '%s'. %s.\n", path, strerror(errno));
4273 : exit(EXIT_FAILURE);
4274 : /* LCOV_EXCL_STOP */
4275 : }
4276 :
4277 : /* go further to check other content files */
4278 1824 : while (node) {
4279 : char other_path[PATH_MAX];
4280 : struct stat other_st;
4281 1551 : struct snapraid_content* content = node->data;
4282 1551 : pathcpy(other_path, sizeof(other_path), content->content);
4283 :
4284 1551 : ret = stat(other_path, &other_st);
4285 1551 : if (ret != 0) {
4286 : /* allow missing content files, but not any other kind of error */
4287 1 : if (errno != ENOENT) {
4288 : /* LCOV_EXCL_START */
4289 : log_fatal(errno, "Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
4290 : exit(EXIT_FAILURE);
4291 : /* LCOV_EXCL_STOP */
4292 : }
4293 :
4294 : /* ensure to rewrite all the content files */
4295 1 : state->need_write = 1;
4296 : } else {
4297 : /* if the size is different */
4298 1550 : if (other_st.st_size != st.st_size) {
4299 39 : log_fatal(ECONTENT, "WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
4300 39 : log_fatal(ECONTENT, "Likely one of the two is broken!\n");
4301 :
4302 : /* ensure to rewrite all the content files */
4303 39 : state->need_write = 1;
4304 : }
4305 : }
4306 :
4307 : /* next content file */
4308 1551 : node = node->next;
4309 : }
4310 :
4311 : /* start with a undefined default. */
4312 : /* it's for compatibility with version 1.0 where MD5 was implicit. */
4313 273 : state->hash = HASH_UNDEFINED;
4314 :
4315 : /* start with a zero seed, it was the default in old versions */
4316 273 : memset(state->hashseed, 0, HASH_MAX);
4317 :
4318 : /* previous hash, start with an undefined value */
4319 273 : state->prevhash = HASH_UNDEFINED;
4320 :
4321 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
4322 :
4323 : /* get the first char to detect the file type */
4324 273 : c = sgetc(f);
4325 273 : sungetc(c, f);
4326 :
4327 : /* guess the file type from the first char */
4328 273 : if (c == 'S') {
4329 273 : state_read_content(state, path, f);
4330 : } else {
4331 : /* LCOV_EXCL_START */
4332 : log_fatal(EUSER, "From SnapRAID v9.0 the text content file is not supported anymore.\n");
4333 : log_fatal(EUSER, "You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
4334 : exit(EXIT_FAILURE);
4335 : /* LCOV_EXCL_STOP */
4336 : }
4337 :
4338 272 : sclose(f);
4339 :
4340 272 : if (state->hash == HASH_UNDEFINED) {
4341 : /* LCOV_EXCL_START */
4342 : log_fatal(EUSER, "The checksum to use is not specified.\n");
4343 : log_fatal(EUSER, "This happens because you are likely upgrading from SnapRAID 1.0.\n");
4344 : log_fatal(EUSER, "To use a new SnapRAID you must restart from scratch,\n");
4345 : log_fatal(EUSER, "deleting all the content and parity files.\n");
4346 : exit(EXIT_FAILURE);
4347 : /* LCOV_EXCL_STOP */
4348 : }
4349 :
4350 : /* update the mapping */
4351 272 : state_map(state);
4352 :
4353 272 : state_content_check(state, path);
4354 :
4355 : /* mark that we read the content file, and it passed all the checks */
4356 272 : state->checked_read = 1;
4357 : }
4358 :
4359 : struct state_verify_thread_context {
4360 : struct snapraid_state* state;
4361 : struct snapraid_content* content;
4362 : #if HAVE_MT_VERIFY
4363 : thread_id_t thread;
4364 : #else
4365 : void* retval;
4366 : #endif
4367 : /* input */
4368 : uint32_t crc;
4369 : STREAM* f;
4370 : };
4371 :
4372 1072 : static void* state_verify_thread(void* arg)
4373 : {
4374 1072 : struct state_verify_thread_context* context = arg;
4375 1072 : struct snapraid_content* content = context->content;
4376 1072 : STREAM* f = context->f;
4377 : unsigned char buf[4];
4378 : uint32_t crc_stored;
4379 : uint32_t crc_computed;
4380 : uint64_t start;
4381 :
4382 1072 : start = os_tick_ms();
4383 :
4384 1072 : if (sdeplete(f, buf) != 0) {
4385 : /* LCOV_EXCL_START */
4386 : log_fatal(errno, "Failed to flush content file '%s'. %s.\n", serrorfile(f), strerror(errno));
4387 : return context;
4388 : /* LCOV_EXCL_STOP */
4389 : }
4390 :
4391 : /* get the stored crc from the last four bytes */
4392 1072 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
4393 :
4394 1072 : if (crc_stored != context->crc) {
4395 : /* LCOV_EXCL_START */
4396 : log_fatal(ECONTENT, "DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
4397 : return context;
4398 : /* LCOV_EXCL_STOP */
4399 : }
4400 :
4401 : /* get the computed crc */
4402 1072 : crc_computed = scrc(f);
4403 :
4404 : /* adjust the stored crc to include itself */
4405 1072 : crc_stored = crc32c(crc_stored, buf, 4);
4406 :
4407 1072 : if (crc_computed != crc_stored) {
4408 : /* LCOV_EXCL_START */
4409 : log_fatal(ECONTENT, "DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
4410 : return context;
4411 : /* LCOV_EXCL_STOP */
4412 : }
4413 :
4414 1072 : msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (os_tick_ms() - start) / 1000);
4415 :
4416 1072 : return 0;
4417 : }
4418 :
4419 161 : static void state_verify_content(struct snapraid_state* state, uint32_t crc)
4420 : {
4421 : tommy_node* i;
4422 : int fail;
4423 :
4424 161 : msg_progress("Verifying...\n");
4425 :
4426 : /* start all reading threads */
4427 161 : i = tommy_list_head(&state->contentlist);
4428 1233 : while (i) {
4429 1072 : struct snapraid_content* content = i->data;
4430 : struct state_verify_thread_context* context;
4431 : char tmp[PATH_MAX];
4432 : STREAM* f;
4433 :
4434 1072 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4435 1072 : f = sopen_read(tmp, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
4436 1072 : if (f == 0) {
4437 : /* LCOV_EXCL_START */
4438 : log_fatal(errno, "Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
4439 : exit(EXIT_FAILURE);
4440 : /* LCOV_EXCL_STOP */
4441 : }
4442 :
4443 : /* allocate the thread context */
4444 1072 : context = malloc_nofail(sizeof(struct state_verify_thread_context));
4445 1072 : content->context = context;
4446 :
4447 : /* initialize */
4448 1072 : context->state = state;
4449 1072 : context->content = content;
4450 1072 : context->crc = crc;
4451 1072 : context->f = f;
4452 :
4453 : #if HAVE_MT_VERIFY
4454 1072 : thread_create(&context->thread, state_verify_thread, context);
4455 : #else
4456 : context->retval = state_verify_thread(context);
4457 : #endif
4458 :
4459 1072 : i = i->next;
4460 : }
4461 :
4462 : /* join all thread */
4463 161 : fail = 0;
4464 161 : i = tommy_list_head(&state->contentlist);
4465 1233 : while (i) {
4466 1072 : struct snapraid_content* content = i->data;
4467 1072 : struct state_verify_thread_context* context = content->context;
4468 : void* retval;
4469 :
4470 : #if HAVE_MT_VERIFY
4471 1072 : thread_join(context->thread, &retval);
4472 : #else
4473 : retval = context->retval;
4474 : #endif
4475 1072 : if (retval) {
4476 : /* LCOV_EXCL_START */
4477 : fail = 1;
4478 : /* LCOV_EXCL_STOP */
4479 : } else {
4480 1072 : STREAM* f = context->f;
4481 :
4482 1072 : if (sclose(f) != 0) {
4483 : /* LCOV_EXCL_START */
4484 : log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
4485 : exit(EXIT_FAILURE);
4486 : /* LCOV_EXCL_STOP */
4487 : }
4488 : }
4489 :
4490 1072 : free(context);
4491 :
4492 1072 : i = i->next;
4493 : }
4494 :
4495 : /* abort on failure */
4496 161 : if (fail) {
4497 : /* LCOV_EXCL_START */
4498 : exit(EXIT_FAILURE);
4499 : /* LCOV_EXCL_STOP */
4500 : }
4501 161 : }
4502 :
4503 161 : static void state_rename_content(struct snapraid_state* state)
4504 : {
4505 : tommy_node* i;
4506 :
4507 : #if defined(_linux) /* this sequence is linux specific */
4508 : i = tommy_list_head(&state->contentlist);
4509 : while (i) {
4510 : struct snapraid_content* content = i->data;
4511 : char tmp[PATH_MAX];
4512 : char dir[PATH_MAX];
4513 : char* slash;
4514 : int handle;
4515 :
4516 : pathcpy(dir, sizeof(dir), content->content);
4517 :
4518 : slash = strrchr(tmp, '/');
4519 : if (slash)
4520 : *slash = 0;
4521 : else
4522 : pathcpy(dir, sizeof(dir), ".");
4523 :
4524 : /* open the directory to get the handle */
4525 : handle = open(dir, O_RDONLY | O_DIRECTORY);
4526 : if (handle < 0) {
4527 : /* LCOV_EXCL_START */
4528 : log_fatal(errno, "Error opening the directory '%s'. %s.\n", dir, strerror(errno));
4529 : exit(EXIT_FAILURE);
4530 : /* LCOV_EXCL_STOP */
4531 : }
4532 :
4533 : /* now rename the just written copy with the correct name */
4534 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4535 : if (rename(tmp, content->content) != 0) {
4536 : /* LCOV_EXCL_START */
4537 : log_fatal(errno, "Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
4538 : exit(EXIT_FAILURE);
4539 : /* LCOV_EXCL_STOP */
4540 : }
4541 :
4542 : /* sync the directory */
4543 : if (fsync(handle) != 0) {
4544 : /* LCOV_EXCL_START */
4545 : log_fatal(errno, "Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
4546 : exit(EXIT_FAILURE);
4547 : /* LCOV_EXCL_STOP */
4548 : }
4549 :
4550 : if (close(handle) != 0) {
4551 : /* LCOV_EXCL_START */
4552 : log_fatal(errno, "Error closing the directory '%s'. %s.\n", dir, strerror(errno));
4553 : exit(EXIT_FAILURE);
4554 : /* LCOV_EXCL_STOP */
4555 : }
4556 :
4557 : i = i->next;
4558 : }
4559 : #else
4560 161 : i = tommy_list_head(&state->contentlist);
4561 1233 : while (i) {
4562 1072 : struct snapraid_content* content = i->data;
4563 : char tmp[PATH_MAX];
4564 :
4565 : /* now renames the just written copy with the correct name */
4566 1072 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4567 1072 : if (rename(tmp, content->content) != 0) {
4568 : /* LCOV_EXCL_START */
4569 : log_fatal(errno, "Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
4570 : exit(EXIT_FAILURE);
4571 : /* LCOV_EXCL_STOP */
4572 : }
4573 :
4574 1072 : i = i->next;
4575 : }
4576 : #endif
4577 161 : }
4578 :
4579 161 : void state_write(struct snapraid_state* state)
4580 : {
4581 : uint32_t crc;
4582 :
4583 : /* write all the content files */
4584 161 : state_write_content(state, &crc);
4585 :
4586 : /* verify the just written files */
4587 161 : state_verify_content(state, crc);
4588 :
4589 : /* rename the new files, over the old ones */
4590 161 : state_rename_content(state);
4591 :
4592 161 : state->need_write = 0; /* no write needed anymore */
4593 161 : state->checked_read = 0; /* what we wrote is not checked in read */
4594 161 : state->written = 1;
4595 161 : }
4596 :
4597 127 : void state_skip(struct snapraid_state* state)
4598 : {
4599 : tommy_node* i;
4600 :
4601 : /* for each disk */
4602 885 : for (i = state->disklist; i != 0; i = i->next) {
4603 : tommy_node* j;
4604 758 : struct snapraid_disk* disk = i->data;
4605 :
4606 758 : if (!disk->skip_access)
4607 757 : continue;
4608 :
4609 : /* for each file */
4610 1885 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4611 1884 : struct snapraid_file* file = j->data;
4612 1884 : file_flag_set(file, FILE_IS_EXCLUDED);
4613 : }
4614 :
4615 : /* for each link */
4616 57 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4617 56 : struct snapraid_link* slink = j->data;
4618 56 : link_flag_set(slink, FILE_IS_EXCLUDED);
4619 : }
4620 :
4621 : /* for each dir */
4622 1 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4623 0 : struct snapraid_dir* dir = j->data;
4624 0 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4625 : }
4626 : }
4627 127 : }
4628 :
4629 127 : void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
4630 : {
4631 : tommy_node* i;
4632 : unsigned l;
4633 :
4634 : /* if no filter, include all */
4635 127 : if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
4636 117 : return;
4637 :
4638 10 : msg_progress("Selecting...\n");
4639 :
4640 19 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4641 9 : struct snapraid_filter* filter = i->data;
4642 9 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4643 : }
4644 13 : for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4645 3 : struct snapraid_filter* filter = i->data;
4646 3 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4647 : }
4648 10 : if (filter_missing)
4649 2 : msg_verbose("\t<missing>\n");
4650 10 : if (filter_error)
4651 2 : msg_verbose("\t<error>\n");
4652 :
4653 : /* for each disk */
4654 70 : for (i = state->disklist; i != 0; i = i->next) {
4655 : tommy_node* j;
4656 60 : struct snapraid_disk* disk = i->data;
4657 :
4658 : /* if we filter for presence, we have to access the disk, so better to print something */
4659 60 : if (filter_missing)
4660 12 : msg_progress("Scanning disk %s...\n", disk->name);
4661 :
4662 : /* for each file */
4663 150479 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4664 150419 : struct snapraid_file* file = j->data;
4665 :
4666 150419 : if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4667 80894 : || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4668 75166 : || filter_existence(filter_missing, disk->dir, file->sub) != 0
4669 35970 : || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4670 : ) {
4671 128716 : file_flag_set(file, FILE_IS_EXCLUDED);
4672 : }
4673 : }
4674 :
4675 : /* for each link */
4676 4935 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4677 4875 : struct snapraid_link* slink = j->data;
4678 :
4679 4875 : if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4680 2618 : || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4681 2407 : || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4682 : ) {
4683 3852 : link_flag_set(slink, FILE_IS_EXCLUDED);
4684 : }
4685 : }
4686 :
4687 : /* for each empty dir */
4688 92 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4689 32 : struct snapraid_dir* dir = j->data;
4690 :
4691 32 : if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4692 17 : || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4693 17 : || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4694 : ) {
4695 23 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4696 : }
4697 : }
4698 : }
4699 :
4700 : /* if we are filtering by disk, exclude any parity not explicitly included */
4701 10 : if (!tommy_list_empty(filterlist_disk)) {
4702 : /* for each parity disk */
4703 24 : for (l = 0; l < state->level; ++l) {
4704 : /* check if the parity is excluded by name */
4705 19 : if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4706 : /* excluded the parity from further operation */
4707 13 : state->parity[l].is_excluded_by_filter = 1;
4708 : }
4709 : }
4710 : } else {
4711 : /* if we are filtering by file, exclude all parity */
4712 5 : if (filter_missing || !tommy_list_empty(filterlist_file)) {
4713 : /* for each parity disk */
4714 16 : for (l = 0; l < state->level; ++l) {
4715 13 : state->parity[l].is_excluded_by_filter = 1;
4716 : }
4717 : }
4718 : }
4719 : }
4720 :
4721 235 : int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4722 : {
4723 : time_t now;
4724 :
4725 235 : if (state->opt.gui) {
4726 5 : log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4727 5 : log_flush();
4728 : }
4729 :
4730 235 : now = time(0);
4731 :
4732 235 : state->progress_whole_start = now;
4733 235 : state->progress_tick = 0;
4734 235 : state->progress_ptr = 0;
4735 235 : state->progress_wasted = 0;
4736 :
4737 : /* start of thermal control */
4738 235 : if (!state_thermal_begin(state, now))
4739 0 : return -1;
4740 :
4741 : /* stop if requested */
4742 235 : if (global_interrupt) {
4743 : /* LCOV_EXCL_START */
4744 : if (!state->opt.gui) {
4745 : msg_status("Not starting due to interruption\n");
4746 : }
4747 : log_tag("sigint:0: SIGINT received\n");
4748 : log_flush();
4749 : return 1;
4750 : /* LCOV_EXCL_STOP */
4751 : }
4752 :
4753 235 : return 0;
4754 : }
4755 :
4756 235 : void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize, const char* msg)
4757 : {
4758 235 : if (state->opt.gui) {
4759 5 : log_tag("run:end\n");
4760 5 : log_flush();
4761 230 : } else if (countmax == 0) {
4762 22 : if (state->need_write || state->written) {
4763 17 : msg_status("100%% completed\n");
4764 : } else {
4765 5 : msg_status("%s", msg);
4766 : }
4767 : } else {
4768 : time_t now;
4769 : time_t elapsed;
4770 :
4771 208 : unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4772 :
4773 208 : now = time(0);
4774 :
4775 208 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4776 :
4777 208 : msg_bar("%u%% completed, %u MB accessed", muldiv(countpos, 100, countmax), countsize_MB);
4778 :
4779 208 : msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4780 :
4781 208 : msg_bar("\n");
4782 208 : msg_flush();
4783 : }
4784 235 : }
4785 :
4786 1 : void state_progress_stop(struct snapraid_state* state)
4787 : {
4788 : time_t now;
4789 :
4790 1 : now = time(0);
4791 :
4792 1 : state->progress_interruption = now;
4793 1 : }
4794 :
4795 1 : void state_progress_restart(struct snapraid_state* state)
4796 : {
4797 : time_t now;
4798 :
4799 1 : now = time(0);
4800 :
4801 : /* reset the progress counter */
4802 1 : state->progress_tick = 0;
4803 1 : state->progress_ptr = 0;
4804 :
4805 1 : if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4806 1 : state->progress_wasted += now - state->progress_interruption;
4807 1 : }
4808 :
4809 : /**
4810 : * Set the latest os_tick data in the progress vector.
4811 : */
4812 511 : static void state_progress_latest(struct snapraid_state* state)
4813 : {
4814 : tommy_node* i;
4815 : unsigned l;
4816 :
4817 511 : state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4818 511 : state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4819 511 : state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4820 511 : state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4821 511 : state->progress_tick_io[state->progress_ptr] = state->tick_io;
4822 3569 : for (i = state->disklist; i != 0; i = i->next) {
4823 3058 : struct snapraid_disk* disk = i->data;
4824 3058 : disk->progress_tick[state->progress_ptr] = disk->tick;
4825 : }
4826 2902 : for (l = 0; l < state->level; ++l)
4827 2391 : state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4828 511 : }
4829 :
4830 : /**
4831 : * Number of columns
4832 : */
4833 : #define GRAPH_COLUMN 78
4834 :
4835 : /**
4836 : * Get the reference value, 0 if index is PROGRESS_MAX
4837 : */
4838 : #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4839 :
4840 0 : static struct snapraid_thermal* state_thermal_find(struct snapraid_state* state, const char* name)
4841 : {
4842 : tommy_node* i;
4843 0 : struct snapraid_thermal* found = 0;
4844 :
4845 0 : for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
4846 0 : struct snapraid_thermal* thermal = i->data;
4847 0 : if (strcmp(thermal->name, name) == 0) {
4848 0 : if (found == 0
4849 : /* if multiple matches, return the one with highest temperature */
4850 0 : || found->latest_temperature < thermal->latest_temperature
4851 : /* or, if the temperature is equal, the one with the higher r_squared */
4852 0 : || (found->latest_temperature == thermal->latest_temperature && found->params.r_squared < thermal->params.r_squared)
4853 : ) {
4854 0 : found = thermal;
4855 : }
4856 : }
4857 : }
4858 :
4859 0 : return found;
4860 : }
4861 :
4862 : #define THERMAL_PAD " "
4863 :
4864 1 : static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4865 : {
4866 : uint64_t v;
4867 : uint64_t tick_total;
4868 : unsigned bar;
4869 : tommy_node* i;
4870 : unsigned l;
4871 : size_t pad;
4872 : size_t pre;
4873 :
4874 1 : tick_total = 0;
4875 :
4876 1 : tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4877 1 : tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4878 1 : tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4879 1 : tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4880 1 : tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4881 1 : if (!tick_total)
4882 0 : return;
4883 :
4884 1 : pad = 4;
4885 7 : for (i = state->disklist; i != 0; i = i->next) {
4886 6 : struct snapraid_disk* disk = i->data;
4887 6 : size_t len = strlen(disk->name);
4888 6 : if (pad < len)
4889 1 : pad = len;
4890 : }
4891 7 : for (l = 0; l < state->level; ++l) {
4892 6 : size_t len = strlen(lev_config_name(l));
4893 6 : if (pad < len)
4894 2 : pad = len;
4895 : }
4896 :
4897 : /* extra space */
4898 1 : pad += 1;
4899 :
4900 1 : pre = 4;
4901 1 : if (pad + pre + 30 < GRAPH_COLUMN)
4902 1 : bar = GRAPH_COLUMN - pad - pre;
4903 : else
4904 0 : bar = 30; /* at least a bar of 30 */
4905 :
4906 1 : if (io) {
4907 0 : const char* legend = "cached blocks (instant, more is better)";
4908 :
4909 : /* refresh the cached blocks info */
4910 0 : io_refresh(io);
4911 :
4912 0 : printf("\n");
4913 :
4914 0 : for (i = state->disklist; i != 0; i = i->next) {
4915 0 : struct snapraid_disk* disk = i->data;
4916 0 : v = disk->cached_blocks;
4917 0 : printr(disk->name, pad);
4918 0 : printf("%4" PRIu64 " | ", v);
4919 :
4920 0 : if (disk->progress_file && disk->progress_file->sub)
4921 0 : printf("%s", disk->progress_file->sub);
4922 : else
4923 0 : printf("-");
4924 :
4925 0 : printf("\n");
4926 : }
4927 :
4928 0 : for (l = 0; l < state->level; ++l) {
4929 0 : v = state->parity[l].cached_blocks;
4930 0 : printr(lev_config_name(l), pad);
4931 0 : printf("%4" PRIu64 " | ", v);
4932 0 : printc('o', v * bar / io->io_max);
4933 0 : printf("\n");
4934 : }
4935 :
4936 0 : printc(' ', pad);
4937 0 : printf(" |_");
4938 0 : printc('_', bar);
4939 0 : printf("\n");
4940 :
4941 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4942 0 : printf("%s", legend);
4943 0 : printf("\n");
4944 : }
4945 :
4946 1 : printf("\n");
4947 :
4948 1 : pre = 4;
4949 1 : if (state->thermal_temperature_limit != 0)
4950 0 : pre += strlen(THERMAL_PAD);
4951 1 : if (pad + pre + 30 < GRAPH_COLUMN)
4952 1 : bar = GRAPH_COLUMN - pad - pre;
4953 : else
4954 0 : bar = 30; /* at least a bar of 30 */
4955 :
4956 7 : for (i = state->disklist; i != 0; i = i->next) {
4957 6 : struct snapraid_disk* disk = i->data;
4958 :
4959 6 : v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4960 6 : printr(disk->name, pad);
4961 :
4962 6 : if (state->thermal_temperature_limit != 0) {
4963 0 : struct snapraid_thermal* thermal = state_thermal_find(state, disk->name);
4964 0 : if (thermal) {
4965 0 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
4966 0 : printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
4967 : else
4968 0 : printf(" %2u ", thermal->latest_temperature);
4969 : } else {
4970 0 : printf(THERMAL_PAD);
4971 : }
4972 : }
4973 :
4974 6 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4975 6 : printc('*', v * bar / tick_total);
4976 6 : printf("\n");
4977 :
4978 : /* clear the file in progress */
4979 6 : disk->progress_file = 0;
4980 : }
4981 :
4982 7 : for (l = 0; l < state->level; ++l) {
4983 6 : v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4984 6 : printr(lev_config_name(l), pad);
4985 :
4986 6 : if (state->thermal_temperature_limit != 0) {
4987 0 : struct snapraid_thermal* thermal = state_thermal_find(state, lev_config_name(l));
4988 0 : if (thermal) {
4989 0 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
4990 0 : printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
4991 : else
4992 0 : printf(" %2u ", thermal->latest_temperature);
4993 : } else {
4994 0 : printf(THERMAL_PAD);
4995 : }
4996 : }
4997 :
4998 6 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4999 6 : printc('*', v * bar / tick_total);
5000 6 : printf("\n");
5001 : }
5002 :
5003 1 : v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
5004 1 : printr("raid", pad);
5005 1 : if (state->thermal_temperature_limit != 0)
5006 0 : printf(THERMAL_PAD);
5007 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
5008 1 : printc('*', v * bar / tick_total);
5009 1 : printf("\n");
5010 :
5011 1 : v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
5012 1 : printr("hash", pad);
5013 1 : if (state->thermal_temperature_limit != 0)
5014 0 : printf(THERMAL_PAD);
5015 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
5016 1 : printc('*', v * bar / tick_total);
5017 1 : printf("\n");
5018 :
5019 1 : v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
5020 1 : printr("sched", pad);
5021 1 : if (state->thermal_temperature_limit != 0)
5022 0 : printf(THERMAL_PAD);
5023 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
5024 1 : printc('*', v * bar / tick_total);
5025 1 : printf("\n");
5026 :
5027 1 : v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
5028 1 : printr("misc", pad);
5029 1 : if (state->thermal_temperature_limit != 0)
5030 0 : printf(THERMAL_PAD);
5031 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
5032 1 : printc('*', v * bar / tick_total);
5033 1 : printf("\n");
5034 :
5035 1 : printc(' ', pad);
5036 1 : if (state->thermal_temperature_limit != 0)
5037 0 : printf(THERMAL_PAD);
5038 1 : printf(" |_");
5039 1 : printc('_', bar);
5040 1 : printf("\n");
5041 :
5042 1 : if (oldest == PROGRESS_MAX) {
5043 1 : const char* legend = "wait time (total, less is better)";
5044 1 : if (state->thermal_temperature_limit != 0)
5045 0 : printf(THERMAL_PAD);
5046 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
5047 1 : printf("%s", legend);
5048 1 : printf("\n");
5049 : } else {
5050 0 : const char* legend_d = "wait time (last %d secs, less is better)";
5051 0 : if (state->thermal_temperature_limit != 0)
5052 0 : printf(THERMAL_PAD);
5053 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
5054 0 : printf(legend_d, PROGRESS_MAX);
5055 0 : printf("\n");
5056 : }
5057 :
5058 1 : printf("\n");
5059 : }
5060 :
5061 1116834 : int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
5062 : {
5063 : time_t now;
5064 : int pred;
5065 :
5066 1116834 : now = time(0);
5067 :
5068 : /* thermal measure */
5069 1116834 : if (now > state->thermal_latest + THERMAL_PERIOD_SECONDS || state->opt.fake_device) {
5070 2619 : state->thermal_latest = now;
5071 2619 : state_thermal(state, now);
5072 : }
5073 :
5074 : /* previous position */
5075 1116834 : pred = state->progress_ptr + PROGRESS_MAX - 1;
5076 1116834 : if (pred >= PROGRESS_MAX)
5077 1116625 : pred -= PROGRESS_MAX;
5078 :
5079 : /* if the previous measure is different */
5080 1116834 : if (state->progress_tick == 0
5081 1116625 : || state->progress_time[pred] != now
5082 : ) {
5083 : time_t elapsed;
5084 405 : unsigned out_perc = 0;
5085 405 : unsigned out_size_speed = 0;
5086 405 : unsigned out_block_speed = 0;
5087 405 : unsigned out_cpu = 0;
5088 405 : unsigned out_eta = 0;
5089 405 : int out_temperature = 0;
5090 405 : int out_steady = 0;
5091 405 : int out_computed = 0;
5092 :
5093 : /* store the new measure */
5094 405 : state->progress_time[state->progress_ptr] = now;
5095 405 : state->progress_pos[state->progress_ptr] = countpos;
5096 405 : state->progress_size[state->progress_ptr] = countsize;
5097 405 : state_progress_latest(state);
5098 :
5099 405 : elapsed = now - state->progress_whole_start - state->progress_wasted;
5100 :
5101 : /* completion percentage */
5102 405 : if (countmax)
5103 405 : out_perc = muldiv(countpos, 100, countmax);
5104 :
5105 : /* if we have at least 5 measures */
5106 405 : if (state->progress_tick >= 5
5107 : /* or if we are running in test mode, with at least one measure */
5108 387 : || (state->opt.force_progress && state->progress_tick >= 1)
5109 : ) {
5110 : int oldest;
5111 : int past;
5112 : time_t delta_time;
5113 : block_off_t delta_pos;
5114 : data_off_t delta_size;
5115 : uint64_t tick_cpu;
5116 : uint64_t tick_total;
5117 : uint64_t delta_tick_cpu;
5118 : uint64_t delta_tick_total;
5119 : uint64_t oldest_tick_cpu;
5120 : uint64_t oldest_tick_total;
5121 : tommy_node* i;
5122 :
5123 : /* number of past measures */
5124 196 : past = state->progress_tick;
5125 :
5126 : /* drop the oldest ones, to promptly */
5127 : /* skip the startup phase */
5128 196 : past -= past / 5;
5129 :
5130 : /* check how much we can go in the past */
5131 196 : if (past >= PROGRESS_MAX - 1) {
5132 : /* the vector is filled, so we are already in position */
5133 : /* to get the possible oldest one */
5134 0 : oldest = state->progress_ptr + 1;
5135 : } else {
5136 : /* go backward the number of positions selected */
5137 196 : oldest = state->progress_ptr + PROGRESS_MAX - past;
5138 : }
5139 196 : if (oldest >= PROGRESS_MAX)
5140 196 : oldest -= PROGRESS_MAX;
5141 :
5142 196 : tick_cpu = state->progress_tick_misc[state->progress_ptr]
5143 196 : + state->progress_tick_sched[state->progress_ptr]
5144 196 : + state->progress_tick_raid[state->progress_ptr]
5145 196 : + state->progress_tick_hash[state->progress_ptr];
5146 196 : tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
5147 :
5148 196 : oldest_tick_cpu = state->progress_tick_misc[oldest]
5149 196 : + state->progress_tick_sched[oldest]
5150 196 : + state->progress_tick_raid[oldest]
5151 196 : + state->progress_tick_hash[oldest];
5152 196 : oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
5153 :
5154 196 : delta_time = now - state->progress_time[oldest];
5155 196 : delta_pos = countpos - state->progress_pos[oldest];
5156 196 : delta_size = countsize - state->progress_size[oldest];
5157 196 : delta_tick_cpu = tick_cpu - oldest_tick_cpu;
5158 196 : delta_tick_total = tick_total - oldest_tick_total;
5159 :
5160 : /* estimate the speed in MB/s */
5161 196 : if (delta_time != 0)
5162 196 : out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
5163 :
5164 : /* estimate the speed in block/s */
5165 196 : if (delta_time != 0)
5166 196 : out_block_speed = (unsigned)(delta_pos / delta_time);
5167 :
5168 : /* estimate the cpu usage percentage */
5169 196 : if (delta_tick_total != 0)
5170 54 : out_cpu = muldiv(delta_tick_cpu, 100, delta_tick_total);
5171 :
5172 : /* estimate the remaining time in minutes */
5173 196 : if (delta_pos != 0)
5174 196 : out_eta = muldiv(countmax - countpos, delta_time, delta_pos);
5175 :
5176 196 : if (state->opt.force_stats) {
5177 0 : os_clear();
5178 0 : state_progress_graph(state, io, state->progress_ptr, oldest);
5179 : }
5180 :
5181 196 : if (state->thermal_temperature_limit != 0) {
5182 : /* get the max temperature */
5183 5 : out_temperature = 0;
5184 155 : for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
5185 150 : struct snapraid_thermal* thermal = i->data;
5186 150 : if (out_temperature < thermal->latest_temperature) {
5187 5 : out_temperature = thermal->latest_temperature;
5188 5 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
5189 5 : out_steady = thermal->params.t_steady;
5190 : else
5191 0 : out_steady = 0;
5192 : }
5193 : }
5194 : } else {
5195 191 : out_temperature = 0;
5196 191 : out_steady = 0;
5197 : }
5198 :
5199 : /* we have the output value */
5200 196 : out_computed = 1;
5201 : }
5202 :
5203 405 : if (state->opt.gui) {
5204 : char str_eta[32];
5205 : char str_size_speed[32];
5206 : char str_cpu[32];
5207 : char str_temp[32];
5208 : char str_steady[32];
5209 4 : if (out_computed) {
5210 0 : snprintf(str_eta, sizeof(str_eta), "%u", out_eta);
5211 0 : snprintf(str_size_speed, sizeof(str_size_speed), "%u", out_size_speed);
5212 0 : snprintf(str_cpu, sizeof(str_cpu), "%u", out_cpu);
5213 : } else {
5214 4 : str_eta[0] = 0;
5215 4 : str_size_speed[0] = 0;
5216 4 : str_cpu[0] = 0;
5217 : }
5218 4 : if (out_temperature) {
5219 0 : snprintf(str_temp, sizeof(str_eta), "%u", out_temperature);
5220 : } else {
5221 4 : str_temp[0] = 0;
5222 : }
5223 4 : if (out_steady) {
5224 0 : snprintf(str_steady, sizeof(str_steady), "%u", out_steady);
5225 : } else {
5226 4 : str_steady[0] = 0;
5227 : }
5228 4 : log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%s:%s:%s:%" PRIu64 ":%s:%s\n", blockpos, countpos, countsize, out_perc, str_eta, str_size_speed, str_cpu, (uint64_t)elapsed, str_temp, str_steady);
5229 4 : log_flush();
5230 : } else {
5231 401 : msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
5232 401 : if (out_computed) {
5233 196 : msg_bar(", %u MB/s", out_size_speed);
5234 196 : msg_bar(", %u stripe/s", out_block_speed);
5235 196 : msg_bar(", CPU %u%%", out_cpu);
5236 196 : if (out_temperature != 0) {
5237 5 : if (out_steady != 0)
5238 5 : msg_bar(", Tmax %u (%u)", out_temperature, out_steady);
5239 : else
5240 0 : msg_bar(", Tmax %u", out_temperature);
5241 : }
5242 196 : unsigned out_eta_minutes = out_eta / 60; /* minutes */
5243 196 : msg_bar(", %u:%02u ETA", out_eta_minutes / 60, out_eta_minutes % 60);
5244 : }
5245 401 : msg_bar("\r");
5246 401 : msg_flush();
5247 : }
5248 :
5249 : /* next position to fill */
5250 405 : ++state->progress_ptr;
5251 405 : if (state->progress_ptr >= PROGRESS_MAX)
5252 0 : state->progress_ptr -= PROGRESS_MAX;
5253 :
5254 : /* one more measure */
5255 405 : ++state->progress_tick;
5256 : }
5257 :
5258 : /* stop if requested */
5259 1116834 : if (global_interrupt) {
5260 : /* LCOV_EXCL_START */
5261 : if (!state->opt.gui) {
5262 : log_fatal(EUSER, "\n");
5263 : log_fatal(EUSER, "Stopping for interruption at block %u\n", blockpos);
5264 : }
5265 : log_tag("sigint:%u: SIGINT received\n", blockpos);
5266 : log_flush();
5267 : return 1;
5268 : /* LCOV_EXCL_STOP */
5269 : }
5270 :
5271 1116834 : return 0;
5272 : }
5273 :
5274 114 : void state_usage_waste(struct snapraid_state* state)
5275 : {
5276 114 : uint64_t now = os_tick();
5277 :
5278 114 : state->tick_last = now;
5279 114 : }
5280 :
5281 1143594 : void state_usage_misc(struct snapraid_state* state)
5282 : {
5283 1143594 : uint64_t now = os_tick();
5284 1143594 : uint64_t delta = now - state->tick_last;
5285 :
5286 : /* increment the time spent in computations */
5287 1143594 : state->tick_misc += delta;
5288 :
5289 1143594 : state->tick_last = now;
5290 1143594 : }
5291 :
5292 161386 : void state_usage_sched(struct snapraid_state* state)
5293 : {
5294 161386 : uint64_t now = os_tick();
5295 161386 : uint64_t delta = now - state->tick_last;
5296 :
5297 : /* increment the time spent in computations */
5298 161386 : state->tick_sched += delta;
5299 :
5300 161386 : state->tick_last = now;
5301 161386 : }
5302 :
5303 150856 : void state_usage_raid(struct snapraid_state* state)
5304 : {
5305 150856 : uint64_t now = os_tick();
5306 150856 : uint64_t delta = now - state->tick_last;
5307 :
5308 : /* increment the time spent in computations */
5309 150856 : state->tick_raid += delta;
5310 :
5311 150856 : state->tick_last = now;
5312 150856 : }
5313 :
5314 812651 : void state_usage_hash(struct snapraid_state* state)
5315 : {
5316 812651 : uint64_t now = os_tick();
5317 812651 : uint64_t delta = now - state->tick_last;
5318 :
5319 : /* increment the time spent in computations */
5320 812651 : state->tick_hash += delta;
5321 :
5322 812651 : state->tick_last = now;
5323 812651 : }
5324 :
5325 976659 : void state_usage_file(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
5326 : {
5327 : (void)state;
5328 :
5329 976659 : disk->progress_file = file;
5330 976659 : }
5331 :
5332 981346 : void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
5333 : {
5334 981346 : uint64_t now = os_tick();
5335 981346 : uint64_t delta = now - state->tick_last;
5336 : unsigned i;
5337 :
5338 : /* increment the time spent in the data disks */
5339 1051467 : for (i = 0; i < waiting_mac; ++i) {
5340 70121 : struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
5341 :
5342 70121 : if (!disk)
5343 0 : continue;
5344 :
5345 70121 : disk->tick += delta;
5346 : }
5347 981346 : state->tick_io += delta;
5348 :
5349 981346 : state->tick_last = now;
5350 981346 : }
5351 :
5352 796248 : void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
5353 : {
5354 796248 : uint64_t now = os_tick();
5355 796248 : uint64_t delta = now - state->tick_last;
5356 : unsigned i;
5357 :
5358 : /* increment the time spent in the parity disk */
5359 815186 : for (i = 0; i < waiting_mac; ++i)
5360 18938 : state->parity[waiting_map[i]].tick += delta;
5361 796248 : state->tick_io += delta;
5362 :
5363 796248 : state->tick_last = now;
5364 796248 : }
5365 :
5366 106 : void state_usage_print(struct snapraid_state* state)
5367 : {
5368 : /* set the latest data */
5369 106 : state_progress_latest(state);
5370 :
5371 106 : if (msg_level < MSG_PROGRESS)
5372 105 : return;
5373 :
5374 : /* print a graph for it */
5375 1 : state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
5376 : }
5377 :
5378 537 : void state_fscheck(struct snapraid_state* state, const char* ope)
5379 : {
5380 : tommy_node* i;
5381 :
5382 : /* check the file-system on all disks */
5383 3753 : for (i = state->disklist; i != 0; i = i->next) {
5384 3216 : struct snapraid_disk* disk = i->data;
5385 :
5386 3216 : if (fs_check(disk) != 0) {
5387 : /* LCOV_EXCL_START */
5388 : log_fatal(EINTERNAL, "Internal inconsistency: File-system check for disk '%s' %s\n", disk->name, ope);
5389 : os_abort();
5390 : /* LCOV_EXCL_STOP */
5391 : }
5392 : }
5393 537 : }
5394 :
5395 1 : void generate_configuration(const char* path)
5396 : {
5397 : struct snapraid_state state;
5398 : struct snapraid_content* content;
5399 : char esc_buffer[ESC_MAX];
5400 : unsigned l, s;
5401 : tommy_node* j;
5402 :
5403 1 : state_init(&state);
5404 :
5405 : /* mark that we are without a configuration file */
5406 1 : state.no_conf = 1;
5407 :
5408 : /* create the dummy content entry */
5409 1 : content = content_alloc(path, -1);
5410 :
5411 : /* adds the content entry */
5412 1 : tommy_list_insert_tail(&state.contentlist, &content->node, content);
5413 :
5414 : /* read the content file */
5415 1 : state_read(&state);
5416 :
5417 : /* output a dummy configuration file */
5418 1 : printf("# Configuration file generated from %s\n", path);
5419 1 : printf("\n");
5420 1 : printf("# Use this blocksize\n");
5421 1 : printf("blocksize %u\n", state.block_size / KIBI);
5422 1 : printf("\n");
5423 1 : printf("# Use this hashsize\n");
5424 1 : printf("hashsize %u\n", BLOCK_HASH_SIZE);
5425 1 : printf("\n");
5426 7 : for (l = 0; l < state.level; ++l) {
5427 6 : printf("# Set the correct path for the %s files\n", lev_name(l));
5428 6 : printf("# You had %u of them:\n", state.parity[l].split_mac);
5429 30 : for (s = 0; s < state.parity[l].split_mac; ++s) {
5430 24 : printf("# %u:\n", s);
5431 24 : printf("# PATH:");
5432 24 : if (state.parity[l].split_map[s].path[0])
5433 24 : printf("%s", state.parity[l].split_map[s].path);
5434 : else
5435 0 : printf("?");
5436 24 : printf("\n");
5437 24 : printf("# SIZE:");
5438 24 : if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
5439 24 : printf("%" PRIu64, state.parity[l].split_map[s].size);
5440 : else
5441 0 : printf("?");
5442 24 : printf("\n");
5443 24 : printf("# UUID:");
5444 24 : if (state.parity[l].split_map[s].uuid[0])
5445 24 : printf("%s", state.parity[l].split_map[s].uuid);
5446 : else
5447 0 : printf("?");
5448 24 : printf("\n");
5449 24 : printf("#\n");
5450 : }
5451 6 : printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
5452 6 : printf("\n");
5453 : }
5454 1 : printf("# Add any other content file\n");
5455 1 : printf("content %s\n", path);
5456 1 : printf("\n");
5457 7 : for (j = state.maplist; j; j = j->next) {
5458 6 : struct snapraid_map* map = j->data;
5459 : struct snapraid_disk* disk;
5460 6 : printf("# Set the correct dir for disk '%s'\n", map->name);
5461 6 : if (map->uuid[0])
5462 6 : printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
5463 6 : disk = find_disk_by_name(&state, map->name);
5464 6 : if (disk && disk->filelist) {
5465 6 : struct snapraid_file* file = disk->filelist->data;
5466 6 : if (file) {
5467 6 : printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
5468 : }
5469 : }
5470 6 : printf("data %s ENTER_HERE_THE_DIR\n", map->name);
5471 6 : printf("\n");
5472 : }
5473 :
5474 1 : state_done(&state);
5475 1 : }
5476 :
5477 :
5478 : /**
5479 : * Flush and sync the parity files.
5480 : */
5481 93 : int state_flush(struct snapraid_state* state, struct snapraid_io* io, struct snapraid_parity_handle* parity_handle, block_off_t blockcur)
5482 : {
5483 : unsigned l;
5484 :
5485 93 : if (io)
5486 93 : io_flush(io);
5487 :
5488 : /* flush all parity handles to ensure data is written to disk */
5489 606 : for (l = 0; l < state->level; ++l) {
5490 513 : int ret = parity_sync(&parity_handle[l]);
5491 513 : if (ret == -1) {
5492 : /* LCOV_EXCL_START */
5493 : log_tag("parity_error_io:%u:%s: Sync error. %s.\n", blockcur, lev_config_name(l), strerror(errno));
5494 : log_fatal(errno, "DANGER! Unexpected input/output error in disk %s. It isn't possible to continue.\n", lev_config_name(l));
5495 : return -1;
5496 : /* LCOV_EXCL_STOP */
5497 : }
5498 : }
5499 :
5500 93 : return 0;
5501 : }
5502 :
5503 66 : void state_load_ignore_file(tommy_list* filter_list, const char* path, const char* sub)
5504 : {
5505 : STREAM* f;
5506 : int line;
5507 :
5508 66 : f = sopen_read(path, 0);
5509 66 : if (!f) {
5510 : /* LCOV_EXCL_START */
5511 : log_error(errno, "Error opening the ignore file '%s'. %s.\n", path, strerror(errno));
5512 : return;
5513 : /* LCOV_EXCL_STOP */
5514 : }
5515 :
5516 66 : line = 1;
5517 66 : while (1) {
5518 : char buffer[PATH_MAX];
5519 : int ret;
5520 : int c;
5521 :
5522 : /* skip initial spaces */
5523 132 : sgetspace(f);
5524 :
5525 : /* get the whole line */
5526 132 : ret = sgetline(f, buffer, sizeof(buffer));
5527 132 : if (ret < 0) {
5528 : /* LCOV_EXCL_START */
5529 : log_error(EUSER, "Too long line in '%s' at line %u\n", path, line);
5530 : break;
5531 : /* LCOV_EXCL_STOP */
5532 : }
5533 :
5534 132 : if (buffer[0] == 0) {
5535 : /* allow empty lines */
5536 66 : } else if (buffer[0] == '#') {
5537 : /* ignore comment lines */
5538 : } else {
5539 : struct snapraid_filter* filter;
5540 :
5541 66 : filter = filter_alloc_file(-1, sub, buffer);
5542 66 : if (!filter) {
5543 : /* LCOV_EXCL_START */
5544 : log_error(EUSER, "Invalid ignore specification '%s' in '%s' at line %u\n", buffer, path, line);
5545 : break;
5546 : /* LCOV_EXCL_STOP */
5547 : }
5548 :
5549 66 : tommy_list_insert_tail(filter_list, &filter->node, filter);
5550 : }
5551 :
5552 : /* next line */
5553 132 : c = sgeteol(f);
5554 132 : if (c == EOF) {
5555 66 : break;
5556 : }
5557 66 : if (c != '\n') {
5558 : /* LCOV_EXCL_START */
5559 : log_error(EUSER, "Extra data in '%s' at line %u\n", path, line);
5560 : break;
5561 : /* LCOV_EXCL_STOP */
5562 : }
5563 66 : ++line;
5564 : }
5565 :
5566 66 : if (serror(f)) {
5567 : /* LCOV_EXCL_START */
5568 : log_error(errno, "Error reading the ignore file '%s' at line %u\n", path, line);
5569 : /* LCOV_EXCL_STOP */
5570 : }
5571 :
5572 66 : sclose(f);
5573 : }
5574 :
|
