conv.c (15267B)
1 #include "stdinc.h" 2 #include "dat.h" 3 #include "fns.h" 4 5 /* 6 * disk structure conversion routines 7 */ 8 #define U8GET(p) ((p)[0]) 9 #define U16GET(p) (((p)[0]<<8)|(p)[1]) 10 #define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3])) 11 #define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4)) 12 13 #define U8PUT(p,v) (p)[0]=(v)&0xFF 14 #define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF 15 #define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF 16 #define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32) 17 18 int debugarena = -1; /* hack to improve error reporting */ 19 20 static struct { 21 u32int m; 22 char *s; 23 } magics[] = { 24 ArenaPartMagic, "ArenaPartMagic", 25 ArenaHeadMagic, "ArenaHeadMagic", 26 ArenaMagic, "ArenaMagic", 27 ISectMagic, "ISectMagic", 28 BloomMagic, "BloomMagic", 29 }; 30 31 static char* 32 fmtmagic(char *s, u32int m) 33 { 34 int i; 35 36 for(i=0; i<nelem(magics); i++) 37 if(magics[i].m == m) 38 return magics[i].s; 39 sprint(s, "%#08ux", m); 40 return s; 41 } 42 43 u32int 44 unpackmagic(u8int *buf) 45 { 46 return U32GET(buf); 47 } 48 49 void 50 packmagic(u32int magic, u8int *buf) 51 { 52 U32PUT(buf, magic); 53 } 54 55 int 56 unpackarenapart(ArenaPart *ap, u8int *buf) 57 { 58 u8int *p; 59 u32int m; 60 char fbuf[20]; 61 62 p = buf; 63 64 m = U32GET(p); 65 if(m != ArenaPartMagic){ 66 seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic); 67 return -1; 68 } 69 p += U32Size; 70 ap->version = U32GET(p); 71 p += U32Size; 72 ap->blocksize = U32GET(p); 73 p += U32Size; 74 ap->arenabase = U32GET(p); 75 p += U32Size; 76 77 if(buf + ArenaPartSize != p) 78 sysfatal("unpackarenapart unpacked wrong amount"); 79 80 return 0; 81 } 82 83 int 84 packarenapart(ArenaPart *ap, u8int *buf) 85 { 86 u8int *p; 87 88 p = buf; 89 90 U32PUT(p, ArenaPartMagic); 91 p += U32Size; 92 U32PUT(p, ap->version); 93 p += U32Size; 94 U32PUT(p, ap->blocksize); 95 p += U32Size; 96 U32PUT(p, ap->arenabase); 97 p += U32Size; 98 99 if(buf + ArenaPartSize != p) 100 sysfatal("packarenapart packed wrong amount"); 101 102 return 0; 103 } 104 105 int 106 unpackarena(Arena *arena, u8int *buf) 107 { 108 int sz; 109 u8int *p; 110 u32int m; 111 char fbuf[20]; 112 113 p = buf; 114 115 m = U32GET(p); 116 if(m != ArenaMagic){ 117 seterr(ECorrupt, "arena %d has wrong magic number: %s " 118 "expected ArenaMagic (%#lux)", debugarena, 119 fmtmagic(fbuf, m), ArenaMagic); 120 return -1; 121 } 122 p += U32Size; 123 arena->version = U32GET(p); 124 p += U32Size; 125 namecp(arena->name, (char*)p); 126 p += ANameSize; 127 arena->diskstats.clumps = U32GET(p); 128 p += U32Size; 129 arena->diskstats.cclumps = U32GET(p); 130 p += U32Size; 131 arena->ctime = U32GET(p); 132 p += U32Size; 133 arena->wtime = U32GET(p); 134 p += U32Size; 135 if(arena->version == ArenaVersion5){ 136 arena->clumpmagic = U32GET(p); 137 p += U32Size; 138 } 139 arena->diskstats.used = U64GET(p); 140 p += U64Size; 141 arena->diskstats.uncsize = U64GET(p); 142 p += U64Size; 143 arena->diskstats.sealed = U8GET(p); 144 p += U8Size; 145 switch(arena->version){ 146 case ArenaVersion4: 147 sz = ArenaSize4; 148 arena->clumpmagic = _ClumpMagic; 149 break; 150 case ArenaVersion5: 151 sz = ArenaSize5; 152 break; 153 default: 154 seterr(ECorrupt, "arena has bad version number %d", arena->version); 155 return -1; 156 } 157 /* 158 * Additional fields for the memstats version of the stats. 159 * Diskstats reflects what is committed to the index. 160 * Memstats reflects what is in the arena. Originally intended 161 * this to be a version 5 extension, but might as well use for 162 * all the existing version 4 arenas too. 163 * 164 * To maintain backwards compatibility with existing venti 165 * installations using the older format, we define that if 166 * memstats == diskstats, then the extension fields are not 167 * included (see packarena below). That is, only partially 168 * indexed arenas have these fields. Fully indexed arenas 169 * (in particular, sealed arenas) do not. 170 */ 171 if(U8GET(p) == 1){ 172 sz += ArenaSize5a-ArenaSize5; 173 p += U8Size; 174 arena->memstats.clumps = U32GET(p); 175 p += U32Size; 176 arena->memstats.cclumps = U32GET(p); 177 p += U32Size; 178 arena->memstats.used = U64GET(p); 179 p += U64Size; 180 arena->memstats.uncsize = U64GET(p); 181 p += U64Size; 182 arena->memstats.sealed = U8GET(p); 183 p += U8Size; 184 185 /* 186 * 2008/4/2 187 * Packarena (below) used to have a bug in which it would 188 * not zero out any existing extension fields when writing 189 * the arena metadata. This would manifest itself as arenas 190 * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0 191 * after a server restart. Because arena->memstats.sealed wouldn't 192 * be set, the server might try to fit another block into the arena 193 * (and succeed), violating the append-only structure of the log 194 * and invalidating any already-computed seal on the arena. 195 * 196 * It might end up that other fields in arena->memstats end up 197 * behind arena->diskstats too, but that would be considerably 198 * more rare, and the bug is fixed now. The case we need to 199 * handle is just the sealed mismatch. 200 * 201 * If we encounter such a bogus arena, fix the sealed field. 202 */ 203 if(arena->diskstats.sealed) 204 arena->memstats.sealed = 1; 205 }else 206 arena->memstats = arena->diskstats; 207 if(buf + sz != p) 208 sysfatal("unpackarena unpacked wrong amount"); 209 210 return 0; 211 } 212 213 int 214 packarena(Arena *arena, u8int *buf) 215 { 216 return _packarena(arena, buf, 0); 217 } 218 219 int 220 _packarena(Arena *arena, u8int *buf, int forceext) 221 { 222 int sz; 223 u8int *p; 224 u32int t32; 225 226 switch(arena->version){ 227 case ArenaVersion4: 228 sz = ArenaSize4; 229 if(arena->clumpmagic != _ClumpMagic) 230 fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n", 231 (ulong)arena->clumpmagic, (ulong)_ClumpMagic); 232 break; 233 case ArenaVersion5: 234 sz = ArenaSize5; 235 break; 236 default: 237 sysfatal("packarena unknown version %d", arena->version); 238 return -1; 239 } 240 241 p = buf; 242 243 U32PUT(p, ArenaMagic); 244 p += U32Size; 245 U32PUT(p, arena->version); 246 p += U32Size; 247 namecp((char*)p, arena->name); 248 p += ANameSize; 249 U32PUT(p, arena->diskstats.clumps); 250 p += U32Size; 251 U32PUT(p, arena->diskstats.cclumps); 252 p += U32Size; 253 U32PUT(p, arena->ctime); 254 p += U32Size; 255 U32PUT(p, arena->wtime); 256 p += U32Size; 257 if(arena->version == ArenaVersion5){ 258 U32PUT(p, arena->clumpmagic); 259 p += U32Size; 260 } 261 U64PUT(p, arena->diskstats.used, t32); 262 p += U64Size; 263 U64PUT(p, arena->diskstats.uncsize, t32); 264 p += U64Size; 265 U8PUT(p, arena->diskstats.sealed); 266 p += U8Size; 267 268 /* 269 * Extension fields; see above. 270 */ 271 if(forceext 272 || arena->memstats.clumps != arena->diskstats.clumps 273 || arena->memstats.cclumps != arena->diskstats.cclumps 274 || arena->memstats.used != arena->diskstats.used 275 || arena->memstats.uncsize != arena->diskstats.uncsize 276 || arena->memstats.sealed != arena->diskstats.sealed){ 277 sz += ArenaSize5a - ArenaSize5; 278 U8PUT(p, 1); 279 p += U8Size; 280 U32PUT(p, arena->memstats.clumps); 281 p += U32Size; 282 U32PUT(p, arena->memstats.cclumps); 283 p += U32Size; 284 U64PUT(p, arena->memstats.used, t32); 285 p += U64Size; 286 U64PUT(p, arena->memstats.uncsize, t32); 287 p += U64Size; 288 U8PUT(p, arena->memstats.sealed); 289 p += U8Size; 290 }else{ 291 /* Clear any extension fields already on disk. */ 292 memset(p, 0, ArenaSize5a - ArenaSize5); 293 p += ArenaSize5a - ArenaSize5; 294 sz += ArenaSize5a - ArenaSize5; 295 } 296 297 if(buf + sz != p) 298 sysfatal("packarena packed wrong amount"); 299 300 return 0; 301 } 302 303 int 304 unpackarenahead(ArenaHead *head, u8int *buf) 305 { 306 u8int *p; 307 u32int m; 308 int sz; 309 char fbuf[20]; 310 311 p = buf; 312 313 m = U32GET(p); 314 if(m != ArenaHeadMagic){ 315 seterr(ECorrupt, "arena %d head has wrong magic number: %s " 316 "expected ArenaHeadMagic (%#lux)", debugarena, 317 fmtmagic(fbuf, m), ArenaHeadMagic); 318 return -1; 319 } 320 321 p += U32Size; 322 head->version = U32GET(p); 323 p += U32Size; 324 namecp(head->name, (char*)p); 325 p += ANameSize; 326 head->blocksize = U32GET(p); 327 p += U32Size; 328 head->size = U64GET(p); 329 p += U64Size; 330 if(head->version == ArenaVersion5){ 331 head->clumpmagic = U32GET(p); 332 p += U32Size; 333 } 334 335 switch(head->version){ 336 case ArenaVersion4: 337 sz = ArenaHeadSize4; 338 head->clumpmagic = _ClumpMagic; 339 break; 340 case ArenaVersion5: 341 sz = ArenaHeadSize5; 342 break; 343 default: 344 seterr(ECorrupt, "arena head has unexpected version %d", head->version); 345 return -1; 346 } 347 348 if(buf + sz != p) 349 sysfatal("unpackarenahead unpacked wrong amount"); 350 351 return 0; 352 } 353 354 int 355 packarenahead(ArenaHead *head, u8int *buf) 356 { 357 u8int *p; 358 int sz; 359 u32int t32; 360 361 switch(head->version){ 362 case ArenaVersion4: 363 sz = ArenaHeadSize4; 364 if(head->clumpmagic != _ClumpMagic) 365 fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n", 366 (ulong)head->clumpmagic, (ulong)_ClumpMagic); 367 break; 368 case ArenaVersion5: 369 sz = ArenaHeadSize5; 370 break; 371 default: 372 sysfatal("packarenahead unknown version %d", head->version); 373 return -1; 374 } 375 376 p = buf; 377 378 U32PUT(p, ArenaHeadMagic); 379 p += U32Size; 380 U32PUT(p, head->version); 381 p += U32Size; 382 namecp((char*)p, head->name); 383 p += ANameSize; 384 U32PUT(p, head->blocksize); 385 p += U32Size; 386 U64PUT(p, head->size, t32); 387 p += U64Size; 388 if(head->version == ArenaVersion5){ 389 U32PUT(p, head->clumpmagic); 390 p += U32Size; 391 } 392 if(buf + sz != p) 393 sysfatal("packarenahead packed wrong amount"); 394 395 return 0; 396 } 397 398 static int 399 checkclump(Clump *w) 400 { 401 if(w->encoding == ClumpENone){ 402 if(w->info.size != w->info.uncsize){ 403 seterr(ECorrupt, "uncompressed wad size mismatch"); 404 return -1; 405 } 406 }else if(w->encoding == ClumpECompress){ 407 if(w->info.size >= w->info.uncsize){ 408 seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize); 409 return -1; 410 } 411 }else{ 412 seterr(ECorrupt, "clump has illegal encoding"); 413 return -1; 414 } 415 416 return 0; 417 } 418 419 int 420 unpackclump(Clump *c, u8int *buf, u32int cmagic) 421 { 422 u8int *p; 423 u32int magic; 424 425 p = buf; 426 magic = U32GET(p); 427 if(magic != cmagic){ 428 seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic); 429 return -1; 430 } 431 p += U32Size; 432 433 c->info.type = vtfromdisktype(U8GET(p)); 434 p += U8Size; 435 c->info.size = U16GET(p); 436 p += U16Size; 437 c->info.uncsize = U16GET(p); 438 p += U16Size; 439 scorecp(c->info.score, p); 440 p += VtScoreSize; 441 442 c->encoding = U8GET(p); 443 p += U8Size; 444 c->creator = U32GET(p); 445 p += U32Size; 446 c->time = U32GET(p); 447 p += U32Size; 448 449 if(buf + ClumpSize != p) 450 sysfatal("unpackclump unpacked wrong amount"); 451 452 return checkclump(c); 453 } 454 455 int 456 packclump(Clump *c, u8int *buf, u32int magic) 457 { 458 u8int *p; 459 460 p = buf; 461 U32PUT(p, magic); 462 p += U32Size; 463 464 U8PUT(p, vttodisktype(c->info.type)); 465 p += U8Size; 466 U16PUT(p, c->info.size); 467 p += U16Size; 468 U16PUT(p, c->info.uncsize); 469 p += U16Size; 470 scorecp(p, c->info.score); 471 p += VtScoreSize; 472 473 U8PUT(p, c->encoding); 474 p += U8Size; 475 U32PUT(p, c->creator); 476 p += U32Size; 477 U32PUT(p, c->time); 478 p += U32Size; 479 480 if(buf + ClumpSize != p) 481 sysfatal("packclump packed wrong amount"); 482 483 return checkclump(c); 484 } 485 486 void 487 unpackclumpinfo(ClumpInfo *ci, u8int *buf) 488 { 489 u8int *p; 490 491 p = buf; 492 ci->type = vtfromdisktype(U8GET(p)); 493 p += U8Size; 494 ci->size = U16GET(p); 495 p += U16Size; 496 ci->uncsize = U16GET(p); 497 p += U16Size; 498 scorecp(ci->score, p); 499 p += VtScoreSize; 500 501 if(buf + ClumpInfoSize != p) 502 sysfatal("unpackclumpinfo unpacked wrong amount"); 503 } 504 505 void 506 packclumpinfo(ClumpInfo *ci, u8int *buf) 507 { 508 u8int *p; 509 510 p = buf; 511 U8PUT(p, vttodisktype(ci->type)); 512 p += U8Size; 513 U16PUT(p, ci->size); 514 p += U16Size; 515 U16PUT(p, ci->uncsize); 516 p += U16Size; 517 scorecp(p, ci->score); 518 p += VtScoreSize; 519 520 if(buf + ClumpInfoSize != p) 521 sysfatal("packclumpinfo packed wrong amount"); 522 } 523 524 int 525 unpackisect(ISect *is, u8int *buf) 526 { 527 u8int *p; 528 u32int m; 529 char fbuf[20]; 530 531 p = buf; 532 533 534 m = U32GET(p); 535 if(m != ISectMagic){ 536 seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)", 537 fmtmagic(fbuf, m), ISectMagic); 538 return -1; 539 } 540 p += U32Size; 541 is->version = U32GET(p); 542 p += U32Size; 543 namecp(is->name, (char*)p); 544 p += ANameSize; 545 namecp(is->index, (char*)p); 546 p += ANameSize; 547 is->blocksize = U32GET(p); 548 p += U32Size; 549 is->blockbase = U32GET(p); 550 p += U32Size; 551 is->blocks = U32GET(p); 552 p += U32Size; 553 is->start = U32GET(p); 554 p += U32Size; 555 is->stop = U32GET(p); 556 p += U32Size; 557 if(buf + ISectSize1 != p) 558 sysfatal("unpackisect unpacked wrong amount"); 559 is->bucketmagic = 0; 560 if(is->version == ISectVersion2){ 561 is->bucketmagic = U32GET(p); 562 p += U32Size; 563 if(buf + ISectSize2 != p) 564 sysfatal("unpackisect unpacked wrong amount"); 565 } 566 567 return 0; 568 } 569 570 int 571 packisect(ISect *is, u8int *buf) 572 { 573 u8int *p; 574 575 p = buf; 576 577 U32PUT(p, ISectMagic); 578 p += U32Size; 579 U32PUT(p, is->version); 580 p += U32Size; 581 namecp((char*)p, is->name); 582 p += ANameSize; 583 namecp((char*)p, is->index); 584 p += ANameSize; 585 U32PUT(p, is->blocksize); 586 p += U32Size; 587 U32PUT(p, is->blockbase); 588 p += U32Size; 589 U32PUT(p, is->blocks); 590 p += U32Size; 591 U32PUT(p, is->start); 592 p += U32Size; 593 U32PUT(p, is->stop); 594 p += U32Size; 595 if(buf + ISectSize1 != p) 596 sysfatal("packisect packed wrong amount"); 597 if(is->version == ISectVersion2){ 598 U32PUT(p, is->bucketmagic); 599 p += U32Size; 600 if(buf + ISectSize2 != p) 601 sysfatal("packisect packed wrong amount"); 602 } 603 604 return 0; 605 } 606 607 void 608 unpackientry(IEntry *ie, u8int *buf) 609 { 610 u8int *p; 611 612 p = buf; 613 614 scorecp(ie->score, p); 615 p += VtScoreSize; 616 /* ie->wtime = U32GET(p); */ 617 p += U32Size; 618 /* ie->train = U16GET(p); */ 619 p += U16Size; 620 if(p - buf != IEntryAddrOff) 621 sysfatal("unpackentry bad IEntryAddrOff amount"); 622 ie->ia.addr = U64GET(p); 623 if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr); 624 p += U64Size; 625 ie->ia.size = U16GET(p); 626 p += U16Size; 627 if(p - buf != IEntryTypeOff) 628 sysfatal("unpackientry bad IEntryTypeOff amount"); 629 ie->ia.type = vtfromdisktype(U8GET(p)); 630 p += U8Size; 631 ie->ia.blocks = U8GET(p); 632 p += U8Size; 633 634 if(p - buf != IEntrySize) 635 sysfatal("unpackientry unpacked wrong amount"); 636 } 637 638 void 639 packientry(IEntry *ie, u8int *buf) 640 { 641 u32int t32; 642 u8int *p; 643 644 p = buf; 645 646 scorecp(p, ie->score); 647 p += VtScoreSize; 648 U32PUT(p, 0); /* wtime */ 649 p += U32Size; 650 U16PUT(p, 0); /* train */ 651 p += U16Size; 652 U64PUT(p, ie->ia.addr, t32); 653 p += U64Size; 654 U16PUT(p, ie->ia.size); 655 p += U16Size; 656 U8PUT(p, vttodisktype(ie->ia.type)); 657 p += U8Size; 658 U8PUT(p, ie->ia.blocks); 659 p += U8Size; 660 661 if(p - buf != IEntrySize) 662 sysfatal("packientry packed wrong amount"); 663 } 664 665 void 666 unpackibucket(IBucket *b, u8int *buf, u32int magic) 667 { 668 b->n = U16GET(buf); 669 b->data = buf + IBucketSize; 670 if(magic && magic != U32GET(buf+U16Size)) 671 b->n = 0; 672 } 673 674 void 675 packibucket(IBucket *b, u8int *buf, u32int magic) 676 { 677 U16PUT(buf, b->n); 678 U32PUT(buf+U16Size, magic); 679 } 680 681 void 682 packbloomhead(Bloom *b, u8int *buf) 683 { 684 u8int *p; 685 686 p = buf; 687 U32PUT(p, BloomMagic); 688 U32PUT(p+4, BloomVersion); 689 U32PUT(p+8, b->nhash); 690 U32PUT(p+12, b->size); 691 } 692 693 int 694 unpackbloomhead(Bloom *b, u8int *buf) 695 { 696 u8int *p; 697 u32int m; 698 char fbuf[20]; 699 700 p = buf; 701 702 m = U32GET(p); 703 if(m != BloomMagic){ 704 seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic); 705 return -1; 706 } 707 p += U32Size; 708 709 m = U32GET(p); 710 if(m != BloomVersion){ 711 seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion); 712 return -1; 713 } 714 p += U32Size; 715 716 b->nhash = U32GET(p); 717 p += U32Size; 718 719 b->size = U32GET(p); 720 p += U32Size; 721 if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){ 722 seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size); 723 return -1; 724 } 725 726 if(buf + BloomHeadSize != p) 727 sysfatal("unpackarena unpacked wrong amount"); 728 729 return 0; 730 }