Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/osmpbfparse.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1356 - (show annotations) (download) (as text)
Fri May 31 17:23:18 2013 UTC (11 years, 9 months ago) by amb
File MIME type: text/x-csrc
File size: 31220 byte(s)
Move some functions about so that osmparser.c can be replaced for other types of
parsing.

1 /***************************************
2 A simple osm-specific PBF parser where the structure is hard-coded.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2012, 2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <unistd.h>
25 #include <stdlib.h>
26 #include <stdint.h>
27 #include <string.h>
28
29 #if defined(USE_GZIP) && USE_GZIP
30 #include <zlib.h>
31 #endif
32
33 #include "osmparser.h"
34 #include "tagging.h"
35 #include "logging.h"
36
37
38 /* Inside a BlobHeader message */
39
40 #define PBF_VAL_BLOBHEADER_TYPE 1
41 #define PBF_VAL_BLOBHEADER_SIZE 3
42
43 /* Inside a Blob message */
44
45 #define PBF_VAL_BLOB_RAW_DATA 1
46 #define PBF_VAL_BLOB_RAW_SIZE 2
47 #define PBF_VAL_BLOB_ZLIB_DATA 3
48
49 /* Inside a HeaderBlock message */
50
51 #define PBF_VAL_REQUIRED_FEATURES 4
52 #define PBF_VAL_OPTIONAL_FEATURES 5
53
54 /* Inside a PrimitiveBlock message */
55
56 #define PBF_VAL_STRING_TABLE 1
57 #define PBF_VAL_PRIMITIVE_GROUP 2
58 #define PBF_VAL_GRANULARITY 17
59 #define PBF_VAL_LAT_OFFSET 19
60 #define PBF_VAL_LON_OFFSET 20
61
62 /* Inside a PrimitiveGroup message */
63
64 #define PBF_VAL_NODES 1
65 #define PBF_VAL_DENSE_NODES 2
66 #define PBF_VAL_WAYS 3
67 #define PBF_VAL_RELATIONS 4
68
69 /* Inside a StringTable message */
70
71 #define PBF_VAL_STRING 1
72
73 /* Inside a Node message */
74
75 #define PBF_VAL_NODE_ID 1
76 #define PBF_VAL_NODE_KEYS 2
77 #define PBF_VAL_NODE_VALS 3
78 #define PBF_VAL_NODE_LAT 8
79 #define PBF_VAL_NODE_LON 9
80
81 /* Inside a DenseNode message */
82
83 #define PBF_VAL_DENSE_NODE_ID 1
84 #define PBF_VAL_DENSE_NODE_LAT 8
85 #define PBF_VAL_DENSE_NODE_LON 9
86 #define PBF_VAL_DENSE_NODE_KEYS_VALS 10
87
88 /* Inside a Way message */
89
90 #define PBF_VAL_WAY_ID 1
91 #define PBF_VAL_WAY_KEYS 2
92 #define PBF_VAL_WAY_VALS 3
93 #define PBF_VAL_WAY_REFS 8
94
95 /* Inside a Relation message */
96
97 #define PBF_VAL_RELATION_ID 1
98 #define PBF_VAL_RELATION_KEYS 2
99 #define PBF_VAL_RELATION_VALS 3
100 #define PBF_VAL_RELATION_ROLES 8
101 #define PBF_VAL_RELATION_MEMIDS 9
102 #define PBF_VAL_RELATION_TYPES 10
103
104 /* Errors */
105
106 #define PBF_EOF 0
107
108 #define PBF_ERROR_UNEXP_EOF 100
109 #define PBF_ERROR_BLOB_HEADER_LEN 101
110 #define PBF_ERROR_BLOB_LEN 102
111 #define PBF_ERROR_NOT_OSM 103
112 #define PBF_ERROR_BLOB_BOTH 104
113 #define PBF_ERROR_BLOB_NEITHER 105
114 #define PBF_ERROR_NO_GZIP 106
115 #define PBF_ERROR_GZIP_INIT 107
116 #define PBF_ERROR_GZIP_INFLATE 108
117 #define PBF_ERROR_GZIP_WRONG_LEN 109
118 #define PBF_ERROR_GZIP_END 110
119 #define PBF_ERROR_UNSUPPORTED 111
120 #define PBF_ERROR_TOO_MANY_GROUPS 112
121
122
123 /* Parsing variables and functions */
124
125 static uint64_t byteno=0;
126 static uint64_t nnodes=0,nways=0,nrelations=0;
127
128 static uint32_t buffer_allocated,zbuffer_allocated;
129 static unsigned char *buffer=NULL,*zbuffer=NULL;
130 static unsigned char *buffer_ptr,*buffer_end;
131
132 static int string_table_length=0,string_table_allocated=0;
133 static unsigned char **string_table=NULL;
134 static uint32_t *string_table_string_lengths=NULL;
135
136 static int32_t granularity=100;
137 static int64_t lat_offset=0,lon_offset=0;
138
139 #define LENGTH_32M (32*1024*1024)
140
141
142 /*++++++++++++++++++++++++++++++++++++++
143 Refill the data buffer and set the pointers.
144
145 int buffer_refill Return 0 if everything is OK or 1 for EOF.
146
147 int fd The file descriptor to read from.
148
149 uint32_t bytes The number of bytes to read.
150 ++++++++++++++++++++++++++++++++++++++*/
151
152 static inline int buffer_refill(int fd,uint32_t bytes)
153 {
154 ssize_t n;
155
156 if(bytes>buffer_allocated)
157 buffer=(unsigned char *)realloc(buffer,buffer_allocated=bytes);
158
159 byteno+=bytes;
160
161 buffer_ptr=buffer;
162 buffer_end=buffer;
163
164 do
165 {
166 n=read(fd,buffer_end,bytes);
167
168 if(n<=0)
169 return(1);
170
171 buffer_end+=n;
172 bytes-=n;
173 }
174 while(bytes>0);
175
176 return(0);
177 }
178
179 #if defined(USE_GZIP) && USE_GZIP
180 static int uncompress_pbf(unsigned char *data,uint32_t compressed,uint32_t uncompressed);
181 #endif /* USE_GZIP */
182
183 static void process_string_table(unsigned char *data,uint32_t length);
184 static void process_primitive_group(unsigned char *data,uint32_t length);
185 static void process_nodes(unsigned char *data,uint32_t length);
186 static void process_dense_nodes(unsigned char *data,uint32_t length);
187 static void process_ways(unsigned char *data,uint32_t length);
188 static void process_relations(unsigned char *data,uint32_t length);
189
190
191 /* Macros to simplify the parser (and make it look more like the XML parser) */
192
193 #define BEGIN(xx) do{ state=(xx); goto finish_parsing; } while(0)
194
195 #define BUFFER_CHARS_EOF(xx) do{ if(buffer_refill(fd,(xx))) BEGIN(PBF_EOF); } while(0)
196
197 #define BUFFER_CHARS(xx) do{ if(buffer_refill(fd,(xx))) BEGIN(PBF_ERROR_UNEXP_EOF); } while(0)
198
199
200 /* PBF decoding */
201
202 #define PBF_FIELD(xx) (int)(((xx)&0xFFF8)>>3)
203 #define PBF_TYPE(xx) (int)((xx)&0x0007)
204
205 #define PBF_LATITUDE(xx) (double)(1E-9*(granularity*(xx)+lat_offset))
206 #define PBF_LONGITUDE(xx) (double)(1E-9*(granularity*(xx)+lon_offset))
207
208
209 /*++++++++++++++++++++++++++++++++++++++
210 Parse a PBF int32 data value.
211
212 uint32_t pbf_int32 Returns the integer value.
213
214 unsigned char **ptr The pointer to read the data from.
215 ++++++++++++++++++++++++++++++++++++++*/
216
217 static inline uint32_t pbf_int32(unsigned char **ptr)
218 {
219 uint32_t result=(**ptr)&0x7F;
220
221 if((**ptr)&0x80) result+=((*++(*ptr))&0x7F)<<7;
222 if((**ptr)&0x80) result+=((*++(*ptr))&0x7F)<<14;
223 if((**ptr)&0x80) result+=((*++(*ptr))&0x7F)<<21;
224 if((**ptr)&0x80) result+=((*++(*ptr))&0x7F)<<28;
225
226 (*ptr)++;
227
228 return(result);
229 }
230
231
232 /*++++++++++++++++++++++++++++++++++++++
233 Parse a PBF int64 data value.
234
235 int64_t pbf_int64 Returns the integer value.
236
237 unsigned char **ptr The pointer to read the data from.
238 ++++++++++++++++++++++++++++++++++++++*/
239
240 static inline int64_t pbf_int64(unsigned char **ptr)
241 {
242 uint64_t result=(**ptr)&0x7F;
243
244 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<7;
245 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<14;
246 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<21;
247 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<28;
248 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<35;
249 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<42;
250 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<49;
251 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<56;
252 if((**ptr)&0x80) result+=(uint64_t)((*++(*ptr))&0x7F)<<63;
253
254 (*ptr)++;
255
256 return(result);
257 }
258
259
260 /*++++++++++++++++++++++++++++++++++++++
261 Parse a PBF sint64 data value.
262
263 int64_t pbf_sint64 Returns the integer value.
264
265 unsigned char **ptr The pointer to read the data from.
266 ++++++++++++++++++++++++++++++++++++++*/
267
268 static inline int64_t pbf_sint64(unsigned char **ptr)
269 {
270 int64_t result=((**ptr)&0x7E)>>1;
271 int sign=(**ptr)&0x01;
272
273 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<6;
274 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<13;
275 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<20;
276 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<27;
277 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<34;
278 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<41;
279 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<48;
280 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<55;
281 if((**ptr)&0x80) result+=(int64_t)((*++(*ptr))&0x7F)<<62;
282
283 (*ptr)++;
284
285 if(sign)
286 result=-result-1;
287
288 return(result);
289 }
290
291
292 /*++++++++++++++++++++++++++++++++++++++
293 Parse a PBF length delimited data value.
294
295 unsigned char *pbf_length_delimited Returns a pointer to the start of the data.
296
297 unsigned char **ptr The pointer to read the data from.
298
299 uint32_t *length Returns the length of the data.
300 ++++++++++++++++++++++++++++++++++++++*/
301
302 static inline unsigned char *pbf_length_delimited(unsigned char **ptr,uint32_t *length)
303 {
304 uint32_t len=pbf_int32(ptr);
305
306 if(length)
307 *length=len;
308
309 *ptr+=len;
310
311 return(*ptr-len);
312 }
313
314
315 /*++++++++++++++++++++++++++++++++++++++
316 Skip any pbf field from a message.
317
318 unsigned char **ptr The pointer to read the data from.
319
320 int type The type of the data.
321 ++++++++++++++++++++++++++++++++++++++*/
322
323 static inline void pbf_skip(unsigned char **ptr,int type)
324 {
325 uint32_t length;
326
327 switch(type)
328 {
329 case 0: /* varint */
330 while((**ptr)&0x80) (*ptr)++;
331 (*ptr)++;
332 break;
333 case 1: /* 64-bit */
334 *ptr+=8;
335 break;
336 case 2: /* length delimited */
337 length=pbf_int32(ptr);
338 *ptr+=length;
339 break;
340 case 3: /* deprecated */
341 break;
342 case 4: /* deprecated */
343 break;
344 case 5: /* 32-bit */
345 *ptr+=4;
346 break;
347 }
348 }
349
350
351 /*++++++++++++++++++++++++++++++++++++++
352 Parse the PBF and call the functions for each OSM item as seen.
353
354 int ParsePBF Returns 0 if OK or something else in case of an error.
355
356 int fd The file descriptor of the file to parse.
357 ++++++++++++++++++++++++++++++++++++++*/
358
359 int ParsePBF(int fd)
360 {
361 int state;
362 unsigned char *error=NULL;
363
364 /* Print the initial message */
365
366 nnodes=0,nways=0,nrelations=0;
367
368 printf_first("Reading: Bytes=0 Nodes=0 Ways=0 Relations=0");
369
370 /* The actual parser. */
371
372 string_table_allocated=16384;
373 string_table_length=0;
374 string_table=(unsigned char **)malloc(string_table_allocated*sizeof(unsigned char *));
375 string_table_string_lengths=(uint32_t *)malloc(string_table_allocated*sizeof(uint32_t));
376
377 zbuffer_allocated=0;
378 zbuffer=NULL;
379
380 buffer_allocated=65536;
381 buffer=(unsigned char*)malloc(buffer_allocated);
382
383 buffer_ptr=buffer_end=buffer;
384
385 while(1)
386 {
387 int32_t blob_header_length=0;
388 int osm_data=0,osm_header=0;
389 int32_t blob_length=0;
390 uint32_t raw_size=0,compressed_size=0,uncompressed_size=0;
391 unsigned char *raw_data=NULL,*zlib_data=NULL;
392 uint32_t length;
393 unsigned char *data;
394
395 /* ================ Parsing states ================ */
396
397
398 BUFFER_CHARS_EOF(4);
399
400 blob_header_length=(256*(256*(256*(int)buffer_ptr[0])+(int)buffer_ptr[1])+(int)buffer_ptr[2])+buffer_ptr[3];
401 buffer_ptr+=4;
402
403 if(blob_header_length==0 || blob_header_length>LENGTH_32M)
404 BEGIN(PBF_ERROR_BLOB_HEADER_LEN);
405
406
407 BUFFER_CHARS(blob_header_length);
408
409 osm_header=0;
410 osm_data=0;
411
412 while(buffer_ptr<buffer_end)
413 {
414 int fieldtype=pbf_int32(&buffer_ptr);
415 int field=PBF_FIELD(fieldtype);
416
417 switch(field)
418 {
419 case PBF_VAL_BLOBHEADER_TYPE: /* string */
420 {
421 uint32_t length=0;
422 unsigned char *type=NULL;
423
424 type=pbf_length_delimited(&buffer_ptr,&length);
425
426 if(length==9 && !strncmp((char*)type,"OSMHeader",9))
427 osm_header=1;
428
429 if(length==7 && !strncmp((char*)type,"OSMData",7))
430 osm_data=1;
431 }
432 break;
433
434 case PBF_VAL_BLOBHEADER_SIZE: /* int32 */
435 blob_length=pbf_int32(&buffer_ptr);
436 break;
437
438 default:
439 pbf_skip(&buffer_ptr,PBF_TYPE(fieldtype));
440 }
441 }
442
443 if(blob_length==0 || blob_length>LENGTH_32M)
444 BEGIN(PBF_ERROR_BLOB_LEN);
445
446 if(!osm_data && !osm_header)
447 BEGIN(PBF_ERROR_NOT_OSM);
448
449
450 BUFFER_CHARS(blob_length);
451
452 while(buffer_ptr<buffer_end)
453 {
454 int fieldtype=pbf_int32(&buffer_ptr);
455 int field=PBF_FIELD(fieldtype);
456
457 switch(field)
458 {
459 case PBF_VAL_BLOB_RAW_DATA: /* bytes */
460 raw_data=pbf_length_delimited(&buffer_ptr,&raw_size);
461 break;
462
463 case PBF_VAL_BLOB_RAW_SIZE: /* int32 */
464 uncompressed_size=pbf_int32(&buffer_ptr);
465 break;
466
467 case PBF_VAL_BLOB_ZLIB_DATA: /* bytes */
468 zlib_data=pbf_length_delimited(&buffer_ptr,&compressed_size);
469 break;
470
471 default:
472 pbf_skip(&buffer_ptr,PBF_TYPE(fieldtype));
473 }
474 }
475
476 if(raw_data && zlib_data)
477 BEGIN(PBF_ERROR_BLOB_BOTH);
478
479 if(!raw_data && !zlib_data)
480 BEGIN(PBF_ERROR_BLOB_NEITHER);
481
482 if(zlib_data)
483 {
484 #if defined(USE_GZIP) && USE_GZIP
485 int newstate=uncompress_pbf(zlib_data,compressed_size,uncompressed_size);
486
487 if(newstate)
488 BEGIN(newstate);
489 #else
490 BEGIN(PBF_ERROR_NO_GZIP);
491 #endif
492 }
493 else
494 {
495 buffer_ptr=raw_data;
496 buffer_end=raw_data+raw_size;
497 }
498
499
500 if(osm_header)
501 {
502 while(buffer_ptr<buffer_end)
503 {
504 int fieldtype=pbf_int32(&buffer_ptr);
505 int field=PBF_FIELD(fieldtype);
506
507 switch(field)
508 {
509 case PBF_VAL_REQUIRED_FEATURES: /* string */
510 {
511 uint32_t length=0;
512 unsigned char *feature=NULL;
513
514 feature=pbf_length_delimited(&buffer_ptr,&length);
515
516 if(strncmp((char*)feature,"OsmSchema-V0.6",14) &&
517 strncmp((char*)feature,"DenseNodes",10))
518 {
519 feature[length]=0;
520 error=feature;
521 BEGIN(PBF_ERROR_UNSUPPORTED);
522 }
523 }
524 break;
525
526 case PBF_VAL_OPTIONAL_FEATURES: /* string */
527 pbf_length_delimited(&buffer_ptr,NULL);
528 break;
529
530 default:
531 pbf_skip(&buffer_ptr,PBF_TYPE(fieldtype));
532 }
533 }
534 }
535
536
537 if(osm_data)
538 {
539 unsigned char *primitive_group[8]={NULL};
540 uint32_t primitive_group_length[8]={0};
541 uint32_t nprimitive_groups=0,i;
542
543 granularity=100;
544 lat_offset=lon_offset=0;
545
546 while(buffer_ptr<buffer_end)
547 {
548 int fieldtype=pbf_int32(&buffer_ptr);
549 int field=PBF_FIELD(fieldtype);
550
551 switch(field)
552 {
553 case PBF_VAL_STRING_TABLE: /* bytes */
554 data=pbf_length_delimited(&buffer_ptr,&length);
555 process_string_table(data,length);
556 break;
557
558 case PBF_VAL_PRIMITIVE_GROUP: /* bytes */
559 primitive_group[nprimitive_groups]=pbf_length_delimited(&buffer_ptr,&primitive_group_length[nprimitive_groups]);
560
561 if(++nprimitive_groups>(sizeof(primitive_group)/sizeof(primitive_group[0])))
562 BEGIN(PBF_ERROR_TOO_MANY_GROUPS);
563 break;
564
565 case PBF_VAL_GRANULARITY: /* int32 */
566 granularity=pbf_int32(&buffer_ptr);
567 break;
568
569 case PBF_VAL_LAT_OFFSET: /* int64 */
570 lat_offset=pbf_int64(&buffer_ptr);
571 break;
572
573 case PBF_VAL_LON_OFFSET: /* int64 */
574 lon_offset=pbf_int64(&buffer_ptr);
575 break;
576
577 default:
578 pbf_skip(&buffer_ptr,PBF_TYPE(fieldtype));
579 }
580 }
581
582 if(nprimitive_groups)
583 for(i=0;i<nprimitive_groups;i++)
584 process_primitive_group(primitive_group[i],primitive_group_length[i]);
585 }
586 }
587
588
589 finish_parsing:
590
591 switch(state)
592 {
593 /* End of file */
594
595 case PBF_EOF:
596 break;
597
598
599 /* ================ Error states ================ */
600
601
602 case PBF_ERROR_UNEXP_EOF:
603 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": unexpected end of file seen.\n",byteno);
604 break;
605
606 case PBF_ERROR_BLOB_HEADER_LEN:
607 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": BlobHeader length is wrong (0<x<=32M).\n",byteno);
608 break;
609
610 case PBF_ERROR_BLOB_LEN:
611 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob length is wrong (0<x<=32M).\n",byteno);
612 break;
613
614 case PBF_ERROR_NOT_OSM:
615 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": BlobHeader is neither 'OSMData' or 'OSMHeader'.\n",byteno);
616 break;
617
618 case PBF_ERROR_BLOB_BOTH:
619 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob has both zlib compressed and raw uncompressed data.\n",byteno);
620 break;
621
622 case PBF_ERROR_BLOB_NEITHER:
623 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob has neither zlib compressed or raw uncompressed data.\n",byteno);
624 break;
625
626 case PBF_ERROR_NO_GZIP:
627 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob is compressed but no gzip support is available.\n",byteno);
628 break;
629
630 case PBF_ERROR_GZIP_INIT:
631 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob is compressed but failed to initialise decompression.\n",byteno);
632 break;
633
634 case PBF_ERROR_GZIP_INFLATE:
635 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob is compressed but failed to uncompress it.\n",byteno);
636 break;
637
638 case PBF_ERROR_GZIP_WRONG_LEN:
639 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob is compressed and wrong size when uncompressed.\n",byteno);
640 break;
641
642 case PBF_ERROR_GZIP_END:
643 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Blob is compressed but failed to finalise decompression.\n",byteno);
644 break;
645
646 case PBF_ERROR_UNSUPPORTED:
647 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": Unsupported required feature '%s'.\n",byteno,error);
648 break;
649
650 case PBF_ERROR_TOO_MANY_GROUPS:
651 fprintf(stderr,"PBF Parser: Error at byte %"PRIu64": OsmData message contains too many PrimitiveGroup messages.\n",byteno);
652 break;
653 }
654
655 /* Free the parser variables */
656
657 free(string_table);
658 free(string_table_string_lengths);
659
660 free(buffer);
661 if(zbuffer)
662 free(zbuffer);
663
664 /* Print the final message */
665
666 printf_last("Read: Bytes=%"PRIu64" Nodes=%"PRIu64" Ways=%"PRIu64" Relations=%"PRIu64,byteno,nnodes,nways,nrelations);
667
668 return(state);
669 }
670
671
672 /*++++++++++++++++++++++++++++++++++++++
673 Process a PBF StringTable message.
674
675 unsigned char *data The data to process.
676
677 uint32_t length The length of the data.
678 ++++++++++++++++++++++++++++++++++++++*/
679
680 static void process_string_table(unsigned char *data,uint32_t length)
681 {
682 unsigned char *end=data+length;
683 unsigned char *string;
684 uint32_t string_length;
685
686 string_table_length=0;
687
688 while(data<end)
689 {
690 int fieldtype=pbf_int32(&data);
691 int field=PBF_FIELD(fieldtype);
692
693 switch(field)
694 {
695 case PBF_VAL_STRING: /* string */
696 string=pbf_length_delimited(&data,&string_length);
697
698 if(string_table_length==string_table_allocated)
699 {
700 string_table_allocated+=8192;
701 string_table=(unsigned char **)realloc(string_table,string_table_allocated*sizeof(unsigned char *));
702 string_table_string_lengths=(uint32_t *)realloc(string_table_string_lengths,string_table_allocated*sizeof(uint32_t));
703 }
704
705 string_table[string_table_length]=string;
706 string_table_string_lengths[string_table_length]=string_length;
707
708 string_table_length++;
709 break;
710
711 default:
712 pbf_skip(&data,PBF_TYPE(fieldtype));
713 }
714 }
715 }
716
717
718 /*++++++++++++++++++++++++++++++++++++++
719 Process a PBF PrimitiveGroup message.
720
721 unsigned char *data The data to process.
722
723 uint32_t length The length of the data.
724 ++++++++++++++++++++++++++++++++++++++*/
725
726 static void process_primitive_group(unsigned char *data,uint32_t length)
727 {
728 unsigned char *end=data+length;
729 unsigned char *subdata;
730 uint32_t sublength;
731 int i;
732
733 /* Fixup the strings (not null terminated in buffer) */
734
735 for(i=0;i<string_table_length;i++)
736 string_table[i][string_table_string_lengths[i]]=0;
737
738
739 while(data<end)
740 {
741 int fieldtype=pbf_int32(&data);
742 int field=PBF_FIELD(fieldtype);
743
744 switch(field)
745 {
746 case PBF_VAL_NODES: /* message */
747 subdata=pbf_length_delimited(&data,&sublength);
748 process_nodes(subdata,sublength);
749 break;
750
751 case PBF_VAL_DENSE_NODES: /* message */
752 subdata=pbf_length_delimited(&data,&sublength);
753 process_dense_nodes(subdata,sublength);
754 break;
755
756 case PBF_VAL_WAYS: /* message */
757 subdata=pbf_length_delimited(&data,&sublength);
758 process_ways(subdata,sublength);
759 break;
760
761 case PBF_VAL_RELATIONS: /* message */
762 subdata=pbf_length_delimited(&data,&sublength);
763 process_relations(subdata,sublength);
764 break;
765
766 default:
767 pbf_skip(&data,PBF_TYPE(fieldtype));
768 }
769 }
770 }
771
772
773 /*++++++++++++++++++++++++++++++++++++++
774 Process a PBF Node message.
775
776 unsigned char *data The data to process.
777
778 uint32_t length The length of the data.
779 ++++++++++++++++++++++++++++++++++++++*/
780
781 static void process_nodes(unsigned char *data,uint32_t length)
782 {
783 unsigned char *end=data+length;
784 int64_t id=0;
785 unsigned char *keys=NULL,*vals=NULL;
786 unsigned char *keys_end=NULL,*vals_end=NULL;
787 uint32_t keylen=0,vallen=0;
788 int64_t lat=0,lon=0;
789 TagList *tags=NULL,*result=NULL;
790
791 while(data<end)
792 {
793 int fieldtype=pbf_int32(&data);
794 int field=PBF_FIELD(fieldtype);
795
796 switch(field)
797 {
798 case PBF_VAL_NODE_ID: /* sint64 */
799 id=pbf_sint64(&data);
800 break;
801
802 case PBF_VAL_NODE_KEYS: /* packed int32 */
803 keys=pbf_length_delimited(&data,&keylen);
804 keys_end=keys+keylen;
805 break;
806
807 case PBF_VAL_NODE_VALS: /* packed int32 */
808 vals=pbf_length_delimited(&data,&vallen);
809 vals_end=vals+vallen;
810 break;
811
812 case PBF_VAL_NODE_LAT: /* sint64 */
813 lat=pbf_sint64(&data);
814 break;
815
816 case PBF_VAL_NODE_LON: /* sint64 */
817 lon=pbf_sint64(&data);
818 break;
819
820 default:
821 pbf_skip(&data,PBF_TYPE(fieldtype));
822 }
823 }
824
825 /* Mangle the data and send it to the OSM parser */
826
827 nnodes++;
828
829 if(!(nnodes%10000))
830 printf_middle("Reading: Bytes=%"PRIu64" Nodes=%"PRIu64" Ways=%"PRIu64" Relations=%"PRIu64,byteno,nnodes,nways,nrelations);
831
832 tags=NewTagList();
833
834 if(keys && vals)
835 {
836 while(keys<keys_end && vals<vals_end)
837 {
838 uint32_t key=pbf_int32(&keys);
839 uint32_t val=pbf_int32(&vals);
840
841 AppendTag(tags,(char*)string_table[key],(char*)string_table[val]);
842 }
843 }
844
845 result=ApplyNodeTaggingRules(tags,id);
846
847 ProcessNodeTags(result,id,PBF_LATITUDE(lat),PBF_LONGITUDE(lon),MODE_NORMAL);
848
849 DeleteTagList(tags);
850 DeleteTagList(result);
851 }
852
853
854 /*++++++++++++++++++++++++++++++++++++++
855 Process a PBF DenseNode message.
856
857 unsigned char *data The data to process.
858
859 uint32_t length The length of the data.
860 ++++++++++++++++++++++++++++++++++++++*/
861
862 static void process_dense_nodes(unsigned char *data,uint32_t length)
863 {
864 unsigned char *end=data+length;
865 unsigned char *ids=NULL,*keys_vals=NULL,*lats=NULL,*lons=NULL;
866 unsigned char *ids_end=NULL;
867 uint32_t idlen=0;
868 int64_t id=0;
869 int64_t lat=0,lon=0;
870 TagList *tags=NULL,*result;
871
872 while(data<end)
873 {
874 int fieldtype=pbf_int32(&data);
875 int field=PBF_FIELD(fieldtype);
876
877 switch(field)
878 {
879 case PBF_VAL_DENSE_NODE_ID: /* packed sint64 */
880 ids=pbf_length_delimited(&data,&idlen);
881 ids_end=ids+idlen;
882 break;
883
884 case PBF_VAL_DENSE_NODE_LAT: /* packed sint64 */
885 lats=pbf_length_delimited(&data,NULL);
886 break;
887
888 case PBF_VAL_DENSE_NODE_LON: /* packed sint64 */
889 lons=pbf_length_delimited(&data,NULL);
890 break;
891
892 case PBF_VAL_DENSE_NODE_KEYS_VALS: /* packed int32 */
893 keys_vals=pbf_length_delimited(&data,NULL);
894 break;
895
896 default:
897 pbf_skip(&data,PBF_TYPE(fieldtype));
898 }
899 }
900
901 while(ids<ids_end)
902 {
903 int64_t delta_id;
904 int64_t delta_lat,delta_lon;
905
906 delta_id=pbf_sint64(&ids);
907 delta_lat=pbf_sint64(&lats);
908 delta_lon=pbf_sint64(&lons);
909
910 id+=delta_id;
911 lat+=delta_lat;
912 lon+=delta_lon;
913
914 /* Mangle the data and send it to the OSM parser */
915
916 nnodes++;
917
918 if(!(nnodes%10000))
919 printf_middle("Reading: Bytes=%"PRIu64" Nodes=%"PRIu64" Ways=%"PRIu64" Relations=%"PRIu64,byteno,nnodes,nways,nrelations);
920
921 tags=NewTagList();
922
923 if(keys_vals)
924 {
925 while(1)
926 {
927 uint32_t key=pbf_int32(&keys_vals),val;
928
929 if(key==0)
930 break;
931
932 val=pbf_int32(&keys_vals);
933
934 AppendTag(tags,(char*)string_table[key],(char*)string_table[val]);
935 }
936 }
937
938 result=ApplyNodeTaggingRules(tags,id);
939
940 ProcessNodeTags(result,id,PBF_LATITUDE(lat),PBF_LONGITUDE(lon),MODE_NORMAL);
941
942 DeleteTagList(tags);
943 DeleteTagList(result);
944 }
945 }
946
947
948 /*++++++++++++++++++++++++++++++++++++++
949 Process a PBF Way message.
950
951 unsigned char *data The data to process.
952
953 uint32_t length The length of the data.
954 ++++++++++++++++++++++++++++++++++++++*/
955
956 static void process_ways(unsigned char *data,uint32_t length)
957 {
958 unsigned char *end=data+length;
959 int64_t id=0;
960 unsigned char *keys=NULL,*vals=NULL,*refs=NULL;
961 unsigned char *keys_end=NULL,*vals_end=NULL,*refs_end=NULL;
962 uint32_t keylen=0,vallen=0,reflen=0;
963 int64_t ref=0;
964 TagList *tags=NULL,*result;
965
966 while(data<end)
967 {
968 int fieldtype=pbf_int32(&data);
969 int field=PBF_FIELD(fieldtype);
970
971 switch(field)
972 {
973 case PBF_VAL_WAY_ID: /* int64 */
974 id=pbf_int64(&data);
975 break;
976
977 case PBF_VAL_WAY_KEYS: /* packed int32 */
978 keys=pbf_length_delimited(&data,&keylen);
979 keys_end=keys+keylen;
980 break;
981
982 case PBF_VAL_WAY_VALS: /* packed int32 */
983 vals=pbf_length_delimited(&data,&vallen);
984 vals_end=vals+vallen;
985 break;
986
987 case PBF_VAL_WAY_REFS: /* packed sint64 */
988 refs=pbf_length_delimited(&data,&reflen);
989 refs_end=refs+reflen;
990 break;
991
992 default:
993 pbf_skip(&data,PBF_TYPE(fieldtype));
994 }
995 }
996
997 /* Mangle the data and send it to the OSM parser */
998
999 nways++;
1000
1001 if(!(nways%1000))
1002 printf_middle("Reading: Bytes=%"PRIu64" Nodes=%"PRIu64" Ways=%"PRIu64" Relations=%"PRIu64,byteno,nnodes,nways,nrelations);
1003
1004 tags=NewTagList();
1005
1006 if(keys && vals)
1007 {
1008 while(keys<keys_end && vals<vals_end)
1009 {
1010 uint32_t key=pbf_int32(&keys);
1011 uint32_t val=pbf_int32(&vals);
1012
1013 AppendTag(tags,(char*)string_table[key],(char*)string_table[val]);
1014 }
1015 }
1016
1017 AddWayRefs(0);
1018
1019 if(refs)
1020 while(refs<refs_end)
1021 {
1022 int64_t delta_ref;
1023
1024 delta_ref=pbf_sint64(&refs);
1025
1026 ref+=delta_ref;
1027
1028 if(ref==0)
1029 break;
1030
1031 AddWayRefs(ref);
1032 }
1033
1034 result=ApplyWayTaggingRules(tags,id);
1035
1036 ProcessWayTags(result,id,MODE_NORMAL);
1037
1038 DeleteTagList(tags);
1039 DeleteTagList(result);
1040 }
1041
1042
1043 /*++++++++++++++++++++++++++++++++++++++
1044 Process a PBF Relation message.
1045
1046 unsigned char *data The data to process.
1047
1048 uint32_t length The length of the data.
1049 ++++++++++++++++++++++++++++++++++++++*/
1050
1051 static void process_relations(unsigned char *data,uint32_t length)
1052 {
1053 unsigned char *end=data+length;
1054 int64_t id=0;
1055 unsigned char *keys=NULL,*vals=NULL,*roles=NULL,*memids=NULL,*types=NULL;
1056 unsigned char *keys_end=NULL,*vals_end=NULL,*memids_end=NULL,*types_end=NULL;
1057 uint32_t keylen=0,vallen=0,rolelen=0,memidlen=0,typelen=0;
1058 int64_t memid=0;
1059 TagList *tags=NULL,*result;
1060
1061 while(data<end)
1062 {
1063 int fieldtype=pbf_int32(&data);
1064 int field=PBF_FIELD(fieldtype);
1065
1066 switch(field)
1067 {
1068 case PBF_VAL_RELATION_ID: /* int64 */
1069 id=pbf_int64(&data);
1070 break;
1071
1072 case PBF_VAL_RELATION_KEYS: /* packed string */
1073 keys=pbf_length_delimited(&data,&keylen);
1074 keys_end=keys+keylen;
1075 break;
1076
1077 case PBF_VAL_RELATION_VALS: /* packed string */
1078 vals=pbf_length_delimited(&data,&vallen);
1079 vals_end=vals+vallen;
1080 break;
1081
1082 case PBF_VAL_RELATION_ROLES: /* packed int32 */
1083 roles=pbf_length_delimited(&data,&rolelen);
1084 break;
1085
1086 case PBF_VAL_RELATION_MEMIDS: /* packed sint64 */
1087 memids=pbf_length_delimited(&data,&memidlen);
1088 memids_end=memids+memidlen;
1089 break;
1090
1091 case PBF_VAL_RELATION_TYPES: /* packed enum */
1092 types=pbf_length_delimited(&data,&typelen);
1093 types_end=types+typelen;
1094 break;
1095
1096 default:
1097 pbf_skip(&data,PBF_TYPE(fieldtype));
1098 }
1099 }
1100
1101 /* Mangle the data and send it to the OSM parser */
1102
1103 nrelations++;
1104
1105 if(!(nrelations%1000))
1106 printf_middle("Reading: Bytes=%"PRIu64" Nodes=%"PRIu64" Ways=%"PRIu64" Relations=%"PRIu64,byteno,nnodes,nways,nrelations);
1107
1108 AddRelationRefs(0,0,0,NULL);
1109
1110 tags=NewTagList();
1111
1112 if(keys && vals)
1113 {
1114 while(keys<keys_end && vals<vals_end)
1115 {
1116 uint32_t key=pbf_int32(&keys);
1117 uint32_t val=pbf_int32(&vals);
1118
1119 AppendTag(tags,(char*)string_table[key],(char*)string_table[val]);
1120 }
1121 }
1122
1123 if(memids && types)
1124 while(memids<memids_end && types<types_end)
1125 {
1126 int64_t delta_memid;
1127 unsigned char *role=NULL;
1128 int type;
1129
1130 delta_memid=pbf_sint64(&memids);
1131 type=pbf_int32(&types);
1132
1133 if(roles)
1134 role=string_table[pbf_int32(&roles)];
1135
1136 memid+=delta_memid;
1137
1138 if(type==0)
1139 AddRelationRefs(memid,0,0,(char*)role);
1140 else if(type==1)
1141 AddRelationRefs(0,memid,0,(char*)role);
1142 else if(type==2)
1143 AddRelationRefs(0,0,memid,(char*)role);
1144 }
1145
1146 result=ApplyRelationTaggingRules(tags,id);
1147
1148 ProcessRelationTags(result,id,MODE_NORMAL);
1149
1150 DeleteTagList(tags);
1151 DeleteTagList(result);
1152 }
1153
1154
1155 #if defined(USE_GZIP) && USE_GZIP
1156
1157 /*++++++++++++++++++++++++++++++++++++++
1158 Uncompress the part of the PBF data that is compressed.
1159
1160 int uncompress_pbf Returns the error state or 0 if OK.
1161
1162 unsigned char *data The data to uncompress.
1163
1164 uint32_t compressed The number of bytes to uncompress.
1165
1166 uint32_t uncompressed The number of bytes expected when uncompressed.
1167 ++++++++++++++++++++++++++++++++++++++*/
1168
1169 static int uncompress_pbf(unsigned char *data,uint32_t compressed,uint32_t uncompressed)
1170 {
1171 z_stream z={0};
1172
1173 if(uncompressed>zbuffer_allocated)
1174 zbuffer=(unsigned char *)realloc(zbuffer,zbuffer_allocated=uncompressed);
1175
1176 if(inflateInit2(&z,15+32)!=Z_OK)
1177 return(PBF_ERROR_GZIP_INIT);
1178
1179 z.next_in=data;
1180 z.avail_in=compressed;
1181
1182 z.next_out=zbuffer;
1183 z.avail_out=uncompressed;
1184
1185 if(inflate(&z,Z_FINISH)!=Z_STREAM_END)
1186 return(PBF_ERROR_GZIP_INFLATE);
1187
1188 if(z.avail_out!=0)
1189 return(PBF_ERROR_GZIP_WRONG_LEN);
1190
1191 if(inflateEnd(&z)!=Z_OK)
1192 return(PBF_ERROR_GZIP_END);
1193
1194 buffer_ptr=zbuffer;
1195 buffer_end=zbuffer+uncompressed;
1196
1197 return(0);
1198 }
1199
1200 #endif /* USE_GZIP */
1201
1202
1203 /*++++++++++++++++++++++++++++++++++++++
1204 Parse a PBF format OSM file (from planet download).
1205
1206 int ParsePBFFile Returns 0 if OK or something else in case of an error.
1207
1208 int fd The file descriptor of the file to read from.
1209
1210 NodesX *OSMNodes The data structure of nodes to fill in.
1211
1212 WaysX *OSMWays The data structure of ways to fill in.
1213
1214 RelationsX *OSMRelations The data structure of relations to fill in.
1215 ++++++++++++++++++++++++++++++++++++++*/
1216
1217 int ParsePBFFile(int fd,NodesX *OSMNodes,WaysX *OSMWays,RelationsX *OSMRelations)
1218 {
1219 int retval;
1220
1221 /* Initialise the parser */
1222
1223 InitialiseParser(OSMNodes,OSMWays,OSMRelations);
1224
1225 /* Parse the file */
1226
1227 retval=ParsePBF(fd);
1228
1229 /* Cleanup the parser */
1230
1231 CleanupParser();
1232
1233 return(retval);
1234 }