trunk/src/planetsplitter.c

/***************************************
 $Header: /home/amb/CVS/routino/src/planetsplitter.c,v 1.75 2010-07-14 18:00:09 amb Exp $

 OSM planet file splitter.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2008-2010 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

#include "types.h"
#include "ways.h"

#include "typesx.h"
#include "nodesx.h"
#include "segmentsx.h"
#include "waysx.h"
#include "superx.h"

#include "files.h"
#include "functions.h"
#include "functionsx.h"
#include "tagging.h"


/* Global variables */

/*+ The name of the temporary directory. +*/
char *option_tmpdirname=NULL;

/*+ The amount of RAM to use for filesorting. +*/
size_t option_filesort_ramsize=0;


/* Local functions */

static void print_usage(int detail);


/*++++++++++++++++++++++++++++++++++++++
  The main program for the planetsplitter.
  ++++++++++++++++++++++++++++++++++++++*/

int main(int argc,char** argv)
{
 NodesX    *Nodes;
 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
 WaysX     *Ways;
 int        iteration=0,quit=0;
 int        max_iterations=10;
 char      *dirname=NULL,*prefix=NULL,*tagging=NULL;
 int        option_parse_only=0,option_process_only=0;
 int        option_filenames=0;
 int        arg;

 /* Parse the command line arguments */

 for(arg=1;arg<argc;arg++)
   {
    if(!strcmp(argv[arg],"--help"))
       print_usage(1);
    else if(!strncmp(argv[arg],"--sort-ram-size=",16))
       option_filesort_ramsize=atoi(&argv[arg][16]);
    else if(!strncmp(argv[arg],"--dir=",6))
       dirname=&argv[arg][6];
    else if(!strncmp(argv[arg],"--tmpdir=",9))
       option_tmpdirname=&argv[arg][9];
    else if(!strncmp(argv[arg],"--prefix=",9))
       prefix=&argv[arg][9];
    else if(!strcmp(argv[arg],"--parse-only"))
       option_parse_only=1;
    else if(!strcmp(argv[arg],"--process-only"))
       option_process_only=1;
    else if(!strncmp(argv[arg],"--max-iterations=",17))
       max_iterations=atoi(&argv[arg][17]);
    else if(!strncmp(argv[arg],"--tagging=",10))
       tagging=&argv[arg][10];
    else if(argv[arg][0]=='-' && argv[arg][1]=='-')
       print_usage(0);
    else
       option_filenames++;
   }

 /* Check the specified command line options */

 if(option_parse_only && option_process_only)
    print_usage(0);

 if(option_filenames && option_process_only)
    print_usage(0);

 if(!option_filesort_ramsize)
   {
#if SLIM
       option_filesort_ramsize=64*1024*1024;
#else
       option_filesort_ramsize=256*1024*1024;
#endif
   }
 else
    option_filesort_ramsize*=1024*1024;

 if(!option_tmpdirname)
   {
    if(!dirname)
       option_tmpdirname=".";
    else
       option_tmpdirname=dirname;
   }

 if(tagging && ExistsFile(tagging))
    ;
 else if(!tagging && ExistsFile(FileName(dirname,prefix,"tagging.xml")))
    tagging=FileName(dirname,prefix,"tagging.xml");

 if(tagging && ParseXMLTaggingRules(tagging))
   {
    fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
    return(1);
   }

 if(!tagging)
   {
    fprintf(stderr,"Error: Cannot run without reading some tagging rules.\n");
    return(1);
   }

 /* Create new node, segment and way variables */

 Nodes=NewNodeList(option_parse_only||option_process_only);

 Segments=NewSegmentList(option_parse_only||option_process_only);

 Ways=NewWayList(option_parse_only||option_process_only);

 /* Parse the file */

 if(option_filenames)
   {
    for(arg=1;arg<argc;arg++)
      {
       FILE *file;

       if(argv[arg][0]=='-' && argv[arg][1]=='-')
          continue;

       file=fopen(argv[arg],"rb");

       if(!file)
         {
          fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
          exit(EXIT_FAILURE);
         }

       printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
       fflush(stdout);

       if(ParseOSM(file,Nodes,Segments,Ways))
          exit(EXIT_FAILURE);

       fclose(file);
      }
   }
 else if(!option_process_only)
   {
    printf("\nParse OSM Data\n==============\n\n");
    fflush(stdout);

    if(ParseOSM(stdin,Nodes,Segments,Ways))
       exit(EXIT_FAILURE);
   }

 if(option_parse_only)
   {
    FreeNodeList(Nodes,1);
    FreeSegmentList(Segments,1);
    FreeWayList(Ways,1);

    return(0);
   }

 /* Process the data */

 printf("\nProcess OSM Data\n================\n\n");
 fflush(stdout);

 /* Sort the nodes, segments and ways */

 SortNodeList(Nodes);

 SortSegmentList(Segments);

 SortWayList(Ways);

 /* Remove bad segments (must be after sorting the nodes and segments) */

 RemoveBadSegments(Nodes,Segments);

 /* Remove non-highway nodes (must be after removing the bad segments) */

 RemoveNonHighwayNodes(Nodes,Segments);

 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */

 UpdateSegments(Segments,Nodes,Ways);


 /* Repeated iteration on Super-Nodes and Super-Segments */

 do
   {
    printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
    fflush(stdout);

    if(iteration==0)
      {
       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,Segments,Ways);

       /* Select the super-segments */

       SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration);
      }
    else
      {
       SegmentsX *SuperSegments2;

       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,SuperSegments,Ways);

       /* Select the super-segments */

       SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration);

       if(SuperSegments->xnumber==SuperSegments2->xnumber)
          quit=1;

       FreeSegmentList(SuperSegments,0);

       SuperSegments=SuperSegments2;
      }

    /* Sort the super-segments */

    SortSegmentList(SuperSegments);

    /* Remove duplicated super-segments */

    DeduplicateSegments(SuperSegments,Nodes,Ways);

    iteration++;

    if(iteration>max_iterations)
       quit=1;
   }
 while(!quit);

 /* Combine the super-segments */

 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
 fflush(stdout);

 /* Merge the super-segments */

 MergedSegments=MergeSuperSegments(Segments,SuperSegments);

 FreeSegmentList(Segments,0);

 FreeSegmentList(SuperSegments,0);

 Segments=MergedSegments;

 /* Rotate segments so that node1<node2 */

 RotateSegments(Segments);

 /* Sort the segments */

 SortSegmentList(Segments);

 /* Remove duplicated segments */

 DeduplicateSegments(Segments,Nodes,Ways);

 /* Cross reference the nodes and segments */

 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
 fflush(stdout);

 /* Sort the node list geographically */

 SortNodeListGeographically(Nodes);

 /* Create the real segments and nodes */

 CreateRealNodes(Nodes,iteration);

 CreateRealSegments(Segments,Ways);

 /* Fix the segment and node indexes */

 IndexNodes(Nodes,Segments);

 IndexSegments(Segments,Nodes);

 /* Output the results */

 printf("\nWrite Out Database Files\n========================\n\n");
 fflush(stdout);

 /* Write out the nodes */

 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"));

 FreeNodeList(Nodes,0);

 /* Write out the segments */

 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));

 FreeSegmentList(Segments,0);

 /* Write out the ways */

 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));

 FreeWayList(Ways,0);

 return(0);
}


/*++++++++++++++++++++++++++++++++++++++
  Print out the usage information.

  int detail The level of detail to use - 0 = low, 1 = high.
  ++++++++++++++++++++++++++++++++++++++*/

static void print_usage(int detail)
{
 fprintf(stderr,
         "Usage: planetsplitter [--help]\n"
         "                      [--dir=<dirname>] [--prefix=<name>]\n"
         "                      [--sort-ram-size=<size>]\n"
         "                      [--tmpdir=<dirname>]\n"
         "                      [--parse-only | --process-only]\n"
         "                      [--max-iterations=<number>]\n"
         "                      [--tagging=<filename>]\n"
         "                      [<filename.osm> ...]\n");

 if(detail)
    fprintf(stderr,
            "\n"
            "--help                    Prints this information.\n"
            "\n"
            "--dir=<dirname>           The directory containing the routing database.\n"
            "--prefix=<name>           The filename prefix for the routing database.\n"
            "\n"
            "--sort-ram-size=<size>    The amount of RAM (in MB) to use for data sorting\n"
#if SLIM
            "                          (defaults to 64MB otherwise.)\n"
#else
            "                          (defaults to 256MB otherwise.)\n"
#endif
            "--tmpdir=<dirname>        The directory name for temporary files.\n"
            "                          (defaults to the '--dir' option directory.)\n"
            "\n"
            "--parse-only              Parse the input OSM files and store the results.\n"
            "--process-only            Process the stored results from previous option.\n"
            "\n"
            "--max-iterations=<number> The number of iterations for finding super-nodes.\n"
            "\n"
            "--tagging=<filename>      The name of the XML file containing the tagging rules\n"
            "                          (defaults to 'tagging.xml' with '--dirname' and\n"
            "                           '--prefix' options).\n"
            "\n"
            "<filename.osm> ...        The name(s) of the file(s) to process (by default\n"
            "                          data is read from standard input).\n"
            "\n"
            "<transport> defaults to all but can be set to:\n"
            "%s"
            "\n"
            "<highway> can be selected from:\n"
            "%s"
            "\n"
            "<property> can be selected from:\n"
            "%s",
            TransportList(),HighwayList(),PropertyList());

 exit(!detail);
}
1	/***************************************
2	$Header: /home/amb/CVS/routino/src/planetsplitter.c,v 1.75 2010-07-14 18:00:09 amb Exp $
3
4	OSM planet file splitter.
5
6	Part of the Routino routing software.
7	****************/ /****************
8	This file Copyright 2008-2010 Andrew M. Bishop
9
10	This program is free software: you can redistribute it and/or modify
11	it under the terms of the GNU Affero General Public License as published by
12	the Free Software Foundation, either version 3 of the License, or
13	(at your option) any later version.
14
15	This program is distributed in the hope that it will be useful,
16	but WITHOUT ANY WARRANTY; without even the implied warranty of
17	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	GNU Affero General Public License for more details.
19
20	You should have received a copy of the GNU Affero General Public License
21	along with this program. If not, see <http://www.gnu.org/licenses/>.
22	***************************************/
23
24
25	#include <stdio.h>
26	#include <stdlib.h>
27	#include <string.h>
28	#include <errno.h>
29
30	#include "types.h"
31	#include "ways.h"
32
33	#include "typesx.h"
34	#include "nodesx.h"
35	#include "segmentsx.h"
36	#include "waysx.h"
37	#include "superx.h"
38
39	#include "files.h"
40	#include "functions.h"
41	#include "functionsx.h"
42	#include "tagging.h"
43
44
45	/* Global variables */
46
47	/+ The name of the temporary directory. +/
48	char *option_tmpdirname=NULL;
49
50	/+ The amount of RAM to use for filesorting. +/
51	size_t option_filesort_ramsize=0;
52
53
54	/* Local functions */
55
56	static void print_usage(int detail);
57
58
59	/*++++++++++++++++++++++++++++++++++++++
60	The main program for the planetsplitter.
61	++++++++++++++++++++++++++++++++++++++*/
62
63	int main(int argc,char** argv)
64	{
65	NodesX *Nodes;
66	SegmentsX Segments,SuperSegments=NULL,*MergedSegments=NULL;
67	WaysX *Ways;
68	int iteration=0,quit=0;
69	int max_iterations=10;
70	char dirname=NULL,prefix=NULL,*tagging=NULL;
71	int option_parse_only=0,option_process_only=0;
72	int option_filenames=0;
73	int arg;
74
75	/* Parse the command line arguments */
76
77	for(arg=1;arg<argc;arg++)
78	{
79	if(!strcmp(argv[arg],"--help"))
80	print_usage(1);
81	else if(!strncmp(argv[arg],"--sort-ram-size=",16))
82	option_filesort_ramsize=atoi(&argv[arg][16]);
83	else if(!strncmp(argv[arg],"--dir=",6))
84	dirname=&argv[arg][6];
85	else if(!strncmp(argv[arg],"--tmpdir=",9))
86	option_tmpdirname=&argv[arg][9];
87	else if(!strncmp(argv[arg],"--prefix=",9))
88	prefix=&argv[arg][9];
89	else if(!strcmp(argv[arg],"--parse-only"))
90	option_parse_only=1;
91	else if(!strcmp(argv[arg],"--process-only"))
92	option_process_only=1;
93	else if(!strncmp(argv[arg],"--max-iterations=",17))
94	max_iterations=atoi(&argv[arg][17]);
95	else if(!strncmp(argv[arg],"--tagging=",10))
96	tagging=&argv[arg][10];
97	else if(argv[arg][0]=='-' && argv[arg][1]=='-')
98	print_usage(0);
99	else
100	option_filenames++;
101	}
102
103	/* Check the specified command line options */
104
105	if(option_parse_only && option_process_only)
106	print_usage(0);
107
108	if(option_filenames && option_process_only)
109	print_usage(0);
110
111	if(!option_filesort_ramsize)
112	{
113	#if SLIM
114	option_filesort_ramsize=6410241024;
115	#else
116	option_filesort_ramsize=25610241024;
117	#endif
118	}
119	else
120	option_filesort_ramsize=10241024;
121
122	if(!option_tmpdirname)
123	{
124	if(!dirname)
125	option_tmpdirname=".";
126	else
127	option_tmpdirname=dirname;
128	}
129
130	if(tagging && ExistsFile(tagging))
131	;
132	else if(!tagging && ExistsFile(FileName(dirname,prefix,"tagging.xml")))
133	tagging=FileName(dirname,prefix,"tagging.xml");
134
135	if(tagging && ParseXMLTaggingRules(tagging))
136	{
137	fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
138	return(1);
139	}
140
141	if(!tagging)
142	{
143	fprintf(stderr,"Error: Cannot run without reading some tagging rules.\n");
144	return(1);
145	}
146
147	/* Create new node, segment and way variables */
148
149	Nodes=NewNodeList(option_parse_only\|\|option_process_only);
150
151	Segments=NewSegmentList(option_parse_only\|\|option_process_only);
152
153	Ways=NewWayList(option_parse_only\|\|option_process_only);
154
155	/* Parse the file */
156
157	if(option_filenames)
158	{
159	for(arg=1;arg<argc;arg++)
160	{
161	FILE *file;
162
163	if(argv[arg][0]=='-' && argv[arg][1]=='-')
164	continue;
165
166	file=fopen(argv[arg],"rb");
167
168	if(!file)
169	{
170	fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
171	exit(EXIT_FAILURE);
172	}
173
174	printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
175	fflush(stdout);
176
177	if(ParseOSM(file,Nodes,Segments,Ways))
178	exit(EXIT_FAILURE);
179
180	fclose(file);
181	}
182	}
183	else if(!option_process_only)
184	{
185	printf("\nParse OSM Data\n==============\n\n");
186	fflush(stdout);
187
188	if(ParseOSM(stdin,Nodes,Segments,Ways))
189	exit(EXIT_FAILURE);
190	}
191
192	if(option_parse_only)
193	{
194	FreeNodeList(Nodes,1);
195	FreeSegmentList(Segments,1);
196	FreeWayList(Ways,1);
197
198	return(0);
199	}
200
201	/* Process the data */
202
203	printf("\nProcess OSM Data\n================\n\n");
204	fflush(stdout);
205
206	/* Sort the nodes, segments and ways */
207
208	SortNodeList(Nodes);
209
210	SortSegmentList(Segments);
211
212	SortWayList(Ways);
213
214	/* Remove bad segments (must be after sorting the nodes and segments) */
215
216	RemoveBadSegments(Nodes,Segments);
217
218	/* Remove non-highway nodes (must be after removing the bad segments) */
219
220	RemoveNonHighwayNodes(Nodes,Segments);
221
222	/* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
223
224	UpdateSegments(Segments,Nodes,Ways);
225
226
227	/* Repeated iteration on Super-Nodes and Super-Segments */
228
229	do
230	{
231	printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
232	fflush(stdout);
233
234	if(iteration==0)
235	{
236	/* Select the super-nodes */
237
238	ChooseSuperNodes(Nodes,Segments,Ways);
239
240	/* Select the super-segments */
241
242	SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration);
243	}
244	else
245	{
246	SegmentsX *SuperSegments2;
247
248	/* Select the super-nodes */
249
250	ChooseSuperNodes(Nodes,SuperSegments,Ways);
251
252	/* Select the super-segments */
253
254	SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration);
255
256	if(SuperSegments->xnumber==SuperSegments2->xnumber)
257	quit=1;
258
259	FreeSegmentList(SuperSegments,0);
260
261	SuperSegments=SuperSegments2;
262	}
263
264	/* Sort the super-segments */
265
266	SortSegmentList(SuperSegments);
267
268	/* Remove duplicated super-segments */
269
270	DeduplicateSegments(SuperSegments,Nodes,Ways);
271
272	iteration++;
273
274	if(iteration>max_iterations)
275	quit=1;
276	}
277	while(!quit);
278
279	/* Combine the super-segments */
280
281	printf("\nCombine Segments and Super-Segments\n===================================\n\n");
282	fflush(stdout);
283
284	/* Merge the super-segments */
285
286	MergedSegments=MergeSuperSegments(Segments,SuperSegments);
287
288	FreeSegmentList(Segments,0);
289
290	FreeSegmentList(SuperSegments,0);
291
292	Segments=MergedSegments;
293
294	/* Rotate segments so that node1<node2 */
295
296	RotateSegments(Segments);
297
298	/* Sort the segments */
299
300	SortSegmentList(Segments);
301
302	/* Remove duplicated segments */
303
304	DeduplicateSegments(Segments,Nodes,Ways);
305
306	/* Cross reference the nodes and segments */
307
308	printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
309	fflush(stdout);
310
311	/* Sort the node list geographically */
312
313	SortNodeListGeographically(Nodes);
314
315	/* Create the real segments and nodes */
316
317	CreateRealNodes(Nodes,iteration);
318
319	CreateRealSegments(Segments,Ways);
320
321	/* Fix the segment and node indexes */
322
323	IndexNodes(Nodes,Segments);
324
325	IndexSegments(Segments,Nodes);
326
327	/* Output the results */
328
329	printf("\nWrite Out Database Files\n========================\n\n");
330	fflush(stdout);
331
332	/* Write out the nodes */
333
334	SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"));
335
336	FreeNodeList(Nodes,0);
337
338	/* Write out the segments */
339
340	SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
341
342	FreeSegmentList(Segments,0);
343
344	/* Write out the ways */
345
346	SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
347
348	FreeWayList(Ways,0);
349
350	return(0);
351	}
352
353
354	/*++++++++++++++++++++++++++++++++++++++
355	Print out the usage information.
356
357	int detail The level of detail to use - 0 = low, 1 = high.
358	++++++++++++++++++++++++++++++++++++++*/
359
360	static void print_usage(int detail)
361	{
362	fprintf(stderr,
363	"Usage: planetsplitter [--help]\n"
364	" [--dir=<dirname>] [--prefix=<name>]\n"
365	" [--sort-ram-size=<size>]\n"
366	" [--tmpdir=<dirname>]\n"
367	" [--parse-only \| --process-only]\n"
368	" [--max-iterations=<number>]\n"
369	" [--tagging=<filename>]\n"
370	" [<filename.osm> ...]\n");
371
372	if(detail)
373	fprintf(stderr,
374	"\n"
375	"--help Prints this information.\n"
376	"\n"
377	"--dir=<dirname> The directory containing the routing database.\n"
378	"--prefix=<name> The filename prefix for the routing database.\n"
379	"\n"
380	"--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
381	#if SLIM
382	" (defaults to 64MB otherwise.)\n"
383	#else
384	" (defaults to 256MB otherwise.)\n"
385	#endif
386	"--tmpdir=<dirname> The directory name for temporary files.\n"
387	" (defaults to the '--dir' option directory.)\n"
388	"\n"
389	"--parse-only Parse the input OSM files and store the results.\n"
390	"--process-only Process the stored results from previous option.\n"
391	"\n"
392	"--max-iterations=<number> The number of iterations for finding super-nodes.\n"
393	"\n"
394	"--tagging=<filename> The name of the XML file containing the tagging rules\n"
395	" (defaults to 'tagging.xml' with '--dirname' and\n"
396	" '--prefix' options).\n"
397	"\n"
398	"<filename.osm> ... The name(s) of the file(s) to process (by default\n"
399	" data is read from standard input).\n"
400	"\n"
401	"<transport> defaults to all but can be set to:\n"
402	"%s"
403	"\n"
404	"<highway> can be selected from:\n"
405	"%s"
406	"\n"
407	"<property> can be selected from:\n"
408	"%s",
409	TransportList(),HighwayList(),PropertyList());
410
411	exit(!detail);
412	}