trunk/src/planetsplitter.c

/***************************************
 OSM planet file splitter.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2008-2012 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/time.h>

#include "types.h"
#include "ways.h"

#include "typesx.h"
#include "nodesx.h"
#include "segmentsx.h"
#include "waysx.h"
#include "relationsx.h"
#include "superx.h"
#include "prunex.h"

#include "files.h"
#include "logging.h"
#include "functions.h"
#include "osmparser.h"
#include "tagging.h"
#include "uncompress.h"


/* Global variables */

/*+ The name of the temporary directory. +*/
char *option_tmpdirname=NULL;

/*+ The amount of RAM to use for filesorting. +*/
size_t option_filesort_ramsize=0;

/*+ The number of threads to use for filesorting. +*/
int option_filesort_threads=1;


/* Local functions */

static void print_usage(int detail,const char *argerr,const char *err);


/*++++++++++++++++++++++++++++++++++++++
  The main program for the planetsplitter.
  ++++++++++++++++++++++++++++++++++++++*/

int main(int argc,char** argv)
{
 struct timeval start_time;
 NodesX     *Nodes;
 SegmentsX  *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
 WaysX      *Ways;
 RelationsX *Relations;
 int         iteration=0,quit=0;
 int         max_iterations=5;
 char       *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
 int         option_parse_only=0,option_process_only=0;
 int         option_append=0,option_keep=0,option_changes=0;
 int         option_filenames=0;
 int         option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
 int         arg;

 gettimeofday(&start_time,NULL);

 /* Parse the command line arguments */

 for(arg=1;arg<argc;arg++)
   {
    if(!strcmp(argv[arg],"--help"))
       print_usage(1,NULL,NULL);
    else if(!strncmp(argv[arg],"--dir=",6))
       dirname=&argv[arg][6];
    else if(!strncmp(argv[arg],"--prefix=",9))
       prefix=&argv[arg][9];
    else if(!strncmp(argv[arg],"--sort-ram-size=",16))
       option_filesort_ramsize=atoi(&argv[arg][16]);
#if defined(USE_PTHREADS) && USE_PTHREADS
    else if(!strncmp(argv[arg],"--sort-threads=",15))
       option_filesort_threads=atoi(&argv[arg][15]);
#endif
    else if(!strncmp(argv[arg],"--tmpdir=",9))
       option_tmpdirname=&argv[arg][9];
    else if(!strncmp(argv[arg],"--tagging=",10))
       tagging=&argv[arg][10];
    else if(!strcmp(argv[arg],"--loggable"))
       option_loggable=1;
    else if(!strcmp(argv[arg],"--logtime"))
       option_logtime=1;
    else if(!strcmp(argv[arg],"--errorlog"))
       errorlog="error.log";
    else if(!strncmp(argv[arg],"--errorlog=",11))
       errorlog=&argv[arg][11];
    else if(!strcmp(argv[arg],"--parse-only"))
       option_parse_only=1;
    else if(!strcmp(argv[arg],"--process-only"))
       option_process_only=1;
    else if(!strcmp(argv[arg],"--append"))
       option_append=1;
    else if(!strcmp(argv[arg],"--keep"))
       option_keep=1;
    else if(!strcmp(argv[arg],"--changes"))
       option_changes=1;
    else if(!strncmp(argv[arg],"--max-iterations=",17))
       max_iterations=atoi(&argv[arg][17]);
    else if(!strncmp(argv[arg],"--prune",7))
      {
       if(!strcmp(&argv[arg][7],"-none"))
          option_prune_isolated=option_prune_short=option_prune_straight=0;
       else if(!strncmp(&argv[arg][7],"-isolated=",10))
          option_prune_isolated=atoi(&argv[arg][17]);
       else if(!strncmp(&argv[arg][7],"-short=",7))
          option_prune_short=atoi(&argv[arg][14]);
       else if(!strncmp(&argv[arg][7],"-straight=",10))
          option_prune_straight=atoi(&argv[arg][17]);
       else
          print_usage(0,argv[arg],NULL);
      }
    else if(argv[arg][0]=='-' && argv[arg][1]=='-')
       print_usage(0,argv[arg],NULL);
    else
       option_filenames++;
   }

 /* Check the specified command line options */

 if(option_parse_only && option_process_only)
    print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");

 if(option_append && option_process_only)
    print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");

 if(option_filenames && option_process_only)
    print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");

 if(!option_filenames && !option_process_only)
    print_usage(0,NULL,"File names must be specified unless using '--process-only'");

 if(!option_filesort_ramsize)
   {
#if SLIM
    option_filesort_ramsize=64*1024*1024;
#else
    option_filesort_ramsize=256*1024*1024;
#endif
   }
 else
    option_filesort_ramsize*=1024*1024;

 if(!option_tmpdirname)
   {
    if(!dirname)
       option_tmpdirname=".";
    else
       option_tmpdirname=dirname;
   }

 if(!option_process_only)
   {
    if(tagging)
      {
       if(!ExistsFile(tagging))
         {
          fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
          return(1);
         }
      }
    else
      {
       if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
          tagging=FileName(dirname,prefix,"tagging.xml");
       else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
          tagging=FileName(DATADIR,NULL,"tagging.xml");
       else
         {
          fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
          return(1);
         }
      }

    if(ParseXMLTaggingRules(tagging))
      {
       fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
       return(1);
      }
   }

 /* Create new node, segment, way and relation variables */

 Nodes=NewNodeList(option_append||option_changes,option_process_only);

 Segments=NewSegmentList(option_append||option_changes,option_process_only);

 Ways=NewWayList(option_append||option_changes,option_process_only);

 Relations=NewRelationList(option_append||option_changes,option_process_only);

 /* Create the error log file */

 if(errorlog)
    open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only);

 /* Parse the file */

if(!option_process_only)
  {
   for(arg=1;arg<argc;arg++)
     {
      int fd;
      char *filename,*p;

      if(argv[arg][0]=='-' && argv[arg][1]=='-')
         continue;

      filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);

      fd=ReOpenFile(filename);

      if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
        {
         fd=Uncompress_Bzip2(fd);
         *p=0;
        }

      if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
        {
         fd=Uncompress_Gzip(fd);
         *p=0;
        }

      if(option_changes)
        {
         printf("\nParse OSC Data [%s]\n==============\n\n",filename);
         fflush(stdout);

         if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
           {
            logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
           }
         else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
           {
            if(ParseO5CFile(fd,Nodes,Segments,Ways,Relations))
               exit(EXIT_FAILURE);
           }
         else
           {
            if(ParseOSCFile(fd,Nodes,Segments,Ways,Relations))
               exit(EXIT_FAILURE);
           }
        }
      else
        {
         printf("\nParse OSM Data [%s]\n==============\n\n",filename);
         fflush(stdout);

         if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
           {
            if(ParsePBFFile(fd,Nodes,Segments,Ways,Relations))
               exit(EXIT_FAILURE);
           }
         else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
           {
            if(ParseO5MFile(fd,Nodes,Segments,Ways,Relations))
               exit(EXIT_FAILURE);
           }
         else
           {
            if(ParseOSMFile(fd,Nodes,Segments,Ways,Relations))
               exit(EXIT_FAILURE);
           }
        }

      CloseFile(fd);

      free(filename);
     }

   DeleteXMLTaggingRules();
  }

 FinishNodeList(Nodes);
 FinishSegmentList(Segments);
 FinishWayList(Ways);
 FinishRelationList(Relations);

 if(option_parse_only)
   {
    FreeNodeList(Nodes,1);
    FreeSegmentList(Segments,1);
    FreeWayList(Ways,1);
    FreeRelationList(Relations,1);

    return(0);
   }


 /* Sort the data */

 printf("\nSort OSM Data\n=============\n\n");
 fflush(stdout);

 /* Sort the nodes, segments, ways and relations */

 SortNodeList(Nodes);

 if(option_changes)
    ApplySegmentChanges(Segments);

 SortSegmentList(Segments);

 SortWayList(Ways);

 SortRelationList(Relations);

 /* Process the data */

 printf("\nProcess OSM Data\n================\n\n");
 fflush(stdout);

 /* Extract the way names (must be before using the ways) */

 ExtractWayNames(Ways,option_keep||option_changes);

 /* Remove bad segments (must be after sorting the nodes, segments and ways) */

 RemoveBadSegments(Segments,Nodes,Ways,option_keep||option_changes);

 /* Remove non-highway nodes (must be after removing the bad segments) */

 RemoveNonHighwayNodes(Nodes,Segments,option_keep||option_changes);

 /* Process the route relations and first part of turn relations (must be before compacting the ways) */

 ProcessRouteRelations(Relations,Ways,option_keep||option_changes);

 ProcessTurnRelations1(Relations,Nodes,Ways,option_keep||option_changes);

 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */

 MeasureSegments(Segments,Nodes,Ways);

 /* Index the segments */

 IndexSegments(Segments,Nodes,Ways);

 /* Convert the turn relations from ways into nodes */

 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);

 /* Compact the ways (must be after turn relations 2) */

 CompactWayList(Ways,Segments);

 /* Index the segments */

 IndexSegments(Segments,Nodes,Ways);

 /* Prune unwanted nodes/segments. */

 if(option_prune_straight || option_prune_isolated || option_prune_short)
   {
    printf("\nPrune Unneeded Data\n===================\n\n");
    fflush(stdout);

    StartPruning(Nodes,Segments,Ways);

    if(option_prune_straight)
       PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);

    if(option_prune_isolated)
       PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);

    if(option_prune_short)
       PruneShortSegments(Nodes,Segments,Ways,option_prune_short);

    FinishPruning(Nodes,Segments,Ways);

    /* Remove the pruned nodes and segments and update the indexes */

    RemovePrunedNodes(Nodes,Segments);
    RemovePrunedSegments(Segments,Ways);
    CompactWayList(Ways,Segments);
    RemovePrunedTurnRelations(Relations,Nodes);
    IndexSegments(Segments,Nodes,Ways);
   }

 /* Repeated iteration on Super-Nodes and Super-Segments */

 do
   {
    int nsuper;

    printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
    fflush(stdout);

    if(iteration==0)
      {
       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,Segments,Ways);

       /* Select the super-segments */

       SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);

       nsuper=Segments->number;
      }
    else
      {
       SegmentsX *SuperSegments2;

       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,SuperSegments,Ways);

       /* Select the super-segments */

       SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);

       nsuper=SuperSegments->number;

       FreeSegmentList(SuperSegments,0);

       SuperSegments=SuperSegments2;
      }

    /* Sort the super-segments and remove duplicates */

    DeduplicateSuperSegments(SuperSegments,Ways);

    /* Index the segments */

    IndexSegments(SuperSegments,Nodes,Ways);

    /* Check for end condition */

    if(SuperSegments->number==nsuper)
       quit=1;

    iteration++;

    if(iteration>max_iterations)
       quit=1;
   }
 while(!quit);

 /* Combine the super-segments */

 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
 fflush(stdout);

 /* Merge the super-segments */

 MergedSegments=MergeSuperSegments(Segments,SuperSegments);

 FreeSegmentList(Segments,0);

 FreeSegmentList(SuperSegments,0);

 Segments=MergedSegments;

 /* Re-index the merged segments */

 IndexSegments(Segments,Nodes,Ways);

 /* Cross reference the nodes and segments */

 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
 fflush(stdout);

 /* Sort the nodes and segments geographically */

 SortNodeListGeographically(Nodes);

 SortSegmentListGeographically(Segments,Nodes);

 /* Re-index the segments */

 IndexSegments(Segments,Nodes,Ways);

 /* Sort the turn relations geographically */

 SortTurnRelationListGeographically(Relations,Nodes,Segments);

 /* Output the results */

 printf("\nWrite Out Database Files\n========================\n\n");
 fflush(stdout);

 /* Write out the nodes */

 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);

 FreeNodeList(Nodes,0);

 /* Write out the segments */

 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));

 FreeSegmentList(Segments,0);

 /* Write out the ways */

 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));

 FreeWayList(Ways,0);

 /* Write out the relations */

 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));

 FreeRelationList(Relations,0);

 /* Close the error log file */

 if(errorlog)
    close_errorlog();

 /* Print the total time */

 if(option_logtime)
   {
    printf("\n");
    fprintf_elapsed_time(stdout,&start_time);
    printf("Complete\n");
    fflush(stdout);
   }

 return(0);
}


/*++++++++++++++++++++++++++++++++++++++
  Print out the usage information.

  int detail The level of detail to use - 0 = low, 1 = high.

  const char *argerr The argument that gave the error (if there is one).

  const char *err Other error message (if there is one).
  ++++++++++++++++++++++++++++++++++++++*/

static void print_usage(int detail,const char *argerr,const char *err)
{
 fprintf(stderr,
         "Usage: planetsplitter [--help]\n"
         "                      [--dir=<dirname>] [--prefix=<name>]\n"
#if defined(USE_PTHREADS) && USE_PTHREADS
         "                      [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
#else
         "                      [--sort-ram-size=<size>]\n"
#endif
         "                      [--tmpdir=<dirname>]\n"
         "                      [--tagging=<filename>]\n"
         "                      [--loggable] [--logtime]\n"
         "                      [--errorlog[=<name>]]\n"
         "                      [--parse-only | --process-only]\n"
         "                      [--append] [--keep] [--changes]\n"
         "                      [--max-iterations=<number>]\n"
         "                      [--prune-none]\n"
         "                      [--prune-isolated=<len>]\n"
         "                      [--prune-short=<len>]\n"
         "                      [--prune-straight=<len>]\n"
         "                      [<filename.osm> ... | <filename.osc> ...\n"
         "                       | <filename.pbf> ...\n"
         "                       | <filename.osm> ... | <filename.osc> ..."
#if defined(USE_BZIP2) && USE_BZIP2
         "\n                       | <filename.(osm|osc|o5m|o5c).bz2> ..."
#endif
#if defined(USE_GZIP) && USE_GZIP
         "\n                       | <filename.(osm|osc|o5m|o5c).gz> ..."
#endif
         "]\n");

 if(argerr)
    fprintf(stderr,
            "\n"
            "Error with command line parameter: %s\n",argerr);

 if(err)
    fprintf(stderr,
            "\n"
            "Error: %s\n",err);

 if(detail)
    fprintf(stderr,
            "\n"
            "--help                    Prints this information.\n"
            "\n"
            "--dir=<dirname>           The directory containing the routing database.\n"
            "--prefix=<name>           The filename prefix for the routing database.\n"
            "\n"
            "--sort-ram-size=<size>    The amount of RAM (in MB) to use for data sorting\n"
#if SLIM
            "                          (defaults to 64MB otherwise.)\n"
#else
            "                          (defaults to 256MB otherwise.)\n"
#endif
#if defined(USE_PTHREADS) && USE_PTHREADS
            "--sort-threads=<number>   The number of threads to use for data sorting.\n"
#endif
            "\n"
            "--tmpdir=<dirname>        The directory name for temporary files.\n"
            "                          (defaults to the '--dir' option directory.)\n"
            "\n"
            "--tagging=<filename>      The name of the XML file containing the tagging rules\n"
            "                          (defaults to 'tagging.xml' with '--dir' and\n"
            "                           '--prefix' options or the file installed in\n"
            "                           '" DATADIR "').\n"
            "\n"
            "--loggable                Print progress messages suitable for logging to file.\n"
            "--logtime                 Print the elapsed time for each processing step.\n"
            "--errorlog[=<name>]       Log parsing errors to 'error.log' or the given name\n"
            "                          (the '--dir' and '--prefix' options are applied).\n"
            "\n"
            "--parse-only              Parse the OSM/OSC file(s) and store the results.\n"
            "--process-only            Process the stored results from previous option.\n"
            "--append                  Parse the OSM file(s) and append to existing results.\n"
            "--keep                    Keep the intermediate files after parsing & sorting.\n"
            "--changes                 Parse the data as an OSC file and apply the changes.\n"
            "\n"
            "--max-iterations=<number> The number of iterations for finding super-nodes\n"
            "                          (defaults to 5).\n"
            "\n"
            "--prune-none              Disable the prune options below, they are re-enabled\n"
            "                          by adding them to the command line after this option.\n"
            "--prune-isolated=<len>    Remove access from small disconnected segment groups\n"
            "                          (defaults to removing groups under 500m).\n"
            "--prune-short=<len>       Remove short segments (defaults to removing segments\n"
            "                          up to a maximum length of 5m).\n"
            "--prune-straight=<len>    Remove nodes in almost straight highways (defaults to\n"
            "                          removing nodes up to 3m offset from a straight line).\n"
            "\n"
            "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
            "                          The name(s) of the file(s) to read and parse.\n"
            "                          Filenames ending '.pbf' read as PBF, filenames ending\n"
            "                          '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
#if defined(USE_BZIP2) && USE_BZIP2
            "                          Filenames ending '.bz2' will be bzip2 uncompressed.\n"
#endif
#if defined(USE_GZIP) && USE_GZIP
            "                          Filenames ending '.gz' will be gzip uncompressed.\n"
#endif
            "\n"
            "<transport> defaults to all but can be set to:\n"
            "%s"
            "\n"
            "<highway> can be selected from:\n"
            "%s"
            "\n"
            "<property> can be selected from:\n"
            "%s",
            TransportList(),HighwayList(),PropertyList());

 exit(!detail);
}
1	/***************************************
2	OSM planet file splitter.
3
4	Part of the Routino routing software.
5	****************/ /****************
6	This file Copyright 2008-2012 Andrew M. Bishop
7
8	This program is free software: you can redistribute it and/or modify
9	it under the terms of the GNU Affero General Public License as published by
10	the Free Software Foundation, either version 3 of the License, or
11	(at your option) any later version.
12
13	This program is distributed in the hope that it will be useful,
14	but WITHOUT ANY WARRANTY; without even the implied warranty of
15	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	GNU Affero General Public License for more details.
17
18	You should have received a copy of the GNU Affero General Public License
19	along with this program. If not, see <http://www.gnu.org/licenses/>.
20	***************************************/
21
22
23	#include <stdio.h>
24	#include <stdlib.h>
25	#include <unistd.h>
26	#include <string.h>
27	#include <errno.h>
28	#include <sys/time.h>
29
30	#include "types.h"
31	#include "ways.h"
32
33	#include "typesx.h"
34	#include "nodesx.h"
35	#include "segmentsx.h"
36	#include "waysx.h"
37	#include "relationsx.h"
38	#include "superx.h"
39	#include "prunex.h"
40
41	#include "files.h"
42	#include "logging.h"
43	#include "functions.h"
44	#include "osmparser.h"
45	#include "tagging.h"
46	#include "uncompress.h"
47
48
49	/* Global variables */
50
51	/+ The name of the temporary directory. +/
52	char *option_tmpdirname=NULL;
53
54	/+ The amount of RAM to use for filesorting. +/
55	size_t option_filesort_ramsize=0;
56
57	/+ The number of threads to use for filesorting. +/
58	int option_filesort_threads=1;
59
60
61	/* Local functions */
62
63	static void print_usage(int detail,const char argerr,const char err);
64
65
66	/*++++++++++++++++++++++++++++++++++++++
67	The main program for the planetsplitter.
68	++++++++++++++++++++++++++++++++++++++*/
69
70	int main(int argc,char** argv)
71	{
72	struct timeval start_time;
73	NodesX *Nodes;
74	SegmentsX Segments,SuperSegments=NULL,*MergedSegments=NULL;
75	WaysX *Ways;
76	RelationsX *Relations;
77	int iteration=0,quit=0;
78	int max_iterations=5;
79	char dirname=NULL,prefix=NULL,tagging=NULL,errorlog=NULL;
80	int option_parse_only=0,option_process_only=0;
81	int option_append=0,option_keep=0,option_changes=0;
82	int option_filenames=0;
83	int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
84	int arg;
85
86	gettimeofday(&start_time,NULL);
87
88	/* Parse the command line arguments */
89
90	for(arg=1;arg<argc;arg++)
91	{
92	if(!strcmp(argv[arg],"--help"))
93	print_usage(1,NULL,NULL);
94	else if(!strncmp(argv[arg],"--dir=",6))
95	dirname=&argv[arg][6];
96	else if(!strncmp(argv[arg],"--prefix=",9))
97	prefix=&argv[arg][9];
98	else if(!strncmp(argv[arg],"--sort-ram-size=",16))
99	option_filesort_ramsize=atoi(&argv[arg][16]);
100	#if defined(USE_PTHREADS) && USE_PTHREADS
101	else if(!strncmp(argv[arg],"--sort-threads=",15))
102	option_filesort_threads=atoi(&argv[arg][15]);
103	#endif
104	else if(!strncmp(argv[arg],"--tmpdir=",9))
105	option_tmpdirname=&argv[arg][9];
106	else if(!strncmp(argv[arg],"--tagging=",10))
107	tagging=&argv[arg][10];
108	else if(!strcmp(argv[arg],"--loggable"))
109	option_loggable=1;
110	else if(!strcmp(argv[arg],"--logtime"))
111	option_logtime=1;
112	else if(!strcmp(argv[arg],"--errorlog"))
113	errorlog="error.log";
114	else if(!strncmp(argv[arg],"--errorlog=",11))
115	errorlog=&argv[arg][11];
116	else if(!strcmp(argv[arg],"--parse-only"))
117	option_parse_only=1;
118	else if(!strcmp(argv[arg],"--process-only"))
119	option_process_only=1;
120	else if(!strcmp(argv[arg],"--append"))
121	option_append=1;
122	else if(!strcmp(argv[arg],"--keep"))
123	option_keep=1;
124	else if(!strcmp(argv[arg],"--changes"))
125	option_changes=1;
126	else if(!strncmp(argv[arg],"--max-iterations=",17))
127	max_iterations=atoi(&argv[arg][17]);
128	else if(!strncmp(argv[arg],"--prune",7))
129	{
130	if(!strcmp(&argv[arg][7],"-none"))
131	option_prune_isolated=option_prune_short=option_prune_straight=0;
132	else if(!strncmp(&argv[arg][7],"-isolated=",10))
133	option_prune_isolated=atoi(&argv[arg][17]);
134	else if(!strncmp(&argv[arg][7],"-short=",7))
135	option_prune_short=atoi(&argv[arg][14]);
136	else if(!strncmp(&argv[arg][7],"-straight=",10))
137	option_prune_straight=atoi(&argv[arg][17]);
138	else
139	print_usage(0,argv[arg],NULL);
140	}
141	else if(argv[arg][0]=='-' && argv[arg][1]=='-')
142	print_usage(0,argv[arg],NULL);
143	else
144	option_filenames++;
145	}
146
147	/* Check the specified command line options */
148
149	if(option_parse_only && option_process_only)
150	print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
151
152	if(option_append && option_process_only)
153	print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
154
155	if(option_filenames && option_process_only)
156	print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
157
158	if(!option_filenames && !option_process_only)
159	print_usage(0,NULL,"File names must be specified unless using '--process-only'");
160
161	if(!option_filesort_ramsize)
162	{
163	#if SLIM
164	option_filesort_ramsize=6410241024;
165	#else
166	option_filesort_ramsize=25610241024;
167	#endif
168	}
169	else
170	option_filesort_ramsize=10241024;
171
172	if(!option_tmpdirname)
173	{
174	if(!dirname)
175	option_tmpdirname=".";
176	else
177	option_tmpdirname=dirname;
178	}
179
180	if(!option_process_only)
181	{
182	if(tagging)
183	{
184	if(!ExistsFile(tagging))
185	{
186	fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
187	return(1);
188	}
189	}
190	else
191	{
192	if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
193	tagging=FileName(dirname,prefix,"tagging.xml");
194	else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
195	tagging=FileName(DATADIR,NULL,"tagging.xml");
196	else
197	{
198	fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
199	return(1);
200	}
201	}
202
203	if(ParseXMLTaggingRules(tagging))
204	{
205	fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
206	return(1);
207	}
208	}
209
210	/* Create new node, segment, way and relation variables */
211
212	Nodes=NewNodeList(option_append\|\|option_changes,option_process_only);
213
214	Segments=NewSegmentList(option_append\|\|option_changes,option_process_only);
215
216	Ways=NewWayList(option_append\|\|option_changes,option_process_only);
217
218	Relations=NewRelationList(option_append\|\|option_changes,option_process_only);
219
220	/* Create the error log file */
221
222	if(errorlog)
223	open_errorlog(FileName(dirname,prefix,errorlog),option_append\|\|option_changes\|\|option_process_only);
224
225	/* Parse the file */
226
227	if(!option_process_only)
228	{
229	for(arg=1;arg<argc;arg++)
230	{
231	int fd;
232	char filename,p;
233
234	if(argv[arg][0]=='-' && argv[arg][1]=='-')
235	continue;
236
237	filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
238
239	fd=ReOpenFile(filename);
240
241	if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
242	{
243	fd=Uncompress_Bzip2(fd);
244	*p=0;
245	}
246
247	if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
248	{
249	fd=Uncompress_Gzip(fd);
250	*p=0;
251	}
252
253	if(option_changes)
254	{
255	printf("\nParse OSC Data [%s]\n==============\n\n",filename);
256	fflush(stdout);
257
258	if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
259	{
260	logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
261	}
262	else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
263	{
264	if(ParseO5CFile(fd,Nodes,Segments,Ways,Relations))
265	exit(EXIT_FAILURE);
266	}
267	else
268	{
269	if(ParseOSCFile(fd,Nodes,Segments,Ways,Relations))
270	exit(EXIT_FAILURE);
271	}
272	}
273	else
274	{
275	printf("\nParse OSM Data [%s]\n==============\n\n",filename);
276	fflush(stdout);
277
278	if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
279	{
280	if(ParsePBFFile(fd,Nodes,Segments,Ways,Relations))
281	exit(EXIT_FAILURE);
282	}
283	else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
284	{
285	if(ParseO5MFile(fd,Nodes,Segments,Ways,Relations))
286	exit(EXIT_FAILURE);
287	}
288	else
289	{
290	if(ParseOSMFile(fd,Nodes,Segments,Ways,Relations))
291	exit(EXIT_FAILURE);
292	}
293	}
294
295	CloseFile(fd);
296
297	free(filename);
298	}
299
300	DeleteXMLTaggingRules();
301	}
302
303	FinishNodeList(Nodes);
304	FinishSegmentList(Segments);
305	FinishWayList(Ways);
306	FinishRelationList(Relations);
307
308	if(option_parse_only)
309	{
310	FreeNodeList(Nodes,1);
311	FreeSegmentList(Segments,1);
312	FreeWayList(Ways,1);
313	FreeRelationList(Relations,1);
314
315	return(0);
316	}
317
318
319	/* Sort the data */
320
321	printf("\nSort OSM Data\n=============\n\n");
322	fflush(stdout);
323
324	/* Sort the nodes, segments, ways and relations */
325
326	SortNodeList(Nodes);
327
328	if(option_changes)
329	ApplySegmentChanges(Segments);
330
331	SortSegmentList(Segments);
332
333	SortWayList(Ways);
334
335	SortRelationList(Relations);
336
337	/* Process the data */
338
339	printf("\nProcess OSM Data\n================\n\n");
340	fflush(stdout);
341
342	/* Extract the way names (must be before using the ways) */
343
344	ExtractWayNames(Ways,option_keep\|\|option_changes);
345
346	/* Remove bad segments (must be after sorting the nodes, segments and ways) */
347
348	RemoveBadSegments(Segments,Nodes,Ways,option_keep\|\|option_changes);
349
350	/* Remove non-highway nodes (must be after removing the bad segments) */
351
352	RemoveNonHighwayNodes(Nodes,Segments,option_keep\|\|option_changes);
353
354	/* Process the route relations and first part of turn relations (must be before compacting the ways) */
355
356	ProcessRouteRelations(Relations,Ways,option_keep\|\|option_changes);
357
358	ProcessTurnRelations1(Relations,Nodes,Ways,option_keep\|\|option_changes);
359
360	/* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
361
362	MeasureSegments(Segments,Nodes,Ways);
363
364	/* Index the segments */
365
366	IndexSegments(Segments,Nodes,Ways);
367
368	/* Convert the turn relations from ways into nodes */
369
370	ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
371
372	/* Compact the ways (must be after turn relations 2) */
373
374	CompactWayList(Ways,Segments);
375
376	/* Index the segments */
377
378	IndexSegments(Segments,Nodes,Ways);
379
380	/* Prune unwanted nodes/segments. */
381
382	if(option_prune_straight \|\| option_prune_isolated \|\| option_prune_short)
383	{
384	printf("\nPrune Unneeded Data\n===================\n\n");
385	fflush(stdout);
386
387	StartPruning(Nodes,Segments,Ways);
388
389	if(option_prune_straight)
390	PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
391
392	if(option_prune_isolated)
393	PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
394
395	if(option_prune_short)
396	PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
397
398	FinishPruning(Nodes,Segments,Ways);
399
400	/* Remove the pruned nodes and segments and update the indexes */
401
402	RemovePrunedNodes(Nodes,Segments);
403	RemovePrunedSegments(Segments,Ways);
404	CompactWayList(Ways,Segments);
405	RemovePrunedTurnRelations(Relations,Nodes);
406	IndexSegments(Segments,Nodes,Ways);
407	}
408
409	/* Repeated iteration on Super-Nodes and Super-Segments */
410
411	do
412	{
413	int nsuper;
414
415	printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
416	fflush(stdout);
417
418	if(iteration==0)
419	{
420	/* Select the super-nodes */
421
422	ChooseSuperNodes(Nodes,Segments,Ways);
423
424	/* Select the super-segments */
425
426	SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
427
428	nsuper=Segments->number;
429	}
430	else
431	{
432	SegmentsX *SuperSegments2;
433
434	/* Select the super-nodes */
435
436	ChooseSuperNodes(Nodes,SuperSegments,Ways);
437
438	/* Select the super-segments */
439
440	SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
441
442	nsuper=SuperSegments->number;
443
444	FreeSegmentList(SuperSegments,0);
445
446	SuperSegments=SuperSegments2;
447	}
448
449	/* Sort the super-segments and remove duplicates */
450
451	DeduplicateSuperSegments(SuperSegments,Ways);
452
453	/* Index the segments */
454
455	IndexSegments(SuperSegments,Nodes,Ways);
456
457	/* Check for end condition */
458
459	if(SuperSegments->number==nsuper)
460	quit=1;
461
462	iteration++;
463
464	if(iteration>max_iterations)
465	quit=1;
466	}
467	while(!quit);
468
469	/* Combine the super-segments */
470
471	printf("\nCombine Segments and Super-Segments\n===================================\n\n");
472	fflush(stdout);
473
474	/* Merge the super-segments */
475
476	MergedSegments=MergeSuperSegments(Segments,SuperSegments);
477
478	FreeSegmentList(Segments,0);
479
480	FreeSegmentList(SuperSegments,0);
481
482	Segments=MergedSegments;
483
484	/* Re-index the merged segments */
485
486	IndexSegments(Segments,Nodes,Ways);
487
488	/* Cross reference the nodes and segments */
489
490	printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
491	fflush(stdout);
492
493	/* Sort the nodes and segments geographically */
494
495	SortNodeListGeographically(Nodes);
496
497	SortSegmentListGeographically(Segments,Nodes);
498
499	/* Re-index the segments */
500
501	IndexSegments(Segments,Nodes,Ways);
502
503	/* Sort the turn relations geographically */
504
505	SortTurnRelationListGeographically(Relations,Nodes,Segments);
506
507	/* Output the results */
508
509	printf("\nWrite Out Database Files\n========================\n\n");
510	fflush(stdout);
511
512	/* Write out the nodes */
513
514	SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
515
516	FreeNodeList(Nodes,0);
517
518	/* Write out the segments */
519
520	SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
521
522	FreeSegmentList(Segments,0);
523
524	/* Write out the ways */
525
526	SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
527
528	FreeWayList(Ways,0);
529
530	/* Write out the relations */
531
532	SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
533
534	FreeRelationList(Relations,0);
535
536	/* Close the error log file */
537
538	if(errorlog)
539	close_errorlog();
540
541	/* Print the total time */
542
543	if(option_logtime)
544	{
545	printf("\n");
546	fprintf_elapsed_time(stdout,&start_time);
547	printf("Complete\n");
548	fflush(stdout);
549	}
550
551	return(0);
552	}
553
554
555	/*++++++++++++++++++++++++++++++++++++++
556	Print out the usage information.
557
558	int detail The level of detail to use - 0 = low, 1 = high.
559
560	const char *argerr The argument that gave the error (if there is one).
561
562	const char *err Other error message (if there is one).
563	++++++++++++++++++++++++++++++++++++++*/
564
565	static void print_usage(int detail,const char argerr,const char err)
566	{
567	fprintf(stderr,
568	"Usage: planetsplitter [--help]\n"
569	" [--dir=<dirname>] [--prefix=<name>]\n"
570	#if defined(USE_PTHREADS) && USE_PTHREADS
571	" [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
572	#else
573	" [--sort-ram-size=<size>]\n"
574	#endif
575	" [--tmpdir=<dirname>]\n"
576	" [--tagging=<filename>]\n"
577	" [--loggable] [--logtime]\n"
578	" [--errorlog[=<name>]]\n"
579	" [--parse-only \| --process-only]\n"
580	" [--append] [--keep] [--changes]\n"
581	" [--max-iterations=<number>]\n"
582	" [--prune-none]\n"
583	" [--prune-isolated=<len>]\n"
584	" [--prune-short=<len>]\n"
585	" [--prune-straight=<len>]\n"
586	" [<filename.osm> ... \| <filename.osc> ...\n"
587	" \| <filename.pbf> ...\n"
588	" \| <filename.osm> ... \| <filename.osc> ..."
589	#if defined(USE_BZIP2) && USE_BZIP2
590	"\n \| <filename.(osm\|osc\|o5m\|o5c).bz2> ..."
591	#endif
592	#if defined(USE_GZIP) && USE_GZIP
593	"\n \| <filename.(osm\|osc\|o5m\|o5c).gz> ..."
594	#endif
595	"]\n");
596
597	if(argerr)
598	fprintf(stderr,
599	"\n"
600	"Error with command line parameter: %s\n",argerr);
601
602	if(err)
603	fprintf(stderr,
604	"\n"
605	"Error: %s\n",err);
606
607	if(detail)
608	fprintf(stderr,
609	"\n"
610	"--help Prints this information.\n"
611	"\n"
612	"--dir=<dirname> The directory containing the routing database.\n"
613	"--prefix=<name> The filename prefix for the routing database.\n"
614	"\n"
615	"--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
616	#if SLIM
617	" (defaults to 64MB otherwise.)\n"
618	#else
619	" (defaults to 256MB otherwise.)\n"
620	#endif
621	#if defined(USE_PTHREADS) && USE_PTHREADS
622	"--sort-threads=<number> The number of threads to use for data sorting.\n"
623	#endif
624	"\n"
625	"--tmpdir=<dirname> The directory name for temporary files.\n"
626	" (defaults to the '--dir' option directory.)\n"
627	"\n"
628	"--tagging=<filename> The name of the XML file containing the tagging rules\n"
629	" (defaults to 'tagging.xml' with '--dir' and\n"
630	" '--prefix' options or the file installed in\n"
631	" '" DATADIR "').\n"
632	"\n"
633	"--loggable Print progress messages suitable for logging to file.\n"
634	"--logtime Print the elapsed time for each processing step.\n"
635	"--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
636	" (the '--dir' and '--prefix' options are applied).\n"
637	"\n"
638	"--parse-only Parse the OSM/OSC file(s) and store the results.\n"
639	"--process-only Process the stored results from previous option.\n"
640	"--append Parse the OSM file(s) and append to existing results.\n"
641	"--keep Keep the intermediate files after parsing & sorting.\n"
642	"--changes Parse the data as an OSC file and apply the changes.\n"
643	"\n"
644	"--max-iterations=<number> The number of iterations for finding super-nodes\n"
645	" (defaults to 5).\n"
646	"\n"
647	"--prune-none Disable the prune options below, they are re-enabled\n"
648	" by adding them to the command line after this option.\n"
649	"--prune-isolated=<len> Remove access from small disconnected segment groups\n"
650	" (defaults to removing groups under 500m).\n"
651	"--prune-short=<len> Remove short segments (defaults to removing segments\n"
652	" up to a maximum length of 5m).\n"
653	"--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
654	" removing nodes up to 3m offset from a straight line).\n"
655	"\n"
656	"<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
657	" The name(s) of the file(s) to read and parse.\n"
658	" Filenames ending '.pbf' read as PBF, filenames ending\n"
659	" '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
660	#if defined(USE_BZIP2) && USE_BZIP2
661	" Filenames ending '.bz2' will be bzip2 uncompressed.\n"
662	#endif
663	#if defined(USE_GZIP) && USE_GZIP
664	" Filenames ending '.gz' will be gzip uncompressed.\n"
665	#endif
666	"\n"
667	"<transport> defaults to all but can be set to:\n"
668	"%s"
669	"\n"
670	"<highway> can be selected from:\n"
671	"%s"
672	"\n"
673	"<property> can be selected from:\n"
674	"%s",
675	TransportList(),HighwayList(),PropertyList());
676
677	exit(!detail);
678	}