Repository: osmlab/atlas Branch: dev Commit: bf5cb903df88 Files: 2119 Total size: 47.2 MB Directory structure: gitextract_1k5scvdv/ ├── .circleci/ │ ├── config.yml │ └── sonar.sh ├── .github/ │ ├── PULL_REQUEST_TEMPLATE.md │ ├── workflow_data/ │ │ └── secret.gpg.aes256 │ ├── workflow_scripts/ │ │ ├── decrypt_gpg_key.sh │ │ ├── deploy.sh │ │ ├── merge-dev-to-main.sh │ │ ├── sonar.sh │ │ ├── tag-main.sh │ │ └── update_project_version.sh │ └── workflows/ │ └── ci.yml ├── .gitignore ├── .travis/ │ ├── build-pyatlas-gate.sh │ ├── build.sh │ ├── deploy-gate.sh │ ├── deploy.sh │ ├── install.sh │ ├── merge-dev-to-main-gate.sh │ ├── merge-dev-to-main.sh │ ├── secring.gpg.enc │ ├── sonar-gate.sh │ ├── sonar.sh │ ├── tag-main-gate.sh │ ├── tag-main.sh │ └── trigger-release.sh ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── atlas-shell-tools/ │ ├── .atlas-shell-tools-integrity-file │ ├── README.md │ ├── ast_completions.bash │ ├── ast_completions.zsh │ ├── man/ │ │ ├── man1/ │ │ │ ├── atlas-config-activate.1 │ │ │ ├── atlas-config-deactivate.1 │ │ │ ├── atlas-config-install.1 │ │ │ ├── atlas-config-list.1 │ │ │ ├── atlas-config-log.1 │ │ │ ├── atlas-config-preset.1 │ │ │ ├── atlas-config-repo.1 │ │ │ ├── atlas-config-reset.1 │ │ │ ├── atlas-config-sync.1 │ │ │ ├── atlas-config-uninstall.1 │ │ │ ├── atlas-config-update.1 │ │ │ ├── atlas-config.1 │ │ │ └── atlas.1 │ │ ├── man5/ │ │ │ └── atlas-plumbing.5 │ │ └── man7/ │ │ ├── atlas-cli.7 │ │ ├── atlas-cookbook.7 │ │ ├── atlas-environment.7 │ │ ├── atlas-glossary.7 │ │ ├── atlas-presets.7 │ │ └── atlas-shell-tools.7 │ ├── quick_install_bash.sh │ ├── quick_install_zsh.sh │ └── scripts/ │ ├── atlas │ ├── atlas-config │ └── common/ │ ├── ast_completions.pm │ ├── ast_log_subsystem.pm │ ├── ast_module_subsystem.pm │ ├── ast_preset_subsystem.pm │ ├── ast_repo_subsystem.pm │ ├── ast_tty.pm │ └── ast_utilities.pm ├── build.gradle ├── config/ │ ├── checkstyle/ │ │ ├── arrangement.txt │ │ ├── checkstyle.xml │ │ └── suppressions.xml │ ├── format/ │ │ └── code_format.xml │ └── log4j/ │ └── log4j.properties ├── dependencies.gradle ├── gradle.properties ├── pyatlas/ │ ├── README.md │ ├── clean.sh │ ├── doc/ │ │ └── how_to_get_the_docs.txt │ ├── format.sh │ ├── package.sh │ ├── pyatlas/ │ │ ├── __init__.py │ │ ├── atlas.py │ │ ├── atlas_entities.py │ │ ├── atlas_metadata.py │ │ ├── autogen/ │ │ │ └── __init__.py │ │ ├── geometry.py │ │ ├── identifier_converters.py │ │ ├── pyatlas_globalfunc.py │ │ └── spatial_index.py │ ├── resources/ │ │ ├── CreateTestAtlas.java │ │ └── test.atlas │ ├── setup.py │ ├── style.yapf │ ├── test.sh │ ├── unit_tests/ │ │ ├── test_atlas.py │ │ ├── test_identifier_converters.py │ │ ├── test_location.py │ │ ├── test_polygon_converters.py │ │ ├── test_polyline_polygon.py │ │ ├── test_rectangle.py │ │ └── test_spatial_index.py │ ├── venv.sh │ └── yapf_format.py ├── sample/ │ ├── Readme.md │ ├── build.gradle │ ├── settings.gradle │ └── src/ │ └── main/ │ ├── java/ │ │ └── org/ │ │ └── openstreetmap/ │ │ └── atlas/ │ │ └── sample/ │ │ └── Sample.java │ └── resources/ │ └── log4j.properties ├── scripts/ │ └── log4j-atlas/ │ └── log4j.properties ├── settings.gradle └── src/ ├── integrationTest/ │ ├── java/ │ │ └── org/ │ │ └── openstreetmap/ │ │ └── atlas/ │ │ └── geography/ │ │ ├── PolygonPerformanceTest.java │ │ ├── atlas/ │ │ │ ├── AtlasIntegrationTest.java │ │ │ ├── SubAtlasIntegrationTest.java │ │ │ ├── builder/ │ │ │ │ └── text/ │ │ │ │ └── TextAtlasBuilderIntegrationTest.java │ │ │ ├── delta/ │ │ │ │ ├── AtlasDeltaGeoJsonIntegrationTest.java │ │ │ │ └── AtlasDeltaIntegrationTest.java │ │ │ ├── dynamic/ │ │ │ │ └── DynamicAtlasIntegrationTest.java │ │ │ ├── items/ │ │ │ │ ├── AtlasEntityTypeTest.java │ │ │ │ └── complex/ │ │ │ │ └── boundaries/ │ │ │ │ ├── ComplexBoundariesIntegrationTest.java │ │ │ │ └── ComplexBoundaryIntegrationTestRule.java │ │ │ ├── multi/ │ │ │ │ ├── MultiAtlasIntegrationTest.java │ │ │ │ └── MultiAtlasIntegrationTestRule.java │ │ │ ├── packed/ │ │ │ │ ├── PackedAtlasClonerIntegrationTest.java │ │ │ │ └── PackedAtlasIntegrationTest.java │ │ │ ├── pbf/ │ │ │ │ ├── OsmPbfIngestIntegrationTest.java │ │ │ │ └── slicing/ │ │ │ │ └── AtlasSectionProcessorIntegrationTest.java │ │ │ ├── raw/ │ │ │ │ ├── DynamicRawAtlasSectioningTestRule.java │ │ │ │ └── RawAtlasIntegrationTest.java │ │ │ └── routing/ │ │ │ └── AtlasRoutingIntegrationTest.java │ │ └── boundary/ │ │ ├── CountryBoundaryMapArchiverIntegrationTest.java │ │ └── CountryBoundaryMapIntegrationTest.java │ └── resources/ │ └── org/ │ └── openstreetmap/ │ └── atlas/ │ └── geography/ │ ├── atlas/ │ │ ├── pbf/ │ │ │ ├── BHS-6-18-27.pbf │ │ │ ├── BLZ_raw_08242015.osm.pbf │ │ │ └── CUB_72-111.pbf │ │ └── raw/ │ │ ├── 8-122-122-trimmed.osm.pbf │ │ ├── layerIntersectionAtEndBoundaryMap.txt │ │ ├── layerIntersectionAtStartBoundaryMap.txt │ │ ├── layerIntersectionInMiddleBoundaryMap.txt │ │ ├── node-4353689487.pbf │ │ ├── twoWaysWithDifferentLayersIntersectingAtEnd.pbf │ │ ├── twoWaysWithDifferentLayersIntersectingAtStart.pbf │ │ └── twoWaysWithDifferentLayersIntersectingInMiddle.pbf │ └── boundary/ │ └── oceanTestBoundary.txt ├── main/ │ ├── java/ │ │ └── org/ │ │ └── openstreetmap/ │ │ └── atlas/ │ │ ├── event/ │ │ │ ├── Event.java │ │ │ ├── EventService.java │ │ │ ├── EventServiceable.java │ │ │ ├── Processor.java │ │ │ ├── README.md │ │ │ └── ShutdownEvent.java │ │ ├── exception/ │ │ │ ├── CoreException.java │ │ │ ├── ExceptionSearch.java │ │ │ ├── LoadAtlasFromResourceException.java │ │ │ └── change/ │ │ │ ├── FeatureChangeMergeException.java │ │ │ └── MergeFailureType.java │ │ ├── geography/ │ │ │ ├── Altitude.java │ │ │ ├── CompressedPolyLine.java │ │ │ ├── CompressedPolygon.java │ │ │ ├── GeometricObject.java │ │ │ ├── GeometricSurface.java │ │ │ ├── GeometryPrintable.java │ │ │ ├── Heading.java │ │ │ ├── Latitude.java │ │ │ ├── Located.java │ │ │ ├── Location.java │ │ │ ├── Longitude.java │ │ │ ├── MultiPolyLine.java │ │ │ ├── MultiPolygon.java │ │ │ ├── PolyLine.java │ │ │ ├── Polygon.java │ │ │ ├── README.md │ │ │ ├── Rectangle.java │ │ │ ├── Segment.java │ │ │ ├── Snapper.java │ │ │ ├── StringCompressedPolyLine.java │ │ │ ├── StringCompressedPolygon.java │ │ │ ├── WkbPrintable.java │ │ │ ├── WktPrintable.java │ │ │ ├── atlas/ │ │ │ │ ├── AbstractAtlas.java │ │ │ │ ├── Atlas.java │ │ │ │ ├── AtlasLoadingCommand.java │ │ │ │ ├── AtlasMetaData.java │ │ │ │ ├── AtlasResourceLoader.java │ │ │ │ ├── BareAtlas.java │ │ │ │ ├── Crawler.java │ │ │ │ ├── README.md │ │ │ │ ├── ShardFileOverlapsPolygon.java │ │ │ │ ├── builder/ │ │ │ │ │ ├── AtlasBuilder.java │ │ │ │ │ ├── AtlasSize.java │ │ │ │ │ ├── GeoJsonAtlasBuilder.java │ │ │ │ │ ├── RelationBean.java │ │ │ │ │ ├── store/ │ │ │ │ │ │ ├── AtlasPrimitiveArea.java │ │ │ │ │ │ ├── AtlasPrimitiveBigNode.java │ │ │ │ │ │ ├── AtlasPrimitiveEdge.java │ │ │ │ │ │ ├── AtlasPrimitiveEdgeIdentifier.java │ │ │ │ │ │ ├── AtlasPrimitiveEntity.java │ │ │ │ │ │ ├── AtlasPrimitiveLineItem.java │ │ │ │ │ │ ├── AtlasPrimitiveLocationItem.java │ │ │ │ │ │ ├── AtlasPrimitiveObjectStore.java │ │ │ │ │ │ ├── AtlasPrimitiveRelation.java │ │ │ │ │ │ ├── AtlasPrimitiveRoute.java │ │ │ │ │ │ ├── AtlasPrimitiveRouteIdentifier.java │ │ │ │ │ │ └── TemporaryObjectStore.java │ │ │ │ │ └── text/ │ │ │ │ │ └── TextAtlasBuilder.java │ │ │ │ ├── change/ │ │ │ │ │ ├── AtlasChangeGenerator.java │ │ │ │ │ ├── AtlasEntityKey.java │ │ │ │ │ ├── Change.java │ │ │ │ │ ├── ChangeArea.java │ │ │ │ │ ├── ChangeAtlas.java │ │ │ │ │ ├── ChangeBuilder.java │ │ │ │ │ ├── ChangeEdge.java │ │ │ │ │ ├── ChangeEntity.java │ │ │ │ │ ├── ChangeLine.java │ │ │ │ │ ├── ChangeNode.java │ │ │ │ │ ├── ChangePoint.java │ │ │ │ │ ├── ChangeRelation.java │ │ │ │ │ ├── ChangeType.java │ │ │ │ │ ├── FeatureChange.java │ │ │ │ │ ├── FeatureChangeBoundsExpander.java │ │ │ │ │ ├── FeatureChangeMergeGroup.java │ │ │ │ │ ├── FeatureChangeMergingHelpers.java │ │ │ │ │ ├── MemberMergeStrategies.java │ │ │ │ │ ├── MemberMerger.java │ │ │ │ │ ├── description/ │ │ │ │ │ │ ├── ChangeDescription.java │ │ │ │ │ │ ├── ChangeDescriptorGenerator.java │ │ │ │ │ │ ├── ChangeDescriptorType.java │ │ │ │ │ │ └── descriptors/ │ │ │ │ │ │ ├── ChangeDescriptor.java │ │ │ │ │ │ ├── ChangeDescriptorComparator.java │ │ │ │ │ │ ├── ChangeDescriptorName.java │ │ │ │ │ │ ├── GeometricRelationGeometryChangeDescriptor.java │ │ │ │ │ │ ├── GeometryChangeDescriptor.java │ │ │ │ │ │ ├── LongElementChangeDescriptor.java │ │ │ │ │ │ ├── RelationMemberChangeDescriptor.java │ │ │ │ │ │ └── TagChangeDescriptor.java │ │ │ │ │ ├── diff/ │ │ │ │ │ │ ├── AtlasDiff.java │ │ │ │ │ │ └── AtlasDiffHelper.java │ │ │ │ │ ├── eventhandling/ │ │ │ │ │ │ ├── event/ │ │ │ │ │ │ │ ├── EntityChangeEvent.java │ │ │ │ │ │ │ ├── TagChangeEvent.java │ │ │ │ │ │ │ └── consts/ │ │ │ │ │ │ │ └── FieldChangeOperation.java │ │ │ │ │ │ ├── listenable/ │ │ │ │ │ │ │ ├── EntityChangeListenable.java │ │ │ │ │ │ │ └── TagChangeListenable.java │ │ │ │ │ │ └── listener/ │ │ │ │ │ │ ├── EntityChangeListener.java │ │ │ │ │ │ └── TagChangeListener.java │ │ │ │ │ ├── exception/ │ │ │ │ │ │ └── EmptyChangeException.java │ │ │ │ │ ├── serializer/ │ │ │ │ │ │ ├── ChangeGeoJsonSerializer.java │ │ │ │ │ │ └── FeatureChangeGeoJsonSerializer.java │ │ │ │ │ ├── testing/ │ │ │ │ │ │ ├── AtlasChangeGeneratorAddTurnRestrictions.java │ │ │ │ │ │ ├── AtlasChangeGeneratorRemoveReverseEdges.java │ │ │ │ │ │ └── AtlasChangeGeneratorSplitRoundabout.java │ │ │ │ │ └── validators/ │ │ │ │ │ └── ChangeValidator.java │ │ │ │ ├── changeset/ │ │ │ │ │ ├── BinaryChangeSetDeserializer.java │ │ │ │ │ ├── BinaryChangeSetSerializer.java │ │ │ │ │ ├── ChangeAction.java │ │ │ │ │ ├── ChangeItem.java │ │ │ │ │ ├── ChangeItemMember.java │ │ │ │ │ ├── ChangeSet.java │ │ │ │ │ ├── ChangeSetAtlasBuilder.java │ │ │ │ │ ├── ChangeSetDeserializer.java │ │ │ │ │ ├── ChangeSetSerializer.java │ │ │ │ │ ├── GeoJSONChangeSetSerializer.java │ │ │ │ │ ├── MutableChangeItem.java │ │ │ │ │ ├── SimpleChangeItem.java │ │ │ │ │ ├── SimpleChangeItemMember.java │ │ │ │ │ └── SimpleChangeSet.java │ │ │ │ ├── command/ │ │ │ │ │ ├── AbstractAtlasOutputTestSubCommand.java │ │ │ │ │ ├── AbstractAtlasSubCommand.java │ │ │ │ │ ├── AtlasCommandConstants.java │ │ │ │ │ ├── AtlasCountriesSubCommand.java │ │ │ │ │ ├── AtlasFeatureCountsSubCommand.java │ │ │ │ │ ├── AtlasFindByAtlasIdentifierSubCommand.java │ │ │ │ │ ├── AtlasFindByFeatureIdentifierLocatorSubCommand.java │ │ │ │ │ ├── AtlasFindEntitiesByIdSubCommand.java │ │ │ │ │ ├── AtlasGeoJSONSubCommand.java │ │ │ │ │ ├── AtlasItemsWithSharedShapepointsSubCommand.java │ │ │ │ │ ├── AtlasJoinerSubCommand.java │ │ │ │ │ ├── AtlasListRestrictedPathsCommand.java │ │ │ │ │ ├── AtlasListValidTurnRestrictionIds.java │ │ │ │ │ ├── AtlasMetadataSubCommand.java │ │ │ │ │ ├── AtlasMissingISOSubCommand.java │ │ │ │ │ ├── AtlasReader.java │ │ │ │ │ ├── AtlasResourceLoaderErrorSubCommand.java │ │ │ │ │ ├── AtlasSplitterWithSlippyTileCommand.java │ │ │ │ │ ├── FerrySearchSubCommand.java │ │ │ │ │ ├── OsmPbfToAtlasSubCommand.java │ │ │ │ │ ├── PackedToTextAtlasSubCommand.java │ │ │ │ │ ├── SubAtlasSubCommand.java │ │ │ │ │ ├── TextToPackedAtlasSubCommand.java │ │ │ │ │ └── buildings/ │ │ │ │ │ ├── AtlasFindBuildingPartsSubCommand.java │ │ │ │ │ ├── BuildingsWithHeightSearchSubCommand.java │ │ │ │ │ └── TinyBuildingsSearchSubCommand.java │ │ │ │ ├── complete/ │ │ │ │ │ ├── CompleteArea.java │ │ │ │ │ ├── CompleteEdge.java │ │ │ │ │ ├── CompleteEntity.java │ │ │ │ │ ├── CompleteItemType.java │ │ │ │ │ ├── CompleteLine.java │ │ │ │ │ ├── CompleteLineItem.java │ │ │ │ │ ├── CompleteLocationItem.java │ │ │ │ │ ├── CompleteNode.java │ │ │ │ │ ├── CompletePoint.java │ │ │ │ │ ├── CompleteRelation.java │ │ │ │ │ ├── EmptyAtlas.java │ │ │ │ │ ├── PrettifyStringFormat.java │ │ │ │ │ └── TagChangeDelegate.java │ │ │ │ ├── converters/ │ │ │ │ │ └── AtlasDebugTool.java │ │ │ │ ├── delta/ │ │ │ │ │ ├── AtlasDelta.java │ │ │ │ │ ├── AtlasDeltaGenerator.java │ │ │ │ │ ├── Diff.java │ │ │ │ │ └── README.md │ │ │ │ ├── dynamic/ │ │ │ │ │ ├── DynamicArea.java │ │ │ │ │ ├── DynamicAtlas.java │ │ │ │ │ ├── DynamicAtlasExpander.java │ │ │ │ │ ├── DynamicEdge.java │ │ │ │ │ ├── DynamicLine.java │ │ │ │ │ ├── DynamicNode.java │ │ │ │ │ ├── DynamicPoint.java │ │ │ │ │ ├── DynamicRelation.java │ │ │ │ │ ├── README.md │ │ │ │ │ └── policy/ │ │ │ │ │ ├── DynamicAtlasPolicy.java │ │ │ │ │ └── DynamicAtlasResourcePolicy.java │ │ │ │ ├── exception/ │ │ │ │ │ └── AtlasIntegrityException.java │ │ │ │ ├── geojson/ │ │ │ │ │ ├── AtlasGeoJsonConverter.java │ │ │ │ │ └── LineDelimitedGeoJsonConverter.java │ │ │ │ ├── inspection/ │ │ │ │ │ └── EntityClassifier.java │ │ │ │ ├── items/ │ │ │ │ │ ├── Area.java │ │ │ │ │ ├── AtlasEntity.java │ │ │ │ │ ├── AtlasItem.java │ │ │ │ │ ├── AtlasObject.java │ │ │ │ │ ├── ConnectedEdgeType.java │ │ │ │ │ ├── ConnectedEntityType.java │ │ │ │ │ ├── ConnectedNodeType.java │ │ │ │ │ ├── DiffViewFriendlyItem.java │ │ │ │ │ ├── DirectionalizedEdge.java │ │ │ │ │ ├── Edge.java │ │ │ │ │ ├── ItemType.java │ │ │ │ │ ├── Line.java │ │ │ │ │ ├── LineItem.java │ │ │ │ │ ├── LocationItem.java │ │ │ │ │ ├── Node.java │ │ │ │ │ ├── Point.java │ │ │ │ │ ├── README.md │ │ │ │ │ ├── Relation.java │ │ │ │ │ ├── RelationMember.java │ │ │ │ │ ├── RelationMemberList.java │ │ │ │ │ ├── Route.java │ │ │ │ │ ├── SnappedEdge.java │ │ │ │ │ ├── SnappedLineItem.java │ │ │ │ │ ├── TurnRestriction.java │ │ │ │ │ └── complex/ │ │ │ │ │ ├── ComplexEntity.java │ │ │ │ │ ├── Finder.java │ │ │ │ │ ├── MultiPolygonRelationToMemberConverter.java │ │ │ │ │ ├── README.md │ │ │ │ │ ├── RelationOrAreaToMultiPolygonConverter.java │ │ │ │ │ ├── RelationToMultiPolygonMemberConverter.java │ │ │ │ │ ├── WaterIslandConfigurationReader.java │ │ │ │ │ ├── aoi/ │ │ │ │ │ │ ├── ComplexAreaOfInterest.java │ │ │ │ │ │ └── ComplexAreaOfInterestFinder.java │ │ │ │ │ ├── bignode/ │ │ │ │ │ │ ├── BigNode.java │ │ │ │ │ │ ├── BigNodeFinder.java │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── RestrictedPath.java │ │ │ │ │ │ └── converters/ │ │ │ │ │ │ ├── AtlasBigNodeRestrictedPathToGeoJsonConverter.java │ │ │ │ │ │ └── AtlasBigNodesToGeoJsonConverter.java │ │ │ │ │ ├── boundaries/ │ │ │ │ │ │ ├── ComplexBoundary.java │ │ │ │ │ │ ├── ComplexBoundaryFinder.java │ │ │ │ │ │ └── converters/ │ │ │ │ │ │ └── ComplexBoundaryIterableToGeoJsonWriter.java │ │ │ │ │ ├── buildings/ │ │ │ │ │ │ ├── BuildingPart.java │ │ │ │ │ │ ├── ComplexBuilding.java │ │ │ │ │ │ ├── ComplexBuildingFinder.java │ │ │ │ │ │ ├── HeightConverter.java │ │ │ │ │ │ └── converters/ │ │ │ │ │ │ └── ComplexBuildingToGeojsonConverter.java │ │ │ │ │ ├── highwayarea/ │ │ │ │ │ │ ├── ComplexHighwayArea.java │ │ │ │ │ │ ├── ComplexHighwayAreaFinder.java │ │ │ │ │ │ └── ComplexHighwayAreaHelper.java │ │ │ │ │ ├── islands/ │ │ │ │ │ │ ├── ComplexIsland.java │ │ │ │ │ │ ├── ComplexIslandFinder.java │ │ │ │ │ │ └── DefaultIslandConfigurationReader.java │ │ │ │ │ ├── landcover/ │ │ │ │ │ │ ├── ComplexLandCover.java │ │ │ │ │ │ └── ComplexLandCoverFinder.java │ │ │ │ │ ├── restriction/ │ │ │ │ │ │ ├── ComplexTurnRestriction.java │ │ │ │ │ │ ├── ComplexTurnRestrictionFinder.java │ │ │ │ │ │ └── converters/ │ │ │ │ │ │ └── AtlasTurnRestrictionsToGeoJsonConverter.java │ │ │ │ │ ├── roundabout/ │ │ │ │ │ │ ├── ComplexRoundabout.java │ │ │ │ │ │ └── ComplexRoundaboutFinder.java │ │ │ │ │ └── water/ │ │ │ │ │ ├── ComplexWaterEntity.java │ │ │ │ │ ├── ComplexWaterbody.java │ │ │ │ │ ├── ComplexWaterway.java │ │ │ │ │ ├── WaterType.java │ │ │ │ │ └── finder/ │ │ │ │ │ ├── ComplexWaterEntityFinder.java │ │ │ │ │ ├── DefaultWaterConfigurationReader.java │ │ │ │ │ └── WaterConfigurationReader.java │ │ │ │ ├── lightweight/ │ │ │ │ │ ├── LightArea.java │ │ │ │ │ ├── LightEdge.java │ │ │ │ │ ├── LightEntity.java │ │ │ │ │ ├── LightLine.java │ │ │ │ │ ├── LightLineItem.java │ │ │ │ │ ├── LightLocationItem.java │ │ │ │ │ ├── LightNode.java │ │ │ │ │ ├── LightPoint.java │ │ │ │ │ └── LightRelation.java │ │ │ │ ├── multi/ │ │ │ │ │ ├── MultiArea.java │ │ │ │ │ ├── MultiAtlas.java │ │ │ │ │ ├── MultiAtlasLoaderCommand.java │ │ │ │ │ ├── MultiAtlasOverlappingNodesFixer.java │ │ │ │ │ ├── MultiEdge.java │ │ │ │ │ ├── MultiLine.java │ │ │ │ │ ├── MultiNode.java │ │ │ │ │ ├── MultiPoint.java │ │ │ │ │ ├── MultiRelation.java │ │ │ │ │ ├── README.md │ │ │ │ │ ├── SubAreaList.java │ │ │ │ │ ├── SubEdgeList.java │ │ │ │ │ ├── SubLineList.java │ │ │ │ │ ├── SubNodeList.java │ │ │ │ │ ├── SubPointList.java │ │ │ │ │ └── SubRelationList.java │ │ │ │ ├── packed/ │ │ │ │ │ ├── PackedArea.java │ │ │ │ │ ├── PackedAtlas.java │ │ │ │ │ ├── PackedAtlasBuilder.java │ │ │ │ │ ├── PackedAtlasCloner.java │ │ │ │ │ ├── PackedAtlasLogMessages.java │ │ │ │ │ ├── PackedAtlasSerializer.java │ │ │ │ │ ├── PackedEdge.java │ │ │ │ │ ├── PackedLine.java │ │ │ │ │ ├── PackedNode.java │ │ │ │ │ ├── PackedPoint.java │ │ │ │ │ ├── PackedRelation.java │ │ │ │ │ ├── PackedTagStore.java │ │ │ │ │ └── README.md │ │ │ │ ├── pbf/ │ │ │ │ │ ├── AtlasLoadingOption.java │ │ │ │ │ ├── BridgeConfiguredFilter.java │ │ │ │ │ ├── CloseableOsmosisReader.java │ │ │ │ │ ├── converters/ │ │ │ │ │ │ └── TagMapToTagCollectionConverter.java │ │ │ │ │ └── slicing/ │ │ │ │ │ └── identifier/ │ │ │ │ │ ├── AbstractIdentifierFactory.java │ │ │ │ │ ├── CountrySlicingIdentifierFactory.java │ │ │ │ │ ├── PaddingIdentifierFactory.java │ │ │ │ │ ├── PointIdentifierFactory.java │ │ │ │ │ ├── ReverseIdentifierFactory.java │ │ │ │ │ └── WaySectionIdentifierFactory.java │ │ │ │ ├── raw/ │ │ │ │ │ ├── creation/ │ │ │ │ │ │ ├── OsmPbfCounter.java │ │ │ │ │ │ ├── OsmPbfReader.java │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── RawAtlasGenerator.java │ │ │ │ │ │ └── RawAtlasStatistic.java │ │ │ │ │ ├── sectioning/ │ │ │ │ │ │ ├── AtlasSectionProcessor.java │ │ │ │ │ │ ├── PbfOneWay.java │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ └── TagMap.java │ │ │ │ │ └── slicing/ │ │ │ │ │ ├── CountryCodeProperties.java │ │ │ │ │ ├── README.md │ │ │ │ │ └── RawAtlasSlicer.java │ │ │ │ ├── routing/ │ │ │ │ │ ├── AStarRouter.java │ │ │ │ │ ├── AbstractRouter.java │ │ │ │ │ ├── AllPathsRouter.java │ │ │ │ │ ├── README.md │ │ │ │ │ └── Router.java │ │ │ │ ├── statistics/ │ │ │ │ │ ├── AtlasStatistics.java │ │ │ │ │ ├── AtlasStatisticsMerger.java │ │ │ │ │ ├── Counter.java │ │ │ │ │ └── coverage/ │ │ │ │ │ ├── Coverage.java │ │ │ │ │ ├── area/ │ │ │ │ │ │ ├── AreaCoverage.java │ │ │ │ │ │ ├── LakeAreaCoverage.java │ │ │ │ │ │ └── RiverAreaCoverage.java │ │ │ │ │ ├── linear/ │ │ │ │ │ │ ├── BusRouteLinearCoverage.java │ │ │ │ │ │ ├── LinearCoverage.java │ │ │ │ │ │ ├── edge/ │ │ │ │ │ │ │ ├── AllHighwayTagEdgeCoverage.java │ │ │ │ │ │ │ ├── BridgeEdgeCoverage.java │ │ │ │ │ │ │ ├── EdgeCoverage.java │ │ │ │ │ │ │ ├── FerryEdgeCoverage.java │ │ │ │ │ │ │ ├── FreshnessEdgeCoverage.java │ │ │ │ │ │ │ ├── LanesEdgeCoverage.java │ │ │ │ │ │ │ ├── NameEdgeCoverage.java │ │ │ │ │ │ │ ├── NoNameEdgeCoverage.java │ │ │ │ │ │ │ ├── OneWayEdgeCoverage.java │ │ │ │ │ │ │ ├── PrivateAccessEdgeCoverage.java │ │ │ │ │ │ │ ├── ReferenceEdgeCoverage.java │ │ │ │ │ │ │ ├── SpeedLimitEdgeCoverage.java │ │ │ │ │ │ │ ├── SurfaceEdgeCoverage.java │ │ │ │ │ │ │ ├── TollEdgeCoverage.java │ │ │ │ │ │ │ └── TunnelEdgeCoverage.java │ │ │ │ │ │ └── line/ │ │ │ │ │ │ ├── LineCoverage.java │ │ │ │ │ │ ├── RailLineCoverage.java │ │ │ │ │ │ ├── RiverLineCoverage.java │ │ │ │ │ │ └── TransitRailLineCoverage.java │ │ │ │ │ ├── poi/ │ │ │ │ │ │ ├── EdgesCountCoverage.java │ │ │ │ │ │ ├── LastUserNameCountCoverage.java │ │ │ │ │ │ ├── OneWayEdgesCountCoverage.java │ │ │ │ │ │ └── SimpleCoverage.java │ │ │ │ │ └── weird/ │ │ │ │ │ └── NodesPerLength.java │ │ │ │ ├── sub/ │ │ │ │ │ ├── AtlasCutType.java │ │ │ │ │ └── SubAtlasCreator.java │ │ │ │ ├── validators/ │ │ │ │ │ ├── AtlasEdgeValidator.java │ │ │ │ │ ├── AtlasLineItemValidator.java │ │ │ │ │ ├── AtlasLocationItemValidator.java │ │ │ │ │ ├── AtlasNodeValidator.java │ │ │ │ │ ├── AtlasRelationValidator.java │ │ │ │ │ ├── AtlasValidator.java │ │ │ │ │ └── FeatureChangeUsefulnessValidator.java │ │ │ │ └── walker/ │ │ │ │ ├── EdgeWalker.java │ │ │ │ ├── OsmWayWalker.java │ │ │ │ └── SimpleEdgeWalker.java │ │ │ ├── boundary/ │ │ │ │ ├── CountryBoundaryMap.java │ │ │ │ ├── CountryBoundaryMapArchiver.java │ │ │ │ ├── CountryBoundaryMapCompareCommand.java │ │ │ │ ├── CountryBoundaryMapFiller.java │ │ │ │ ├── CountryCodeGenerator.java │ │ │ │ ├── CountryShardListing.java │ │ │ │ ├── CountryShardListingProcessor.java │ │ │ │ ├── CountryToShardListCache.java │ │ │ │ ├── CountryToShardListing.java │ │ │ │ ├── README.md │ │ │ │ └── converters/ │ │ │ │ ├── CountryBoundaryMapGeoJsonConverter.java │ │ │ │ └── CountryListTwoWayStringConverter.java │ │ │ ├── clipping/ │ │ │ │ ├── Clip.java │ │ │ │ ├── GeometryOperation.java │ │ │ │ ├── MultiPolygonClipper.java │ │ │ │ ├── PolygonClipper.java │ │ │ │ └── README.md │ │ │ ├── constants/ │ │ │ │ └── WorldGeodeticStandardConstants.java │ │ │ ├── converters/ │ │ │ │ ├── GeodeticEarthCenteredEarthFixedConverter.java │ │ │ │ ├── MultiPolygonStringConverter.java │ │ │ │ ├── MultiplePolyLineToMultiPolygonConverter.java │ │ │ │ ├── MultiplePolyLineToPolygonsConverter.java │ │ │ │ ├── MultiplePolyLineToPolygonsConverterCommand.java │ │ │ │ ├── PolyLineStringConverter.java │ │ │ │ ├── PolygonStringConverter.java │ │ │ │ ├── PolygonStringFormat.java │ │ │ │ ├── WkMultiPolygonConverter.java │ │ │ │ ├── WkbLocationConverter.java │ │ │ │ ├── WkbMultiPolyLineConverter.java │ │ │ │ ├── WkbMultiPolygonConverter.java │ │ │ │ ├── WkbPolyLineConverter.java │ │ │ │ ├── WkbPolygonConverter.java │ │ │ │ ├── WktLocationConverter.java │ │ │ │ ├── WktMultiPolyLineConverter.java │ │ │ │ ├── WktMultiPolygonConverter.java │ │ │ │ ├── WktPolyLineConverter.java │ │ │ │ ├── WktPolygonConverter.java │ │ │ │ └── jts/ │ │ │ │ ├── GeometryStreamer.java │ │ │ │ ├── JtsCoordinateArrayConverter.java │ │ │ │ ├── JtsLinearRingConverter.java │ │ │ │ ├── JtsLocationConverter.java │ │ │ │ ├── JtsMultiPolyLineConverter.java │ │ │ │ ├── JtsMultiPolygonConverter.java │ │ │ │ ├── JtsMultiPolygonToMultiLineStringConverter.java │ │ │ │ ├── JtsMultiPolygonToMultiPolygonConverter.java │ │ │ │ ├── JtsPointConverter.java │ │ │ │ ├── JtsPolyLineConverter.java │ │ │ │ ├── JtsPolygonConverter.java │ │ │ │ ├── JtsPolygonToMultiPolygonConverter.java │ │ │ │ ├── JtsPrecisionManager.java │ │ │ │ └── JtsUtility.java │ │ │ ├── coordinates/ │ │ │ │ ├── EarthCenteredEarthFixedCoordinate.java │ │ │ │ └── GeodeticCoordinate.java │ │ │ ├── geojson/ │ │ │ │ ├── ConcatenateGeoJsonCommand.java │ │ │ │ ├── GeoJson.java │ │ │ │ ├── GeoJsonBuilder.java │ │ │ │ ├── GeoJsonCollection.java │ │ │ │ ├── GeoJsonConstants.java │ │ │ │ ├── GeoJsonFeature.java │ │ │ │ ├── GeoJsonFeatureCollection.java │ │ │ │ ├── GeoJsonGeometry.java │ │ │ │ ├── GeoJsonObject.java │ │ │ │ ├── GeoJsonProperties.java │ │ │ │ ├── GeoJsonSaver.java │ │ │ │ ├── GeoJsonType.java │ │ │ │ ├── GeoJsonUtils.java │ │ │ │ ├── GeojsonGeometryCollection.java │ │ │ │ └── parser/ │ │ │ │ ├── GeoJsonParser.java │ │ │ │ ├── README.md │ │ │ │ ├── domain/ │ │ │ │ │ ├── annotation/ │ │ │ │ │ │ └── Foreign.java │ │ │ │ │ ├── base/ │ │ │ │ │ │ ├── AbstractGeoJsonItem.java │ │ │ │ │ │ ├── GeoJsonItem.java │ │ │ │ │ │ └── type/ │ │ │ │ │ │ ├── FeatureType.java │ │ │ │ │ │ ├── GeometryType.java │ │ │ │ │ │ ├── Type.java │ │ │ │ │ │ └── TypeUtil.java │ │ │ │ │ ├── bbox/ │ │ │ │ │ │ ├── AbstractBbox.java │ │ │ │ │ │ ├── Bbox.java │ │ │ │ │ │ ├── Bbox2D.java │ │ │ │ │ │ ├── Bbox3D.java │ │ │ │ │ │ └── Dimensions.java │ │ │ │ │ ├── feature/ │ │ │ │ │ │ ├── AbstractFeature.java │ │ │ │ │ │ ├── Feature.java │ │ │ │ │ │ └── FeatureCollection.java │ │ │ │ │ ├── foreign/ │ │ │ │ │ │ ├── DefaultForeignFieldsImpl.java │ │ │ │ │ │ ├── ForeignFields.java │ │ │ │ │ │ └── SupportsForeigners.java │ │ │ │ │ ├── geometry/ │ │ │ │ │ │ ├── AbstractGeometry.java │ │ │ │ │ │ ├── AbstractGeometryWithCoordinateSupport.java │ │ │ │ │ │ ├── Geometry.java │ │ │ │ │ │ ├── GeometryCollection.java │ │ │ │ │ │ ├── GeometryWithCoordinates.java │ │ │ │ │ │ ├── LineString.java │ │ │ │ │ │ ├── MultiLineString.java │ │ │ │ │ │ ├── MultiPoint.java │ │ │ │ │ │ ├── MultiPolygon.java │ │ │ │ │ │ ├── Point.java │ │ │ │ │ │ ├── Polygon.java │ │ │ │ │ │ └── coordinate/ │ │ │ │ │ │ ├── Coordinates.java │ │ │ │ │ │ ├── Position.java │ │ │ │ │ │ └── Positions.java │ │ │ │ │ └── properties/ │ │ │ │ │ ├── Properties.java │ │ │ │ │ └── ext/ │ │ │ │ │ └── change/ │ │ │ │ │ ├── Description.java │ │ │ │ │ ├── Descriptor.java │ │ │ │ │ └── FeatureChangeProperties.java │ │ │ │ ├── impl/ │ │ │ │ │ └── jackson/ │ │ │ │ │ └── GeoJsonParserJacksonImpl.java │ │ │ │ └── mapper/ │ │ │ │ ├── Mapper.java │ │ │ │ └── impl/ │ │ │ │ └── DefaultBeanUtilsBasedMapperImpl.java │ │ │ ├── index/ │ │ │ │ ├── JtsSpatialIndex.java │ │ │ │ ├── PackedSpatialIndex.java │ │ │ │ ├── QuadTree.java │ │ │ │ ├── RTree.java │ │ │ │ └── SpatialIndex.java │ │ │ ├── matching/ │ │ │ │ ├── PolyLineMatch.java │ │ │ │ └── PolyLineRoute.java │ │ │ └── sharding/ │ │ │ ├── CountryShard.java │ │ │ ├── DynamicTileSharding.java │ │ │ ├── GeoHashSharding.java │ │ │ ├── GeoHashTile.java │ │ │ ├── GeoHashTileIterable.java │ │ │ ├── LocationToShardCommand.java │ │ │ ├── README.md │ │ │ ├── Shard.java │ │ │ ├── Sharding.java │ │ │ ├── SlippyTile.java │ │ │ ├── SlippyTileSharding.java │ │ │ ├── converters/ │ │ │ │ ├── DynamicTileShardingGeoJsonConverter.java │ │ │ │ ├── RectangleToSpatial4JRectangleConverter.java │ │ │ │ ├── SlippyTileConverter.java │ │ │ │ └── StringToShardConverter.java │ │ │ └── preparation/ │ │ │ ├── TilePrinter.java │ │ │ ├── tileDownload.sh │ │ │ ├── tileExecute.sh │ │ │ └── tilePrinter.sh │ │ ├── locale/ │ │ │ ├── IsoCountry.java │ │ │ ├── IsoCountryFuzzyMatcher.java │ │ │ └── IsoLanguage.java │ │ ├── proto/ │ │ │ ├── ProtoSerializable.java │ │ │ ├── adapters/ │ │ │ │ ├── ProtoAdapter.java │ │ │ │ ├── ProtoAtlasMetaDataAdapter.java │ │ │ │ ├── ProtoByteArrayOfArraysAdapter.java │ │ │ │ ├── ProtoIntegerArrayOfArraysAdapter.java │ │ │ │ ├── ProtoIntegerStringDictionaryAdapter.java │ │ │ │ ├── ProtoLongArrayAdapter.java │ │ │ │ ├── ProtoLongArrayOfArraysAdapter.java │ │ │ │ ├── ProtoLongToLongMapAdapter.java │ │ │ │ ├── ProtoLongToLongMultiMapAdapter.java │ │ │ │ ├── ProtoPackedTagStoreAdapter.java │ │ │ │ ├── ProtoPolyLineArrayAdapter.java │ │ │ │ └── ProtoPolygonArrayAdapter.java │ │ │ ├── builder/ │ │ │ │ └── ProtoAtlasBuilder.java │ │ │ ├── command/ │ │ │ │ ├── PackedToProtoAtlasSubCommand.java │ │ │ │ └── ProtoToPackedAtlasSubCommand.java │ │ │ └── converters/ │ │ │ ├── ProtoIntegerArrayOfArraysConverter.java │ │ │ ├── ProtoLocationConverter.java │ │ │ └── ProtoTagListConverter.java │ │ ├── streaming/ │ │ │ ├── CounterOutputStream.java │ │ │ ├── NotifyingIOUtils.java │ │ │ ├── SplittableInputStream.java │ │ │ ├── Streams.java │ │ │ ├── StringInputStream.java │ │ │ ├── StringOutputStream.java │ │ │ ├── compression/ │ │ │ │ ├── Compressor.java │ │ │ │ └── Decompressor.java │ │ │ ├── readers/ │ │ │ │ ├── CsvLine.java │ │ │ │ ├── CsvReader.java │ │ │ │ ├── CsvSchema.java │ │ │ │ ├── GeoJsonReader.java │ │ │ │ ├── csv/ │ │ │ │ │ └── converters/ │ │ │ │ │ └── CsvLineConverter.java │ │ │ │ └── json/ │ │ │ │ ├── converters/ │ │ │ │ │ ├── MultiPolyLineCoordinateConverter.java │ │ │ │ │ ├── MultiPolygonCoordinateConverter.java │ │ │ │ │ ├── PointCoordinateConverter.java │ │ │ │ │ ├── PolyLineCoordinateConverter.java │ │ │ │ │ └── PolygonCoordinateConverter.java │ │ │ │ ├── deserializers/ │ │ │ │ │ ├── LocatedDeserializer.java │ │ │ │ │ ├── LocationDeserializer.java │ │ │ │ │ ├── MultiPolyLineDeserializer.java │ │ │ │ │ ├── MultiPolygonDeserializer.java │ │ │ │ │ ├── PolyLineDeserializer.java │ │ │ │ │ └── PolygonDeserializer.java │ │ │ │ └── serializers/ │ │ │ │ ├── LocationSerializer.java │ │ │ │ ├── MultiLocationSerializer.java │ │ │ │ ├── PolyLineSerializer.java │ │ │ │ ├── PolygonSerializer.java │ │ │ │ └── PropertiesLocated.java │ │ │ ├── resource/ │ │ │ │ ├── AbstractResource.java │ │ │ │ ├── AbstractWritableResource.java │ │ │ │ ├── ByteArrayResource.java │ │ │ │ ├── ClassResource.java │ │ │ │ ├── File.java │ │ │ │ ├── FileSuffix.java │ │ │ │ ├── GeoJsonFile.java │ │ │ │ ├── InputStreamResource.java │ │ │ │ ├── InputStreamResourceCloseable.java │ │ │ │ ├── LineFilteredResource.java │ │ │ │ ├── LineWriter.java │ │ │ │ ├── OutputStreamWritableResource.java │ │ │ │ ├── OutputStreamWritableResourceCloseable.java │ │ │ │ ├── README.md │ │ │ │ ├── Resource.java │ │ │ │ ├── ResourceCloseable.java │ │ │ │ ├── StreamOfResourceStreams.java │ │ │ │ ├── StringResource.java │ │ │ │ ├── TemporaryFile.java │ │ │ │ ├── WritableResource.java │ │ │ │ ├── WritableResourceCloseable.java │ │ │ │ ├── http/ │ │ │ │ │ ├── DeleteResource.java │ │ │ │ │ ├── GetResource.java │ │ │ │ │ ├── HttpResource.java │ │ │ │ │ ├── PostResource.java │ │ │ │ │ └── PutResource.java │ │ │ │ └── zip/ │ │ │ │ ├── ZipFileWritableResource.java │ │ │ │ ├── ZipResource.java │ │ │ │ └── ZipWritableResource.java │ │ │ └── writers/ │ │ │ ├── JsonWriter.java │ │ │ └── SafeBufferedWriter.java │ │ ├── tags/ │ │ │ ├── AbandonedAerowayTag.java │ │ │ ├── AbandonedAmenityTag.java │ │ │ ├── AbandonedArtworkTypeTag.java │ │ │ ├── AccessTag.java │ │ │ ├── AddressCityTag.java │ │ │ ├── AddressCountryTag.java │ │ │ ├── AddressFlatsTag.java │ │ │ ├── AddressFullTag.java │ │ │ ├── AddressHousenameTag.java │ │ │ ├── AddressHousenumberTag.java │ │ │ ├── AddressInterpolationTag.java │ │ │ ├── AddressPlaceTag.java │ │ │ ├── AddressPostcodeTag.java │ │ │ ├── AddressProvinceTag.java │ │ │ ├── AddressStateTag.java │ │ │ ├── AddressStreetTag.java │ │ │ ├── AdministrativeLevelTag.java │ │ │ ├── AerialWayTag.java │ │ │ ├── AerowayTag.java │ │ │ ├── AmenityTag.java │ │ │ ├── AreaTag.java │ │ │ ├── ArtworkTypeTag.java │ │ │ ├── AtlasTag.java │ │ │ ├── BarrierTag.java │ │ │ ├── BicycleTag.java │ │ │ ├── BoundaryTag.java │ │ │ ├── BrandTag.java │ │ │ ├── BreakfastTag.java │ │ │ ├── BridgeTag.java │ │ │ ├── BuildingHeightTag.java │ │ │ ├── BuildingLevelsTag.java │ │ │ ├── BuildingMinLevelTag.java │ │ │ ├── BuildingPartTag.java │ │ │ ├── BuildingRoofTag.java │ │ │ ├── BuildingTag.java │ │ │ ├── CheckDateTag.java │ │ │ ├── ConstructionDateTag.java │ │ │ ├── ConstructionTag.java │ │ │ ├── ContactDiasporaTag.java │ │ │ ├── ContactEmailTag.java │ │ │ ├── ContactFacebookTag.java │ │ │ ├── ContactFaxTag.java │ │ │ ├── ContactGooglePlusTag.java │ │ │ ├── ContactInstagramTag.java │ │ │ ├── ContactLinkedInTag.java │ │ │ ├── ContactMobileTag.java │ │ │ ├── ContactPhoneTag.java │ │ │ ├── ContactTwitterTag.java │ │ │ ├── ContactWebsiteTag.java │ │ │ ├── ContactXingTag.java │ │ │ ├── CoveredTag.java │ │ │ ├── CuisineTag.java │ │ │ ├── CyclewayLaneTag.java │ │ │ ├── CyclewayLeftTag.java │ │ │ ├── CyclewayRightTag.java │ │ │ ├── CyclewayTag.java │ │ │ ├── DestinationForwardTag.java │ │ │ ├── DestinationIntRefTag.java │ │ │ ├── DestinationRefTag.java │ │ │ ├── DestinationRefToTag.java │ │ │ ├── DestinationStreetTag.java │ │ │ ├── DestinationTag.java │ │ │ ├── DirectionTag.java │ │ │ ├── DisusedRailwayTag.java │ │ │ ├── DisusedShopTag.java │ │ │ ├── ElevationTag.java │ │ │ ├── EmbankmentTag.java │ │ │ ├── EntranceTag.java │ │ │ ├── EstimatedWidthTag.java │ │ │ ├── ExitToLeftTag.java │ │ │ ├── ExitToRightTag.java │ │ │ ├── ExitToTag.java │ │ │ ├── FaxTag.java │ │ │ ├── FerryTag.java │ │ │ ├── FixMeTag.java │ │ │ ├── FootTag.java │ │ │ ├── FootwayTag.java │ │ │ ├── FordTag.java │ │ │ ├── FourWheelDriveOnlyTag.java │ │ │ ├── HarbourTag.java │ │ │ ├── HeightTag.java │ │ │ ├── HighResolutionTag.java │ │ │ ├── HighwayTag.java │ │ │ ├── HistoricTag.java │ │ │ ├── ISOCountryTag.java │ │ │ ├── IceRoadTag.java │ │ │ ├── IndustrialTag.java │ │ │ ├── IntermittentTag.java │ │ │ ├── InternetAccessFeeTag.java │ │ │ ├── InternetAccessTag.java │ │ │ ├── Iso31662CountryTag.java │ │ │ ├── Iso31663CountryTag.java │ │ │ ├── Iso3166DefaultCountryTag.java │ │ │ ├── JunctionTag.java │ │ │ ├── LandUseTag.java │ │ │ ├── LandcoverTag.java │ │ │ ├── LanesTag.java │ │ │ ├── LastEditChangesetTag.java │ │ │ ├── LastEditTimeTag.java │ │ │ ├── LastEditUserIdentifierTag.java │ │ │ ├── LastEditUserNameTag.java │ │ │ ├── LastEditVersionTag.java │ │ │ ├── LayerTag.java │ │ │ ├── LeisureTag.java │ │ │ ├── LevelTag.java │ │ │ ├── LivingStreetTag.java │ │ │ ├── LocalizedTagNameWithOptionalDate.java │ │ │ ├── LocationTag.java │ │ │ ├── ManMadeTag.java │ │ │ ├── MaxHeightTag.java │ │ │ ├── MaxSpeedBackwardTag.java │ │ │ ├── MaxSpeedForwardTag.java │ │ │ ├── MaxSpeedTag.java │ │ │ ├── MaxWidthTag.java │ │ │ ├── MilitaryTag.java │ │ │ ├── MinHeightTag.java │ │ │ ├── MinSpeedTag.java │ │ │ ├── MotorVehicleTag.java │ │ │ ├── MotorcarTag.java │ │ │ ├── MotorcycleTag.java │ │ │ ├── NaturalTag.java │ │ │ ├── NetworkTag.java │ │ │ ├── NoExitTag.java │ │ │ ├── NotesTag.java │ │ │ ├── OpenDateTag.java │ │ │ ├── OpeningDateTag.java │ │ │ ├── OpeningHoursTag.java │ │ │ ├── OrganicTag.java │ │ │ ├── ParkingTag.java │ │ │ ├── PhoneTag.java │ │ │ ├── PlaceTag.java │ │ │ ├── PowerTag.java │ │ │ ├── ProtectClassTag.java │ │ │ ├── PublicServiceVehiclesTag.java │ │ │ ├── README.md │ │ │ ├── RailwayTag.java │ │ │ ├── RampBicycleTag.java │ │ │ ├── RelationTypeTag.java │ │ │ ├── RouteTag.java │ │ │ ├── SaltTag.java │ │ │ ├── SeasonalTag.java │ │ │ ├── ServiceTag.java │ │ │ ├── ShopTag.java │ │ │ ├── SidewalkLeftTag.java │ │ │ ├── SidewalkRightTag.java │ │ │ ├── SidewalkTag.java │ │ │ ├── SkiTag.java │ │ │ ├── SmokingTag.java │ │ │ ├── SmoothnessTag.java │ │ │ ├── SnowmobileTag.java │ │ │ ├── SourceTag.java │ │ │ ├── SourceTypeTag.java │ │ │ ├── SourceURLTag.java │ │ │ ├── SportTag.java │ │ │ ├── SurfaceTag.java │ │ │ ├── SwimmingPoolTag.java │ │ │ ├── SyntheticBoundaryNodeTag.java │ │ │ ├── SyntheticDuplicateOsmNodeTag.java │ │ │ ├── SyntheticGeometrySlicedTag.java │ │ │ ├── SyntheticInvalidGeometryTag.java │ │ │ ├── SyntheticInvalidMultiPolygonRelationMembersRemovedTag.java │ │ │ ├── SyntheticInvalidWaySectionTag.java │ │ │ ├── SyntheticRelationMemberAdded.java │ │ │ ├── SyntheticRelationRoleUpdated.java │ │ │ ├── SyntheticSyntheticRelationMemberTag.java │ │ │ ├── Taggable.java │ │ │ ├── TemporaryDateOnTag.java │ │ │ ├── TollTag.java │ │ │ ├── TourismTag.java │ │ │ ├── TracktypeTag.java │ │ │ ├── TrafficCalmingTag.java │ │ │ ├── TunnelTag.java │ │ │ ├── TurnLanesBackwardTag.java │ │ │ ├── TurnLanesForwardTag.java │ │ │ ├── TurnLanesTag.java │ │ │ ├── TurnRestrictionTag.java │ │ │ ├── TurnTag.java │ │ │ ├── URLTag.java │ │ │ ├── UsageTag.java │ │ │ ├── VacantTag.java │ │ │ ├── VehicleTag.java │ │ │ ├── VendingTag.java │ │ │ ├── WaterTag.java │ │ │ ├── WaterwayTag.java │ │ │ ├── WebsiteTag.java │ │ │ ├── WetlandTag.java │ │ │ ├── WheelchairDescriptionTag.java │ │ │ ├── WheelchairTag.java │ │ │ ├── WidthTag.java │ │ │ ├── WifiTag.java │ │ │ ├── WikidataTag.java │ │ │ ├── WikipediaTag.java │ │ │ ├── WinterRoadTag.java │ │ │ ├── ZooTag.java │ │ │ ├── annotations/ │ │ │ │ ├── Tag.java │ │ │ │ ├── TagKey.java │ │ │ │ ├── TagValue.java │ │ │ │ ├── TagValueAs.java │ │ │ │ ├── TagValueDeprecated.java │ │ │ │ ├── extraction/ │ │ │ │ │ ├── AltitudeExtractor.java │ │ │ │ │ ├── IsoCountryExtractor.java │ │ │ │ │ ├── LengthExtractor.java │ │ │ │ │ ├── LongExtractor.java │ │ │ │ │ ├── NonEmptyStringExtractor.java │ │ │ │ │ ├── OrdinalExtractor.java │ │ │ │ │ ├── SpeedExtractor.java │ │ │ │ │ └── TagExtractor.java │ │ │ │ └── validation/ │ │ │ │ ├── DoubleValidator.java │ │ │ │ ├── ExactMatchValidator.java │ │ │ │ ├── ISO2CountryValidator.java │ │ │ │ ├── ISO3CountryValidator.java │ │ │ │ ├── ISOCountryValidator.java │ │ │ │ ├── LengthValidator.java │ │ │ │ ├── LongValidator.java │ │ │ │ ├── NonEmptyStringValidator.java │ │ │ │ ├── NoneValidator.java │ │ │ │ ├── NumericValidator.java │ │ │ │ ├── OrdinalValidator.java │ │ │ │ ├── SpeedValidator.java │ │ │ │ ├── TagDocumenter.java │ │ │ │ ├── TagValidator.java │ │ │ │ ├── TimestampValidator.java │ │ │ │ ├── URIValidator.java │ │ │ │ └── Validators.java │ │ │ ├── cache/ │ │ │ │ ├── CachingValidator.java │ │ │ │ └── Tagger.java │ │ │ ├── filters/ │ │ │ │ ├── ConfiguredTaggableFilter.java │ │ │ │ ├── LineFilterConverter.java │ │ │ │ ├── README.md │ │ │ │ ├── RegexTaggableFilter.java │ │ │ │ ├── TaggableFilter.java │ │ │ │ ├── TaggableFilterToMatcherConverter.java │ │ │ │ └── matcher/ │ │ │ │ ├── ConfiguredTaggableMatcher.java │ │ │ │ ├── README.md │ │ │ │ ├── TaggableMatcher.java │ │ │ │ └── parsing/ │ │ │ │ ├── Lexer.java │ │ │ │ ├── Parser.java │ │ │ │ ├── SemanticChecker.java │ │ │ │ ├── Token.java │ │ │ │ └── tree/ │ │ │ │ ├── ASTNode.java │ │ │ │ ├── AndOperator.java │ │ │ │ ├── BangOperator.java │ │ │ │ ├── BinaryOperator.java │ │ │ │ ├── EqualsOperator.java │ │ │ │ ├── LiteralOperand.java │ │ │ │ ├── Operand.java │ │ │ │ ├── OrOperator.java │ │ │ │ ├── RegexOperand.java │ │ │ │ ├── TreePrinter.java │ │ │ │ ├── UnaryOperator.java │ │ │ │ └── XorOperator.java │ │ │ ├── names/ │ │ │ │ ├── AlternativeNameTag.java │ │ │ │ ├── BridgeNameTag.java │ │ │ │ ├── BulkNameFinder.java │ │ │ │ ├── HistoricallyKnownAsTag.java │ │ │ │ ├── HistoricallyReferencedAsTag.java │ │ │ │ ├── InternationallyKnownAsTag.java │ │ │ │ ├── InternationallyReferencedAsTag.java │ │ │ │ ├── LocallyKnownAsTag.java │ │ │ │ ├── LocallyReferencedAsTag.java │ │ │ │ ├── Name1Tag.java │ │ │ │ ├── NameFinder.java │ │ │ │ ├── NameLeftTag.java │ │ │ │ ├── NameRightTag.java │ │ │ │ ├── NameTag.java │ │ │ │ ├── NationallyKnownAsTag.java │ │ │ │ ├── NationallyReferencedAsTag.java │ │ │ │ ├── NoNameTag.java │ │ │ │ ├── OfficialNameTag.java │ │ │ │ ├── OldReferenceTag.java │ │ │ │ ├── ReferenceTag.java │ │ │ │ ├── RegionallyKnownAsTag.java │ │ │ │ ├── RegionallyReferencedAsTag.java │ │ │ │ ├── ShortNameTag.java │ │ │ │ ├── SortingNameTag.java │ │ │ │ └── TunnelNameTag.java │ │ │ └── oneway/ │ │ │ ├── OneWayTag.java │ │ │ ├── bicycle/ │ │ │ │ ├── BicycleOneWayTag.java │ │ │ │ ├── CyclewayLeftOneWayTag.java │ │ │ │ ├── CyclewayOneWayTag.java │ │ │ │ ├── CyclewayRightOneWayTag.java │ │ │ │ └── OneWayBicycleTag.java │ │ │ └── motor/ │ │ │ ├── OneWayMotorVehicleTag.java │ │ │ ├── OneWayMotorcarTag.java │ │ │ └── OneWayVehicleTag.java │ │ └── utilities/ │ │ ├── README.md │ │ ├── archive/ │ │ │ ├── AbstractArchiverOrExtractor.java │ │ │ ├── ArchiveStorageProfileDelegate.java │ │ │ ├── ArchiveVetoDelegate.java │ │ │ ├── Archiver.java │ │ │ ├── ArchiverEventListener.java │ │ │ ├── DefaultZipVetoDelegate.java │ │ │ ├── Extractor.java │ │ │ ├── UnzipperCommand.java │ │ │ └── ZipperCommand.java │ │ ├── arrays/ │ │ │ ├── Arrays.java │ │ │ ├── BitArray.java │ │ │ ├── BooleanArray.java │ │ │ ├── ByteArray.java │ │ │ ├── ByteArrayOfArrays.java │ │ │ ├── IntegerArray.java │ │ │ ├── IntegerArrayOfArrays.java │ │ │ ├── LargeArray.java │ │ │ ├── LongArray.java │ │ │ ├── LongArrayOfArrays.java │ │ │ ├── PolyLineArray.java │ │ │ ├── PolygonArray.java │ │ │ ├── PrimitiveArray.java │ │ │ ├── ShortArray.java │ │ │ └── StringArray.java │ │ ├── caching/ │ │ │ ├── ConcurrentResourceCache.java │ │ │ ├── LocalFileInMemoryCache.java │ │ │ ├── README.md │ │ │ ├── ResourceCache.java │ │ │ └── strategies/ │ │ │ ├── AbstractCachingStrategy.java │ │ │ ├── ByteArrayCachingStrategy.java │ │ │ ├── CachingStrategy.java │ │ │ ├── GlobalNamespaceCachingStrategy.java │ │ │ ├── NamespaceCachingStrategy.java │ │ │ └── NoCachingStrategy.java │ │ ├── checkstyle/ │ │ │ ├── ArrangementCheck.java │ │ │ └── README.md │ │ ├── cli/ │ │ │ └── operations/ │ │ │ ├── AbstractHDFSOperation.java │ │ │ ├── AbstractOperation.java │ │ │ ├── CheckIfFileExistsOperation.java │ │ │ ├── DeepLSOperation.java │ │ │ ├── HDFSCatOperation.java │ │ │ ├── HDFSCheckIfFileExistsOperation.java │ │ │ ├── HDFSCopyOperation.java │ │ │ ├── HDFSLSOperation.java │ │ │ ├── HDFSMkdirOperation.java │ │ │ ├── HDFSPutOperation.java │ │ │ ├── LSOperation.java │ │ │ ├── MkdirOperation.java │ │ │ ├── Operation.java │ │ │ ├── RMDirOperation.java │ │ │ └── base/ │ │ │ ├── AvailableSocketFinder.java │ │ │ ├── OperationResults.java │ │ │ ├── RemoteObject.java │ │ │ ├── SCPOperation.java │ │ │ ├── SCPOperationResults.java │ │ │ ├── SSHForwarder.java │ │ │ ├── SSHOperation.java │ │ │ └── SSHOperationResults.java │ │ ├── collections/ │ │ │ ├── EnhancedCollectors.java │ │ │ ├── EnumSetCollector.java │ │ │ ├── FilteredIterable.java │ │ │ ├── FixedSizePriorityQueue.java │ │ │ ├── ImmutableListCollector.java │ │ │ ├── Iterables.java │ │ │ ├── JoinedCollection.java │ │ │ ├── Maps.java │ │ │ ├── MultiIterable.java │ │ │ ├── OptionalIterable.java │ │ │ ├── ParallelIterable.java │ │ │ ├── Sets.java │ │ │ ├── ShardBucketCollection.java │ │ │ ├── StreamIterable.java │ │ │ ├── StringList.java │ │ │ ├── SubIterable.java │ │ │ ├── UnmodifiableSortedMapCollector.java │ │ │ └── UnmodifiableSortedSetCollector.java │ │ ├── command/ │ │ │ ├── ActiveModuleIndexWriter.java │ │ │ ├── AtlasShellToolsException.java │ │ │ ├── ReflectionUtilities.java │ │ │ ├── abstractcommand/ │ │ │ │ ├── AbstractAtlasShellToolsCommand.java │ │ │ │ ├── AtlasShellToolsCommandTemplate.java │ │ │ │ ├── AtlasShellToolsMarkerInterface.java │ │ │ │ ├── CommandOutputDelegate.java │ │ │ │ └── OptionAndArgumentDelegate.java │ │ │ ├── documentation/ │ │ │ │ ├── DocumentationFormatType.java │ │ │ │ ├── DocumentationFormatter.java │ │ │ │ ├── DocumentationRegistrar.java │ │ │ │ └── PagerHelper.java │ │ │ ├── parsing/ │ │ │ │ ├── ArgumentArity.java │ │ │ │ ├── ArgumentOptionality.java │ │ │ │ ├── OptionArgumentType.java │ │ │ │ ├── OptionOptionality.java │ │ │ │ ├── SimpleOptionAndArgumentParser.java │ │ │ │ └── exceptions/ │ │ │ │ ├── AmbiguousAbbreviationException.java │ │ │ │ ├── ArgumentException.java │ │ │ │ ├── OptionParseException.java │ │ │ │ ├── UnknownOptionException.java │ │ │ │ └── UnparsableContextException.java │ │ │ ├── subcommands/ │ │ │ │ ├── AnyToGeoJsonCommand.java │ │ │ │ ├── AtlasDiffCommand.java │ │ │ │ ├── AtlasMetadataReaderCommand.java │ │ │ │ ├── AtlasSearchCommand.java │ │ │ │ ├── AtlasShardingConverterCommand.java │ │ │ │ ├── AtlasShellToolsDemoCommand.java │ │ │ │ ├── ConcatenateAtlasCommand.java │ │ │ │ ├── CountryBoundaryMapPrinterCommand.java │ │ │ │ ├── CountryShardToBoundsCommand.java │ │ │ │ ├── HelloWorldCommand.java │ │ │ │ ├── IsoCountryCodeCommand.java │ │ │ │ ├── JavaToProtoSerializationCommand.java │ │ │ │ ├── OsmFileParserCommand.java │ │ │ │ ├── OsmToAtlasCommand.java │ │ │ │ ├── PackedToTextAtlasCommand.java │ │ │ │ ├── PbfToAtlasCommand.java │ │ │ │ ├── SubAtlasCommand.java │ │ │ │ ├── TaggableMatcherPrinterCommand.java │ │ │ │ ├── TemplateTestCommand.java │ │ │ │ ├── WKTShardCommand.java │ │ │ │ └── templates/ │ │ │ │ ├── AtlasLoaderCommand.java │ │ │ │ ├── AtlasLoaderTemplate.java │ │ │ │ ├── CountryBoundaryMapTemplate.java │ │ │ │ ├── ListOfNumbersTemplate.java │ │ │ │ ├── MultipleOutputCommand.java │ │ │ │ ├── OutputDirectoryTemplate.java │ │ │ │ ├── PredicateTemplate.java │ │ │ │ └── ShardingTemplate.java │ │ │ └── terminal/ │ │ │ ├── TTYAttribute.java │ │ │ └── TTYStringBuilder.java │ │ ├── compression/ │ │ │ ├── IntegerDictionary.java │ │ │ └── LongDictionary.java │ │ ├── configuration/ │ │ │ ├── Configurable.java │ │ │ ├── Configuration.java │ │ │ ├── ConfigurationDeserializer.java │ │ │ ├── ConfigurationReader.java │ │ │ ├── ConfiguredFilter.java │ │ │ ├── MergedConfiguration.java │ │ │ └── StandardConfiguration.java │ │ ├── conversion/ │ │ │ ├── Converter.java │ │ │ ├── HexStringByteArrayConverter.java │ │ │ ├── StringConverter.java │ │ │ ├── StringToPredicateConverter.java │ │ │ ├── TagConverter.java │ │ │ ├── TwoWayConverter.java │ │ │ └── TwoWayStringConverter.java │ │ ├── direction/ │ │ │ └── EdgeDirectionComparator.java │ │ ├── filters/ │ │ │ ├── AtlasEntityPolygonsFilter.java │ │ │ └── IntersectionPolicy.java │ │ ├── function/ │ │ │ ├── QuaternaryFunction.java │ │ │ ├── QuaternaryOperator.java │ │ │ ├── SenaryFunction.java │ │ │ ├── SenaryOperator.java │ │ │ ├── TernaryConsumer.java │ │ │ ├── TernaryFunction.java │ │ │ └── TernaryOperator.java │ │ ├── graphs/ │ │ │ └── DirectedAcyclicGraph.java │ │ ├── http/ │ │ │ └── rest/ │ │ │ ├── DislikedResponseCodeException.java │ │ │ └── HttpResultHandler.java │ │ ├── identifiers/ │ │ │ └── EntityIdentifierGenerator.java │ │ ├── jsoncompare/ │ │ │ ├── RegularExpressionJSONComparator.java │ │ │ └── RegularExpressionJSONCompareResult.java │ │ ├── maps/ │ │ │ ├── IntegerToIntegerMap.java │ │ │ ├── LargeMap.java │ │ │ ├── LinkedMultiMap.java │ │ │ ├── LongToBooleanMap.java │ │ │ ├── LongToIntegerMap.java │ │ │ ├── LongToIntegerMultiMap.java │ │ │ ├── LongToLongMap.java │ │ │ ├── LongToLongMultiMap.java │ │ │ ├── MultiMap.java │ │ │ └── MultiMapWithSet.java │ │ ├── matching/ │ │ │ └── NameMatcher.java │ │ ├── random/ │ │ │ ├── RandomTagsSupplier.java │ │ │ └── RandomTextGenerator.java │ │ ├── regex/ │ │ │ └── RegexUtils.java │ │ ├── runtime/ │ │ │ ├── BoundedPipeBuffer.java │ │ │ ├── ClassPathTree.java │ │ │ ├── Command.java │ │ │ ├── CommandMap.java │ │ │ ├── FlexibleCommand.java │ │ │ ├── FlexibleSubCommand.java │ │ │ ├── OpenPipeBuffer.java │ │ │ ├── PipeBuffer.java │ │ │ ├── Retry.java │ │ │ ├── RunScript.java │ │ │ ├── RunScriptMonitor.java │ │ │ ├── SingleLineMonitor.java │ │ │ ├── TimedRetry.java │ │ │ └── system/ │ │ │ ├── SystemInfo.java │ │ │ └── memory/ │ │ │ └── Memory.java │ │ ├── scalars/ │ │ │ ├── Angle.java │ │ │ ├── Counter.java │ │ │ ├── Distance.java │ │ │ ├── DoubleCounter.java │ │ │ ├── Duration.java │ │ │ ├── README.md │ │ │ ├── Ratio.java │ │ │ ├── Speed.java │ │ │ └── Surface.java │ │ ├── statistic/ │ │ │ ├── AbstractStatistic.java │ │ │ ├── Statistic.java │ │ │ ├── StatisticUtils.java │ │ │ └── storeless/ │ │ │ ├── CounterWithStatistic.java │ │ │ ├── CustomizedStatistic.java │ │ │ └── StatisticType.java │ │ ├── testing/ │ │ │ ├── Bean.java │ │ │ ├── BeanHandler.java │ │ │ ├── CoreTestRule.java │ │ │ ├── CreationContext.java │ │ │ ├── FeatureIDGenerator.java │ │ │ ├── FieldHandler.java │ │ │ ├── FreezeDryFunction.java │ │ │ ├── OsmFileParser.java │ │ │ ├── OsmFileToPbf.java │ │ │ ├── OsmosisXmlReaderFromResource.java │ │ │ ├── TestAtlas.java │ │ │ ├── TestAtlasHandler.java │ │ │ └── TestTaggable.java │ │ ├── threads/ │ │ │ ├── CustomNamesThreadPoolFactory.java │ │ │ ├── LogTicker.java │ │ │ ├── Pool.java │ │ │ ├── Result.java │ │ │ └── Ticker.java │ │ ├── time/ │ │ │ ├── LocalTime.java │ │ │ └── Time.java │ │ ├── timezone/ │ │ │ ├── TimeZoneBoundary.java │ │ │ └── TimeZoneMap.java │ │ ├── tuples/ │ │ │ ├── Either.java │ │ │ └── Tuple.java │ │ ├── unicode/ │ │ │ ├── AbstractClassifier.java │ │ │ ├── Classification.java │ │ │ ├── Classifier.java │ │ │ └── LoadingClassifier.java │ │ └── vectortiles/ │ │ ├── MinimumZoom.java │ │ ├── README.md │ │ ├── TippecanoeCommands.java │ │ ├── TippecanoeConverter.java │ │ ├── TippecanoeExporter.java │ │ ├── TippecanoeGeoJsonExtension.java │ │ └── TippecanoeSettings.java │ ├── proto/ │ │ ├── Area.proto │ │ ├── Edge.proto │ │ ├── Line.proto │ │ ├── Location.proto │ │ ├── Node.proto │ │ ├── Point.proto │ │ ├── ProtoAtlas.proto │ │ ├── ProtoAtlasMetaData.proto │ │ ├── ProtoByteArray.proto │ │ ├── ProtoByteArrayOfArrays.proto │ │ ├── ProtoIntegerArray.proto │ │ ├── ProtoIntegerArrayOfArrays.proto │ │ ├── ProtoIntegerStringDictionary.proto │ │ ├── ProtoLongArray.proto │ │ ├── ProtoLongArrayOfArrays.proto │ │ ├── ProtoLongToLongMap.proto │ │ ├── ProtoLongToLongMultiMap.proto │ │ ├── ProtoPackedTagStore.proto │ │ ├── ProtoPolyLineArray.proto │ │ ├── ProtoPolygonArray.proto │ │ ├── Relation.proto │ │ └── Tag.proto │ └── resources/ │ └── org/ │ └── openstreetmap/ │ └── atlas/ │ ├── atlas.json │ ├── geography/ │ │ └── atlas/ │ │ ├── items/ │ │ │ └── complex/ │ │ │ ├── aoi/ │ │ │ │ └── aoi-tag-filter.json │ │ │ ├── islands/ │ │ │ │ └── islands.json │ │ │ ├── landcover/ │ │ │ │ └── land-cover-tag-filter.json │ │ │ └── water/ │ │ │ └── finder/ │ │ │ ├── canal.json │ │ │ ├── creek.json │ │ │ ├── ditch.json │ │ │ ├── harbour.json │ │ │ ├── lagoon.json │ │ │ ├── lake.json │ │ │ ├── pond.json │ │ │ ├── pool.json │ │ │ ├── reservoir.json │ │ │ ├── river.json │ │ │ └── wetland.json │ │ ├── pbf/ │ │ │ ├── atlas-area.json │ │ │ ├── atlas-edge.json │ │ │ ├── atlas-relation-slicing.json │ │ │ ├── atlas-way-section.json │ │ │ ├── osm-pbf-node.json │ │ │ ├── osm-pbf-relation.json │ │ │ └── osm-pbf-way.json │ │ └── statistics/ │ │ └── coverage/ │ │ └── poi/ │ │ ├── counts.txt │ │ └── countsOne.txt │ ├── tags/ │ │ └── annotations/ │ │ └── implicit-speed-values.json │ └── utilities/ │ ├── checkstyle/ │ │ └── arrangement.txt │ ├── command/ │ │ └── subcommands/ │ │ ├── AnyToGeoJsonCommandDescriptionSection.txt │ │ ├── AnyToGeoJsonCommandExamplesSection.txt │ │ ├── AtlasDiffCommandDescriptionSection.txt │ │ ├── AtlasDiffCommandExamplesSection.txt │ │ ├── AtlasMetadataReaderCommandDescriptionSection.txt │ │ ├── AtlasMetadataReaderCommandExamplesSection.txt │ │ ├── AtlasSearchCommandDescriptionSection.txt │ │ ├── AtlasSearchCommandExamplesSection.txt │ │ ├── AtlasShardingConverterCommandDescriptionSection.txt │ │ ├── AtlasShardingConverterCommandExamplesSection.txt │ │ ├── AtlasShellToolsDemoCommandDescriptionSection.txt │ │ ├── AtlasShellToolsDemoCommandExamplesSection.txt │ │ ├── ConcatenateAtlasCommandDescriptionSection.txt │ │ ├── ConcatenateAtlasCommandExamplesSection.txt │ │ ├── CountryBoundaryMapPrinterCommandDescriptionSection.txt │ │ ├── CountryBoundaryMapPrinterCommandExamplesSection.txt │ │ ├── CountryShardToBoundsCommandDescriptionSection.txt │ │ ├── CountryShardToBoundsCommandExamplesSection.txt │ │ ├── HelloWorldCommandDescriptionSection.txt │ │ ├── IsoCountryCodeCommandDescriptionSection.txt │ │ ├── IsoCountryCodeCommandExamplesSection.txt │ │ ├── JavaToProtoSerializationCommandDescriptionSection.txt │ │ ├── JavaToProtoSerializationCommandExamplesSection.txt │ │ ├── OsmFileParserCommandDescriptionSection.txt │ │ ├── OsmFileParserCommandExamplesSection.txt │ │ ├── OsmToAtlasCommandDescriptionSection.txt │ │ ├── OsmToAtlasCommandExamplesSection.txt │ │ ├── PackedToTextAtlasCommandDescriptionSection.txt │ │ ├── PackedToTextAtlasCommandExamplesSection.txt │ │ ├── PbfToAtlasCommandExamplesSection.txt │ │ ├── PbfToAtlasDescriptionSection.txt │ │ ├── SubAtlasCommandDescriptionSection.txt │ │ ├── SubAtlasCommandExamplesSection.txt │ │ ├── TaggableMatcherPrinterCommandDescriptionSection.txt │ │ ├── TaggableMatcherPrinterCommandExamplesSection.txt │ │ ├── WKTShardCommandDescriptionSection.txt │ │ ├── WKTShardCommandExamplesSection.txt │ │ └── templates/ │ │ ├── AtlasLoaderCommandSection.txt │ │ ├── AtlasLoaderTemplateSection.txt │ │ ├── CountryBoundaryMapTemplateSection.txt │ │ ├── MultipleOutputCommandSection.txt │ │ ├── OutputDirectoryTemplateSection.txt │ │ ├── PredicateTemplateSection.txt │ │ └── ShardingTemplateSection.txt │ ├── random/ │ │ └── dictionary.txt │ ├── timezone/ │ │ ├── index.html │ │ ├── tz_world.dbf │ │ ├── tz_world.prj │ │ ├── tz_world.qix │ │ ├── tz_world.shp │ │ └── tz_world.shx │ ├── unicode/ │ │ └── unicode.defaults │ └── vectortiles/ │ └── minimum-zooms.json └── test/ ├── groovy/ │ └── org/ │ └── openstreetmap/ │ └── atlas/ │ └── geography/ │ └── converters/ │ └── jts/ │ └── JtsPolyLineConverterTest.java ├── java/ │ └── org/ │ └── openstreetmap/ │ └── atlas/ │ ├── event/ │ │ ├── EventBusTest.java │ │ ├── EventServiceTest.java │ │ ├── TestEvent.java │ │ └── TestProcessor.java │ ├── exception/ │ │ ├── CoreExceptionTest.java │ │ └── change/ │ │ └── FeatureChangeMergeExceptionTest.java │ ├── geography/ │ │ ├── AltitudeTest.java │ │ ├── CompressedPolyLineTest.java │ │ ├── HeadingTest.java │ │ ├── LatitudeTest.java │ │ ├── LocationTest.java │ │ ├── LongitudeTest.java │ │ ├── MultiPolyLineTest.java │ │ ├── MultiPolygonTest.java │ │ ├── PolyLineCoveringPolygonTest.java │ │ ├── PolyLineCoveringPolygonTestRule.java │ │ ├── PolyLineTest.java │ │ ├── PolygonCoveringPolygonTest.java │ │ ├── PolygonCoveringPolygonTestRule.java │ │ ├── PolygonTest.java │ │ ├── PolygonTestRule.java │ │ ├── RectangleTest.java │ │ ├── SegmentTest.java │ │ ├── SelfIntersectingPolyLineTestCase.java │ │ ├── SnapperTest.java │ │ ├── StringCompressedPolyLineTest.java │ │ ├── atlas/ │ │ │ ├── AtlasResourceLoaderTest.java │ │ │ ├── AtlasTest.java │ │ │ ├── AtlasTestRule.java │ │ │ ├── BareAtlasTest.java │ │ │ ├── BareAtlasTestRule.java │ │ │ ├── IsAtlasTestCase.java │ │ │ ├── ShardFileOverlapsPolygonTest.java │ │ │ ├── SubAtlasRule.java │ │ │ ├── SubAtlasTest.java │ │ │ ├── builder/ │ │ │ │ ├── GeoJsonAtlasBuilderTest.java │ │ │ │ ├── PackedAtlasBuilderTest.java │ │ │ │ ├── store/ │ │ │ │ │ ├── AtlasPrimitiveObjectStoreTest.java │ │ │ │ │ └── AtlasPrimitiveRouteTest.java │ │ │ │ └── text/ │ │ │ │ └── TextAtlasBuilderTest.java │ │ │ ├── change/ │ │ │ │ ├── AbstractChangeTest.java │ │ │ │ ├── AtlasChangeGeneratorTest.java │ │ │ │ ├── AtlasChangeGeneratorTestRule.java │ │ │ │ ├── CascadeDeleteTest.java │ │ │ │ ├── CascadeDeleteTestHelper.java │ │ │ │ ├── CascadeDeleteTestRule.java │ │ │ │ ├── ChangeAtlasTest.java │ │ │ │ ├── ChangeAtlasTestRule.java │ │ │ │ ├── ChangeBuilderTest.java │ │ │ │ ├── ChangeMergeTest.java │ │ │ │ ├── ChangeTest.java │ │ │ │ ├── FeatureChangeMergerTest.java │ │ │ │ ├── FeatureChangeTest.java │ │ │ │ ├── FeatureChangeUnitTestFactory.java │ │ │ │ ├── MemberMergeStrategiesTest.java │ │ │ │ ├── MultiCascadeDeleteTest.java │ │ │ │ ├── MultiCascadeDeleteTestRule.java │ │ │ │ ├── MultipleChangeAtlasTest.java │ │ │ │ ├── MultipleChangeAtlasTestRule.java │ │ │ │ ├── TagChangeTest.java │ │ │ │ ├── TagChangeTestRule.java │ │ │ │ ├── description/ │ │ │ │ │ └── descriptors/ │ │ │ │ │ └── ChangeDescriptorComparatorTest.java │ │ │ │ ├── diff/ │ │ │ │ │ ├── AtlasDiffTest.java │ │ │ │ │ └── AtlasDiffTestRule.java │ │ │ │ ├── eventhandling/ │ │ │ │ │ └── listener/ │ │ │ │ │ ├── TagChangeListenerTest.java │ │ │ │ │ └── TestTagChangeListenerImplementation.java │ │ │ │ ├── exception/ │ │ │ │ │ └── EmptyChangeExceptionTest.java │ │ │ │ ├── serializer/ │ │ │ │ │ ├── ChangeGeoJsonSerializerTest.java │ │ │ │ │ ├── FeatureChangeGeoJsonSerializerTest.java │ │ │ │ │ └── FeatureChangeGeoJsonSerializerTestRule.java │ │ │ │ └── validators/ │ │ │ │ └── ChangeValidatorTest.java │ │ │ ├── changeset/ │ │ │ │ └── BinaryChangeSetSerializerTest.java │ │ │ ├── command/ │ │ │ │ ├── AtlasFeatureCountsSubCommandTestCase.java │ │ │ │ ├── AtlasFeatureCountsSubCommandTestCaseRule.java │ │ │ │ ├── AtlasFindByAtlasIdentifierSubCommandTest.java │ │ │ │ ├── AtlasJoinerSubCommandTest.java │ │ │ │ ├── AtlasSplitterWithSlippyTileCommandTest.java │ │ │ │ ├── CaptureOutputStream.java │ │ │ │ ├── ComplexBuildingsTestRule.java │ │ │ │ ├── OsmPbfToAtlasSubCommandTest.java │ │ │ │ └── TinyBuildingsSearchSubCommandTest.java │ │ │ ├── complete/ │ │ │ │ ├── CompleteAreaTest.java │ │ │ │ ├── CompleteEdgeTest.java │ │ │ │ ├── CompleteEntityTest.java │ │ │ │ ├── CompleteItemTypeTest.java │ │ │ │ ├── CompleteItemTypeTestRule.java │ │ │ │ ├── CompleteLineTest.java │ │ │ │ ├── CompleteNodeTest.java │ │ │ │ ├── CompletePointTest.java │ │ │ │ ├── CompleteRelationTest.java │ │ │ │ ├── CompleteTestRule.java │ │ │ │ └── EmptyAtlasTest.java │ │ │ ├── delta/ │ │ │ │ ├── AtlasDeltaAreaTest.java │ │ │ │ ├── AtlasDeltaEdgeTest.java │ │ │ │ ├── AtlasDeltaLineTest.java │ │ │ │ ├── AtlasDeltaNodeTest.java │ │ │ │ ├── AtlasDeltaPointTest.java │ │ │ │ ├── AtlasDeltaRelationTest.java │ │ │ │ ├── AtlasDeltaRelationsTest.java │ │ │ │ └── AtlasDeltaTagTest.java │ │ │ ├── dynamic/ │ │ │ │ ├── DynamicAtlasAggressiveRelationsTest.java │ │ │ │ ├── DynamicAtlasFilteredEntitiesTest.java │ │ │ │ ├── DynamicAtlasMovingTooFastTest.java │ │ │ │ ├── DynamicAtlasMultipleInitialShardTest.java │ │ │ │ ├── DynamicAtlasPartialInitialShardsTest.java │ │ │ │ ├── DynamicAtlasPreemptiveLoadTest.java │ │ │ │ ├── DynamicAtlasRestrainedExpansionWithPolygonTest.java │ │ │ │ ├── DynamicAtlasTest.java │ │ │ │ └── rules/ │ │ │ │ ├── DynamicAtlasAggressiveRelationsTestRule.java │ │ │ │ ├── DynamicAtlasMovingTooFastTestRule.java │ │ │ │ ├── DynamicAtlasPartialInitialShardsTestRule.java │ │ │ │ ├── DynamicAtlasPreemptiveLoadTestRule.java │ │ │ │ ├── DynamicAtlasRestrainedExpansionWithPolygonTestRule.java │ │ │ │ └── DynamicAtlasTestRule.java │ │ │ ├── items/ │ │ │ │ ├── AreaEntityTestRule.java │ │ │ │ ├── AreaTest.java │ │ │ │ ├── AreaTestRule.java │ │ │ │ ├── AtlasEntityTest.java │ │ │ │ ├── AtlasItemIntersectionTest.java │ │ │ │ ├── AtlasItemIntersectionTestRule.java │ │ │ │ ├── EdgeTest.java │ │ │ │ ├── EdgeTestRule.java │ │ │ │ ├── ItemTypeTest.java │ │ │ │ ├── LineItemTest.java │ │ │ │ ├── LineItemTestRule.java │ │ │ │ ├── LoopingRelationTest.java │ │ │ │ ├── LoopingRelationTestRule.java │ │ │ │ ├── RelationBeanTest.java │ │ │ │ ├── RelationBeanTestRule.java │ │ │ │ ├── RelationFlatteningRule.java │ │ │ │ ├── RelationFlatteningTest.java │ │ │ │ ├── RelationMemberComparisonTestCase.java │ │ │ │ ├── RelationMemberComparisonTestCaseRule.java │ │ │ │ ├── RelationMemberListTest.java │ │ │ │ ├── RouteTest.java │ │ │ │ ├── RouteTestRule.java │ │ │ │ ├── SnappedLineItemTest.java │ │ │ │ ├── SnappedLineItemTestRule.java │ │ │ │ ├── WithinTest.java │ │ │ │ ├── WithinTestRule.java │ │ │ │ └── complex/ │ │ │ │ ├── RelationOrAreaToMultiPolygonConverterTest.java │ │ │ │ ├── RelationOrAreaToMultiPolygonConverterTestRule.java │ │ │ │ ├── RelationToMultiPolygonMemberConverterTest.java │ │ │ │ ├── aoi/ │ │ │ │ │ ├── ComplexAreaOfInterestFinderTest.java │ │ │ │ │ └── ComplexAreaOfInterestFinderTestRule.java │ │ │ │ ├── bignode/ │ │ │ │ │ ├── BigNodeFinderTest.java │ │ │ │ │ └── BigNodeFinderTestCaseRule.java │ │ │ │ ├── boundaries/ │ │ │ │ │ ├── ComplexBoundaryTest.java │ │ │ │ │ └── ComplexBoundaryTestRule.java │ │ │ │ ├── buildings/ │ │ │ │ │ ├── AtlasComplexBuildingTestCase.java │ │ │ │ │ ├── AtlasComplexBuildingTestCaseRule.java │ │ │ │ │ ├── BuildingHeightTestCase.java │ │ │ │ │ ├── BuildingHeightTestCaseRule.java │ │ │ │ │ ├── BuildingLevelsTestCase.java │ │ │ │ │ ├── BuildingLevelsTestCaseRule.java │ │ │ │ │ ├── BuildingsContainsOsmIdentifierTestCase.java │ │ │ │ │ ├── BuildingsContainsOsmIdentifierTestCaseRule.java │ │ │ │ │ └── HeightConverterTest.java │ │ │ │ ├── highwayarea/ │ │ │ │ │ ├── ComplexHighwayAreaTestCase.java │ │ │ │ │ ├── ComplexHighwayAreaTestCaseRule.java │ │ │ │ │ ├── OutOfOrderEdgesHighwayAreaTestCase.java │ │ │ │ │ ├── OutOfOrderEdgesHighwayAreaTestCaseRule.java │ │ │ │ │ ├── SelfIntersectingHighwayAreaTestCase.java │ │ │ │ │ ├── SelfIntersectingHighwayAreaTestCaseRule.java │ │ │ │ │ ├── ZeroSizeHighwayAreaTestCase.java │ │ │ │ │ └── ZeroSizeHighwayAreaTestCaseRule.java │ │ │ │ ├── islands/ │ │ │ │ │ ├── ComplexIslandFinderTest.java │ │ │ │ │ └── ComplexIslandFinderTestRule.java │ │ │ │ ├── landcover/ │ │ │ │ │ ├── ComplexLandCoverFinderTest.java │ │ │ │ │ └── ComplexLandCoverFinderTestRule.java │ │ │ │ ├── restriction/ │ │ │ │ │ ├── ComplexTurnRestrictionTest.java │ │ │ │ │ └── ComplexTurnRestrictionTestCaseRule.java │ │ │ │ ├── roundabout/ │ │ │ │ │ ├── ComplexRoundaboutTest.java │ │ │ │ │ └── ComplexRoundaboutTestRule.java │ │ │ │ └── water/ │ │ │ │ ├── AbstractWaterIslandTest.java │ │ │ │ ├── ComplexHarborTestRule.java │ │ │ │ ├── ComplexHarbourTest.java │ │ │ │ ├── ComplexWaterEntityTest.java │ │ │ │ ├── ComplexWaterWayTest.java │ │ │ │ └── ComplexWaterWayTestRule.java │ │ │ ├── lightweight/ │ │ │ │ ├── LightAreaTest.java │ │ │ │ ├── LightEdgeTest.java │ │ │ │ ├── LightLineTest.java │ │ │ │ ├── LightNodeTest.java │ │ │ │ ├── LightPointTest.java │ │ │ │ ├── LightRelationTest.java │ │ │ │ └── LightweightTestAtlasRule.java │ │ │ ├── multi/ │ │ │ │ ├── MissingMultiEntityRelationTest.java │ │ │ │ ├── MultiAtlasOverlappingNodesFixerTest.java │ │ │ │ ├── MultiAtlasOverlappingNodesFixerTestRule.java │ │ │ │ ├── MultiAtlasTest.java │ │ │ │ └── MultiAtlasTestRule.java │ │ │ ├── packed/ │ │ │ │ ├── PackedAtlasClonerTest.java │ │ │ │ ├── PackedAtlasSerializerTest.java │ │ │ │ ├── PackedAtlasTest.java │ │ │ │ ├── PackedAtlasTestRule.java │ │ │ │ ├── PackedRelationTest.java │ │ │ │ ├── PackedRelationTestCaseRule.java │ │ │ │ ├── RandomPackedAtlasBuilder.java │ │ │ │ └── RelationMultipolygonGeometryTest.java │ │ │ ├── pbf/ │ │ │ │ ├── BridgeConfiguredFilterTest.java │ │ │ │ ├── OsmPbfIngestTest.java │ │ │ │ ├── OsmPbfNodeToAtlasItemTest.java │ │ │ │ ├── OsmPbfNodeToAtlasItemTestRule.java │ │ │ │ ├── OsmPbfWayToAtlasEdgeTranslationTest.java │ │ │ │ ├── OsmPbfWayToAtlasEdgeTranslationTestRule.java │ │ │ │ ├── OsmosisReaderMock.java │ │ │ │ ├── converters/ │ │ │ │ │ ├── AtlasPrimitiveAreaToOsmosisWayConverter.java │ │ │ │ │ ├── AtlasPrimitiveLineItemToOsmosisWayConverter.java │ │ │ │ │ ├── AtlasPrimitiveLocationItemToOsmosisNodeConverter.java │ │ │ │ │ ├── AtlasPrimitiveRelationToOsmosisRelationConverter.java │ │ │ │ │ ├── ItemTypeToEntityTypeConverter.java │ │ │ │ │ ├── LocationIterableToWayNodeListConverter.java │ │ │ │ │ └── LocationToOsmosisNodeConverter.java │ │ │ │ ├── slicing/ │ │ │ │ │ └── identifier/ │ │ │ │ │ └── ReverseIdentifierFactoryTest.java │ │ │ │ └── store/ │ │ │ │ └── PbfOneWayTest.java │ │ │ ├── raw/ │ │ │ │ ├── RawAtlasTest.java │ │ │ │ ├── creation/ │ │ │ │ │ └── RawAtlasGeneratorTest.java │ │ │ │ ├── sectioning/ │ │ │ │ │ ├── AtlasSectionProcessorTest.java │ │ │ │ │ └── WaySectionProcessorTestRule.java │ │ │ │ └── slicing/ │ │ │ │ ├── RawAtlasSlicerTest.java │ │ │ │ └── RawAtlasSlicerTestRule.java │ │ │ ├── routing/ │ │ │ │ ├── AStarRouterTest.java │ │ │ │ ├── AStarRouterTestRule.java │ │ │ │ ├── AllPathsRouterTest.java │ │ │ │ └── RoutingTestRule.java │ │ │ ├── statistics/ │ │ │ │ ├── AtlasStatisticsTest.java │ │ │ │ ├── AtlasStatisticsTestRule.java │ │ │ │ ├── CounterTest.java │ │ │ │ └── coverage/ │ │ │ │ └── poi/ │ │ │ │ ├── CountCoverageTest.java │ │ │ │ └── CountCoverageTestCaseRule.java │ │ │ ├── validators/ │ │ │ │ ├── AtlasEdgeValidatorTest.java │ │ │ │ ├── AtlasLineItemValidatorTest.java │ │ │ │ ├── AtlasLocationItemValidatorTest.java │ │ │ │ ├── AtlasNodeValidatorTest.java │ │ │ │ ├── AtlasRelationValidatorTest.java │ │ │ │ └── AtlasValidatorTest.java │ │ │ └── walker/ │ │ │ ├── OsmWayWalkerTest.java │ │ │ ├── OsmWayWalkerTestRule.java │ │ │ ├── SimpleEdgeWalkerTest.java │ │ │ └── SimpleEdgeWalkerTestRule.java │ │ ├── boundary/ │ │ │ ├── CountryBoundaryMapTest.java │ │ │ ├── CountryShardListingTest.java │ │ │ ├── CountryToShardListCacheTest.java │ │ │ └── converters/ │ │ │ └── CountryBoundaryMapGeoJsonConverterTest.java │ │ ├── clipping/ │ │ │ ├── GeometryOperationTest.java │ │ │ ├── MultiPolygonClipperTest.java │ │ │ └── PolygonClipperTest.java │ │ ├── converters/ │ │ │ ├── GeodeticEarthCenteredEarthFixedConverterTest.java │ │ │ ├── MultiPolygonStringConverterTest.java │ │ │ ├── MultiplePolyLineToMultiPolygonConverterTest.java │ │ │ ├── MultiplePolyLineToPolygonsConverterCommandTest.java │ │ │ ├── MultiplePolyLineToPolygonsConverterTest.java │ │ │ ├── SlippyTileConverterTest.java │ │ │ ├── WkbMultiPolygonConverterTest.java │ │ │ ├── WkbPolyLineConverterTest.java │ │ │ ├── WkbPolygonConverterTest.java │ │ │ ├── WktMultiPolygonConverterTest.java │ │ │ └── WktPolygonConverterTest.java │ │ ├── coordinates/ │ │ │ └── CoordinatesTest.java │ │ ├── geojson/ │ │ │ ├── ConcatenateGeoJsonCommandTest.java │ │ │ ├── GeoJsonBuilderTest.java │ │ │ ├── GeoJsonUtilsTest.java │ │ │ └── parser/ │ │ │ ├── domain/ │ │ │ │ ├── bbox/ │ │ │ │ │ └── DimensionsTest.java │ │ │ │ └── geometry/ │ │ │ │ └── coordinate/ │ │ │ │ └── CoordinatesTest.java │ │ │ ├── impl/ │ │ │ │ └── jackson/ │ │ │ │ ├── AbstractGeoJsonParserJacksonImplTestBase.java │ │ │ │ ├── GeoJsonParserJacksonImplExtensionsTest.java │ │ │ │ └── GeoJsonParserJacksonImplTest.java │ │ │ └── testdomain/ │ │ │ ├── BeanA.java │ │ │ └── BeanB.java │ │ ├── index/ │ │ │ └── SpatialIndexTest.java │ │ ├── matching/ │ │ │ └── PolyLineMatchTest.java │ │ └── sharding/ │ │ ├── CountryShardTest.java │ │ ├── DynamicTileShardingTest.java │ │ ├── GeoHashShardingTest.java │ │ ├── GeoHashTileTest.java │ │ ├── SlippyTileShardingTest.java │ │ ├── SlippyTileTest.java │ │ └── converters/ │ │ ├── RectangleToSpatial4JRectangleConverterTest.java │ │ └── StringToShardConverterTest.java │ ├── locale/ │ │ ├── IsoCountryFuzzyMatcherTest.java │ │ ├── IsoCountryTest.java │ │ └── IsoLanguageTest.java │ ├── proto/ │ │ ├── FullProtoSuiteTest.java │ │ ├── adapters/ │ │ │ ├── ProtoAtlasMetaDataAdapterTest.java │ │ │ ├── ProtoByteArrayOfArraysAdapterTest.java │ │ │ ├── ProtoIntegerArrayOfArraysAdapterTest.java │ │ │ ├── ProtoIntegerStringDictionaryAdapterTest.java │ │ │ ├── ProtoLongArrayAdapterTest.java │ │ │ ├── ProtoLongArrayOfArraysAdapterTest.java │ │ │ ├── ProtoLongToLongMapAdapterTest.java │ │ │ ├── ProtoLongToLongMultiMapAdapterTest.java │ │ │ ├── ProtoPackedTagStoreAdapterTest.java │ │ │ ├── ProtoPolyLineArrayAdapterTest.java │ │ │ └── ProtoPolygonArrayAdapterTest.java │ │ ├── builder/ │ │ │ └── ProtoAtlasBuilderTest.java │ │ └── converters/ │ │ ├── ProtoIntegerArrayOfArraysConverterTest.java │ │ ├── ProtoLocationConverterTest.java │ │ └── ProtoTagListConverterTest.java │ ├── streaming/ │ │ ├── SplittableInputStreamTest.java │ │ ├── readers/ │ │ │ ├── CsvReaderTest.java │ │ │ └── GeoJsonReaderTest.java │ │ ├── resource/ │ │ │ ├── ByteArrayOutputStreamExceptional.java │ │ │ ├── FileSuffixTestCase.java │ │ │ ├── FileSuffixTestCaseResource.java │ │ │ ├── FileTest.java │ │ │ ├── InputStreamResourceCloseableTest.java │ │ │ ├── OutputStreamWritableResourceCloseableTest.java │ │ │ ├── ResourceTest.java │ │ │ └── zip/ │ │ │ └── ZipResourceTest.java │ │ └── writers/ │ │ └── JsonWriterTest.java │ ├── tags/ │ │ ├── AbstractNameFinderTestCase.java │ │ ├── BarrierTagTestCase.java │ │ ├── BulkNameFinderForcedLocalizableTestCase.java │ │ ├── BulkNameFinderTestCase.java │ │ ├── CheckDateTagTestCase.java │ │ ├── ConstructionDateTagTestCase.java │ │ ├── DestinationTagTestCase.java │ │ ├── DisusedRailwayTagTestCase.java │ │ ├── EstimatedWidthTagTest.java │ │ ├── FerryTagTest.java │ │ ├── GetTagsTestCase.java │ │ ├── HeightTagTestCase.java │ │ ├── HighwayTagTestCase.java │ │ ├── ISOCountryTagTestCase.java │ │ ├── LayerTagTest.java │ │ ├── LocalizedTagNameWithOptionalDateTestCase.java │ │ ├── LocalizedTaggableTestCase.java │ │ ├── NameFinderTestCase.java │ │ ├── OpenDateTagTestCase.java │ │ ├── OpeningDateTagTestCase.java │ │ ├── ProtectClassTagTest.java │ │ ├── RailwayTagTestCase.java │ │ ├── SmoothnessTagTest.java │ │ ├── StandardNameFinderTestCase.java │ │ ├── SyntheticTagTestCase.java │ │ ├── TagTestSuite.java │ │ ├── TaggableTest.java │ │ ├── TemporaryDateOnTagTestCase.java │ │ ├── TestSyntheticTag.java │ │ ├── TurnLaneBackwardTagTestCase.java │ │ ├── TurnLaneForwardTagTestCase.java │ │ ├── TurnLaneTagTestCase.java │ │ ├── TurnTagTestCase.java │ │ ├── annotations/ │ │ │ ├── extraction/ │ │ │ │ ├── AltitudeExtractorTest.java │ │ │ │ ├── LengthExtractorTest.java │ │ │ │ ├── LongExtractorTestCase.java │ │ │ │ ├── OrdinalExtractorTestCase.java │ │ │ │ └── SpeedExtractorTest.java │ │ │ └── validation/ │ │ │ ├── AddressFlatsTagTestCase.java │ │ │ ├── BaseTagTestCase.java │ │ │ ├── BuildingTagTestCase.java │ │ │ ├── DisusedShopTagTestCase.java │ │ │ ├── FromEnumTestCase.java │ │ │ ├── HighwayTagTestCase.java │ │ │ ├── ISOCountryTagTestCase.java │ │ │ ├── LastEditUserIdentifierTestCase.java │ │ │ ├── LayerTagTestCase.java │ │ │ ├── LengthValidatorTest.java │ │ │ ├── LevelTagTestCase.java │ │ │ ├── MaxHeightTagTestCase.java │ │ │ ├── MaxWidthTagTestCase.java │ │ │ ├── OpeningHoursTagTestCase.java │ │ │ ├── SpeedTagsTestCase.java │ │ │ ├── TagValidationTestSuite.java │ │ │ ├── TagValueAsTestCase.java │ │ │ ├── ValidatorsHasValuesForTestCase.java │ │ │ └── ValidatorsTestCase.java │ │ ├── cache/ │ │ │ ├── TaggerTestCase.java │ │ │ └── TaggerTestRule.java │ │ ├── filters/ │ │ │ ├── ConfiguredTaggableFilterTest.java │ │ │ ├── RegexTaggableFilterTest.java │ │ │ ├── TaggableFilterTest.java │ │ │ ├── TaggableFilterToMatcherConverterTest.java │ │ │ └── matcher/ │ │ │ ├── ConfiguredTaggableMatcherTest.java │ │ │ ├── TaggableMatcherTest.java │ │ │ └── parsing/ │ │ │ ├── LexerTest.java │ │ │ ├── ParserTest.java │ │ │ ├── SemanticCheckerTest.java │ │ │ └── tree/ │ │ │ ├── ASTNodeTest.java │ │ │ └── TreePrinterTest.java │ │ └── oneway/ │ │ └── OneWayTagTest.java │ ├── test/ │ │ └── TestUtility.java │ └── utilities/ │ ├── archive/ │ │ └── ExtractorTest.java │ ├── arrays/ │ │ └── LargeArrayTest.java │ ├── caching/ │ │ ├── ConcurrentResourceCacheTest.java │ │ ├── LocalFileInMemoryCacheTest.java │ │ ├── ResourceCacheTest.java │ │ └── strategies/ │ │ ├── ByteArrayCachingStrategyTest.java │ │ ├── NamespaceCachingStrategiesTest.java │ │ └── NoCachingStrategyTest.java │ ├── checkstyle/ │ │ └── ArrangementCheckTest.java │ ├── collections/ │ │ ├── EnumSetCollectorTestCase.java │ │ ├── FilteredIterableTest.java │ │ ├── FixedSizePriorityQueueTest.java │ │ ├── IterablesAddAllTestCase.java │ │ ├── IterablesTest.java │ │ ├── MapsTest.java │ │ ├── MultiIterableTest.java │ │ ├── OptionalIterableTest.java │ │ ├── ParallelIterableTest.java │ │ ├── SetsTest.java │ │ ├── ShardBucketCollectionTest.java │ │ ├── ShardBucketCollectionTestClasses.java │ │ ├── StreamIterableTest.java │ │ ├── StringListTest.java │ │ └── SubIterableTest.java │ ├── command/ │ │ ├── SimpleOptionAndArgumentParserTest.java │ │ └── subcommands/ │ │ ├── AnyToGeoJsonCommandTest.java │ │ ├── AtlasDiffCommandTest.java │ │ ├── AtlasDiffCommandTestRule.java │ │ ├── AtlasMetadataReaderCommandTest.java │ │ ├── AtlasSearchCommandTest.java │ │ ├── AtlasShardingConverterCommandTest.java │ │ ├── AtlasShellToolsDemoCommandTest.java │ │ ├── ConcatenateAtlasCommandTest.java │ │ ├── CountryBoundaryMapPrinterCommandTest.java │ │ ├── CountryShardToBoundsCommandTest.java │ │ ├── HelloWorldCommandTest.java │ │ ├── IsoCountryCodeCommandTest.java │ │ ├── JavaToProtoSerializationCommandTest.java │ │ ├── OsmFileParserCommandTest.java │ │ ├── OsmToAtlasCommandTest.java │ │ ├── PackedToTextAtlasCommandTest.java │ │ ├── PbfToAtlasCommandTest.java │ │ ├── SubAtlasCommandTest.java │ │ ├── TaggableMatcherPrinterCommandTest.java │ │ ├── TemplateTestCommandTest.java │ │ ├── WKTShardCommandTest.java │ │ └── templates/ │ │ ├── AtlasLoaderCommandTest.java │ │ ├── AtlasLoaderTemplateTest.java │ │ ├── MultipleOutputCommandTest.java │ │ └── OutputDirectoryTemplateTest.java │ ├── configuration/ │ │ ├── ConfiguredFilterTest.java │ │ ├── MergedConfigurationTest.java │ │ └── StandardConfigurationTest.java │ ├── conversion/ │ │ ├── HexStringByteArrayConverterTest.java │ │ ├── StringToPredicateConverterTest.java │ │ └── StringToPredicateConverterTestRule.java │ ├── direction/ │ │ ├── EdgeDirectionComparatorTest.java │ │ └── EdgeDirectionComparatorTestCaseRule.java │ ├── filters/ │ │ ├── AtlasEntityPolygonsFilterTest.java │ │ └── AtlasEntityPolygonsFilterTestRule.java │ ├── graphs/ │ │ └── DirectedAcyclicGraphTest.java │ ├── identifiers/ │ │ └── EntityIdentifierGeneratorTest.java │ ├── jsoncompare/ │ │ ├── ArraysRegularExpressionJSONComparatorTestCase.java │ │ ├── DegenerateRegularExpressionJSONComparatorTestCase.java │ │ ├── MatchingRegularExpressionJSONComparatorTestCase.java │ │ ├── ObjectsRegularExpressionJSONComparatorTestCase.java │ │ └── RegularExpressionJSONComparatorTestSuite.java │ ├── maps/ │ │ ├── LargeMapTest.java │ │ └── MultiMapTest.java │ ├── matching/ │ │ └── NameMatcherTestCase.java │ ├── random/ │ │ └── RandomTextGeneratorTest.java │ ├── runtime/ │ │ ├── BooleanFlagTest.java │ │ ├── FlagNameParsingRegressionTest.java │ │ ├── RetryTest.java │ │ ├── RunScriptTest.java │ │ └── system/ │ │ ├── SystemInfoTest.java │ │ └── memory/ │ │ └── MemoryTest.java │ ├── scalars/ │ │ ├── AngleTest.java │ │ ├── DistanceTest.java │ │ ├── DoubleCounterTest.java │ │ ├── DurationTest.java │ │ ├── RatioTest.java │ │ ├── SpeedTest.java │ │ └── SurfaceTest.java │ ├── statistic/ │ │ ├── CounterWithStatisticTest.java │ │ ├── CustomizedStatisticTest.java │ │ └── StatisticUtilsTest.java │ ├── testing/ │ │ ├── AtlasFromTextTestCase.java │ │ ├── AtlasFromTextTestCaseRule.java │ │ ├── AtlasTestCase.java │ │ ├── AtlasTestCaseRule.java │ │ ├── BeanTestCase.java │ │ ├── BeanTestCaseRule.java │ │ ├── CoreTestExtension.java │ │ ├── OsmFileParserTest.java │ │ ├── SomeTestBean.java │ │ ├── TestAtlasTest.java │ │ └── TestAtlasTestRule.java │ ├── threads/ │ │ └── PoolTest.java │ ├── time/ │ │ └── LocalTimeTest.java │ ├── timezone/ │ │ └── TimeZoneTest.java │ ├── tuples/ │ │ ├── EitherTest.java │ │ └── TupleTest.java │ ├── unicode/ │ │ └── LoadingClassifierTestCase.java │ └── vectortiles/ │ └── MinimumZoomTest.java └── resources/ ├── aql-files/ │ ├── test.aql │ ├── test1/ │ │ └── test.aql │ └── test2/ │ ├── test.aql │ ├── testA/ │ │ ├── test.aql │ │ └── test.aql.sig │ └── testB/ │ ├── test.aql │ ├── test.aql.sig │ ├── test2.aql │ └── test2.aql.sig ├── data/ │ ├── Alcatraz/ │ │ ├── Alcatraz.atlas.txt │ │ └── Alcatraz.osm │ ├── ButterflyPark/ │ │ ├── ButterflyPark.atlas.txt │ │ └── ButterflyPark.osm │ └── polygon/ │ └── jsonl/ │ └── samples.jsonl ├── log4j.properties └── org/ └── openstreetmap/ └── atlas/ ├── geography/ │ ├── MultiPolygonTestGeom1.wkt │ ├── MultiPolygonTestGeom10.josm.osm │ ├── MultiPolygonTestGeom2.josm.osm │ ├── MultiPolygonTestGeom3.josm.osm │ ├── MultiPolygonTestGeom4.josm.osm │ ├── MultiPolygonTestGeom5.josm.osm │ ├── MultiPolygonTestGeom6.josm.osm │ ├── MultiPolygonTestGeom7.josm.osm │ ├── MultiPolygonTestGeom8.josm.osm │ ├── MultiPolygonTestGeom9.josm.osm │ ├── atlas/ │ │ ├── ECU_6-16-31.atlas │ │ ├── builder/ │ │ │ └── overpass-turbo.geojson │ │ ├── change/ │ │ │ ├── ChangeAtlasTest.josm.osm │ │ │ ├── ChangeAtlasTestEdge.josm.osm │ │ │ ├── MultipleChangeAtlasTest.atlas.txt │ │ │ ├── MultipleChangeAtlasTest.osm │ │ │ ├── diff/ │ │ │ │ ├── DiffAtlas1.josm.osm │ │ │ │ └── DiffAtlas2.josm.osm │ │ │ ├── nodeBoundsExpansionAtlas.josm.osm │ │ │ └── serializer/ │ │ │ ├── change.json │ │ │ ├── serializedAddEdgeWithDescription.json │ │ │ ├── serializedAddRelationWithDescription.json │ │ │ ├── serializedAreaFull.json │ │ │ ├── serializedAreaNull.json │ │ │ ├── serializedAreaRemove.json │ │ │ ├── serializedEdgeFull.json │ │ │ ├── serializedEdgeNull.json │ │ │ ├── serializedEdgeRemove.json │ │ │ ├── serializedLineFull.json │ │ │ ├── serializedLineNull.json │ │ │ ├── serializedLineRemove.json │ │ │ ├── serializedNodeFull.json │ │ │ ├── serializedNodeNull.json │ │ │ ├── serializedNodeRemove.json │ │ │ ├── serializedPointFull.json │ │ │ ├── serializedPointNull.json │ │ │ ├── serializedPointRemove.json │ │ │ ├── serializedPointWithTags.json │ │ │ ├── serializedRelationFull.json │ │ │ ├── serializedRelationNull.json │ │ │ ├── serializedRelationRemove.json │ │ │ ├── serializedRemoveAreaWithDescription.json │ │ │ ├── serializedReverseWay.json │ │ │ └── serializedUpdateEdgeWithDescription.json │ │ ├── command/ │ │ │ ├── DNK_Copenhagen/ │ │ │ │ ├── DNK_1.atlas.txt │ │ │ │ ├── DNK_2.atlas.txt │ │ │ │ └── DNK_3.atlas.txt │ │ │ ├── atlas-edge.json │ │ │ ├── atlas-way-section.json │ │ │ ├── complex-SF.txt │ │ │ ├── continent_map.txt │ │ │ ├── osm-pbf-node.json │ │ │ ├── osm-pbf-relation.json │ │ │ ├── osm-pbf-way.json │ │ │ └── world_islands.osm.pbf │ │ ├── dynamic/ │ │ │ └── rules/ │ │ │ ├── DynamicAtlasAgressiveRelationsTest_11-998-708.josm.osm │ │ │ ├── DynamicAtlasAgressiveRelationsTest_11-999-708.josm.osm │ │ │ ├── DynamicAtlasMovingTooFastTest.osm │ │ │ ├── DynamicAtlasPartialInitialShardsTest.osm │ │ │ ├── DynamicAtlasPreemptiveLoadTest.osm │ │ │ ├── DynamicAtlasRestrainedExpansionWithPolygonTest.osm │ │ │ ├── z12all.atlas.geojson │ │ │ ├── z12x1349y1869.atlas.geojson │ │ │ ├── z12x1349y1869.geojson │ │ │ ├── z12x1349y1870.atlas.geojson │ │ │ ├── z12x1349y1870.geojson │ │ │ ├── z12x1350y1869.atlas.geojson │ │ │ ├── z12x1350y1869.geojson │ │ │ ├── z12x1350y1870.atlas.geojson │ │ │ └── z12x1350y1870.geojson │ │ ├── items/ │ │ │ ├── complex/ │ │ │ │ ├── InnerOuterMultiPolygon.osm │ │ │ │ ├── bignode/ │ │ │ │ │ ├── dnk-link-road-test.osm │ │ │ │ │ ├── u-turn-shape-edge.osm │ │ │ │ │ └── ukr-link-road-test.osm │ │ │ │ ├── buildings/ │ │ │ │ │ ├── building_block_atlas.txt │ │ │ │ │ ├── building_with_levels.txt │ │ │ │ │ └── building_with_minheights.txt │ │ │ │ ├── restriction/ │ │ │ │ │ └── atlasBrokenTurnRestrictionRoute.josm.osm │ │ │ │ └── water/ │ │ │ │ ├── canalAsRelation.atlas.txt │ │ │ │ ├── canalAsRelationOfCanals.atlas.txt │ │ │ │ ├── harborAsArea.atlas.txt │ │ │ │ └── harborAsRelation.atlas.txt │ │ │ └── intersectionAtlas.atlas.txt │ │ ├── line-delimited-geojson.txt │ │ ├── pbf/ │ │ │ ├── AIA_boundary.txt │ │ │ ├── DMA_boundary.txt │ │ │ ├── DNK_SWE_boundary.txt │ │ │ ├── bridgeConfiguredFilter1.json │ │ │ ├── bridgeConfiguredFilter2.json │ │ │ ├── edge-filter.json │ │ │ ├── ferryRelation5831018.osm │ │ │ ├── noRelationNoTagsAtIntersectionAtlas.osm │ │ │ ├── noRelationNoTagsNoIntersectionAtlas.osm │ │ │ ├── noRelationWithTagsAtIntersectionAtlas.osm │ │ │ ├── noRelationWithTagsNoIntersectionAtlas.osm │ │ │ ├── one_way_roads_in_AIA.osm │ │ │ ├── osmPbfProcessorTest_keepOutsideWaysThatAreConnected.osm │ │ │ ├── outsideConnectedOneWayWays.osm │ │ │ ├── partOfRelationNoTagsAtIntersectionAtlas.osm │ │ │ ├── partOfRelationNoTagsNoIntersectionAtlas.osm │ │ │ ├── partOfRelationWithTagsAtIntersectionAtlas.osm │ │ │ ├── partOfRelationWithTagsNoIntersectionAtlas.osm │ │ │ └── partialRelation4451979.osm │ │ ├── raw/ │ │ │ ├── DMA_boundary.txt │ │ │ ├── DNK_SWE_boundary.txt │ │ │ ├── creation/ │ │ │ │ ├── 7-105-51.osm.pbf │ │ │ │ ├── 9-433-268.osm.pbf │ │ │ │ ├── nestedSingleRelations.josm.osm │ │ │ │ ├── nestedSingleRelations.osm │ │ │ │ └── nestedSingleRelations.osm.pbf │ │ │ ├── osmPbfProcessorTest_keepOutsideWaysThatAreConnected.osm │ │ │ ├── outsideConnectedOneWayWays.osm │ │ │ └── sectioning/ │ │ │ ├── bidirectionalRing.atlas.txt │ │ │ ├── lineWithBarrier.atlas.txt │ │ │ ├── lineWithInvalidOverlappingGeometry.atlas.txt │ │ │ ├── lineWithLessThanTwoNodesDueToRepeatedLocationAtEndOfLine.atlas.txt │ │ │ ├── lineWithLoopAtEnd.atlas.txt │ │ │ ├── lineWithLoopAtStart.atlas.txt │ │ │ ├── lineWithLoopInMiddle.atlas.txt │ │ │ ├── lineWithRepeatedLocation.atlas.txt │ │ │ ├── loopWithRepeatedLocation.atlas.txt │ │ │ ├── loopingWayWithIntersection.atlas.txt │ │ │ ├── malformedPolyLine.atlas.txt │ │ │ ├── malformedPolyLineBoundaryMap.txt │ │ │ ├── nestedRelationRemoval.atlas.txt │ │ │ ├── nestedRelationRemovalBoundaryMap.txt │ │ │ ├── nodeAndPointAsRelationMember.atlas.txt │ │ │ ├── nodeAndPointRelationMemberBoundaryMap.txt │ │ │ ├── oneWayRing.atlas.txt │ │ │ ├── oneWaySimpleLine.atlas.txt │ │ │ ├── pedestrianRing.atlas.txt │ │ │ ├── rawAtlasSpanningOutsideBoundary.atlas.txt │ │ │ ├── reversedOneWayLine.atlas.txt │ │ │ ├── ringWithSingleIntersection.atlas.txt │ │ │ ├── roundAbout.atlas.txt │ │ │ ├── selfIntersectingLoop.atlas.txt │ │ │ ├── simpleBiDirectionalLine.atlas.txt │ │ │ └── wayExceedingSectioningLimit.atlas.txt │ │ ├── statistics/ │ │ │ ├── addressAtlas.josm.osm │ │ │ ├── ferryAtlas.josm.osm │ │ │ ├── refsAtlas.josm.osm │ │ │ └── waterAtlas.josm.osm │ │ └── walker/ │ │ ├── OsmWayWalker-Way169884263.atlas.txt │ │ └── OsmWayWalker-Way30647513.atlas.txt │ ├── boundary/ │ │ ├── AAA_boundary.expected.json │ │ ├── AAA_boundary.txt │ │ ├── DMA_boundary.txt │ │ ├── DMA_snake_polyline.wkt │ │ ├── HTI_boundary.txt │ │ ├── USA_HTI_overlapping.atlas.txt │ │ ├── USA_boundary_reduced.txt │ │ ├── ZAF_osm_shards_with_14.txt │ │ ├── countryToShardList.txt │ │ ├── duplicate_shape.dbf │ │ ├── duplicate_shape.prj │ │ ├── duplicate_shape.qpj │ │ ├── duplicate_shape.shp │ │ ├── duplicate_shape.shx │ │ └── slicing-filter.json │ ├── clipping/ │ │ ├── testHuggingPolygons.josm.osm │ │ ├── testOverlappingPolygonsToMultiPolygon.josm.osm │ │ ├── testOverlappingPolygonsToPolygon.josm.osm │ │ └── testTouchingPolygons.josm.osm │ ├── converters/ │ │ ├── MultiplePolyLineToPolygonsConverterTest_crossingPolyLines.txt │ │ ├── MultiplePolyLineToPolygonsConverterTest_expectedPolygons.txt │ │ ├── MultiplePolyLineToPolygonsConverterTest_jtsErrorInner1.wkt │ │ ├── MultiplePolyLineToPolygonsConverterTest_jtsErrorInner2.wkt │ │ ├── MultiplePolyLineToPolygonsConverterTest_jtsErrorOuter.wkt │ │ └── MultiplePolyLineToPolygonsConverterTest_multiplePolyLines.txt │ ├── fourSelfIntersectingPolygon.osm │ ├── geojson/ │ │ ├── concatenated_geojson_files_expected │ │ ├── parser/ │ │ │ ├── beanA.json │ │ │ ├── beanBWithArray.json │ │ │ ├── beanBWithoutArray.json │ │ │ ├── description1.json │ │ │ ├── descriptor1.json │ │ │ ├── descriptor2.json │ │ │ ├── descriptor3.json │ │ │ ├── feature1.json │ │ │ ├── feature2.json │ │ │ ├── featureChangeProperties.json │ │ │ ├── featureChangePropertiesBad1.json │ │ │ ├── featureChangePropertiesBad2.json │ │ │ ├── featureChangePropertiesExample1.json │ │ │ ├── featureChangePropertiesExample2.json │ │ │ ├── featureChangePropertiesRelationMemberDescriptor.json │ │ │ ├── featureCollection1.json │ │ │ ├── featureWithExtendedProperties.json │ │ │ ├── foreignFieldsNested.json │ │ │ ├── foreignFieldsSimple.json │ │ │ ├── geometryCollectionBasic.json │ │ │ ├── geometryCollectionChildConversion.json │ │ │ ├── geometryCollectionRecursiveNested.json │ │ │ ├── lineString.json │ │ │ ├── lineStringConversion.json │ │ │ ├── multiLineString.json │ │ │ ├── multiLineStringConversion.json │ │ │ ├── multiPoint.json │ │ │ ├── multiPointConversion.json │ │ │ ├── multiPolygon.json │ │ │ ├── multiPolygonConversion.json │ │ │ ├── multiPolygonDonut.json │ │ │ ├── point.json │ │ │ ├── pointConversion.json │ │ │ ├── pointWithBbox2D.json │ │ │ ├── pointWithBbox3D.json │ │ │ ├── polygon.json │ │ │ ├── polygonConversion.json │ │ │ ├── propertiesNested.json │ │ │ └── propertiesSimple.json │ │ ├── test1.geojson │ │ ├── test2.geojson │ │ └── test3.geojson │ ├── polygonWithFlatZeroAreaPart.osm │ ├── polygonWithInclinedZeroArea.osm │ ├── selfIntersectingPolygon.osm │ └── sharding/ │ ├── testDynamicSharding.txt │ ├── testDynamicShardingChildOrdering.txt │ └── testDynamicShardingMissingChildren.txt ├── streaming/ │ └── readers/ │ ├── data.csv │ ├── geojson-multipolygon.json │ ├── geojson-point.json │ ├── geojson-polygon.json │ ├── geojson-sample.json │ └── wrongData.csv ├── tags/ │ └── filters/ │ ├── matcher/ │ │ └── test-matching.json │ └── test-filtering.json └── utilities/ ├── checkstyle/ │ ├── ArrangementCheckRight.java │ ├── ArrangementCheckWrongField0.java │ ├── ArrangementCheckWrongField1.java │ ├── ArrangementCheckWrongField2.java │ ├── ArrangementCheckWrongInitializerBlock.java │ ├── ArrangementCheckWrongInitializerStaticBlock.java │ ├── ArrangementCheckWrongMethodModifier.java │ ├── ArrangementCheckWrongMethodName.java │ ├── ArrangementCheckWrongMethodVisibility1.java │ └── ArrangementCheckWrongMethodVisibility2.java ├── command/ │ └── subcommands/ │ ├── AtlasDiffCommandAtlas1.josm.osm │ ├── AtlasDiffCommandAtlas2.josm.osm │ ├── AtlasDiffCommandAtlas3.josm.osm │ ├── AtlasDiffCommandAtlas4.josm.osm │ ├── CountryBoundaryMapPrinterCommandTestBoundaries.txt │ ├── MAF_AIA_osm_boundaries_with_grid_index.txt │ ├── MultiPolygonTest.josm.osm │ ├── MultiPolygonTest.osm │ ├── expected-json-features.txt │ ├── sharding-tree-1-expected.geojson │ ├── sharding-tree-1.txt │ ├── shardingConverter.josm.osm │ ├── templates/ │ │ ├── atlas1.atlas.txt │ │ ├── atlas2.atlas.txt │ │ └── atlas_malformed.atlas.txt │ ├── testPbf2Atlas.josm.osm │ ├── test_boundary.txt │ └── test_boundary_expected.geojson ├── configuration/ │ ├── ConfiguredFilterTest.json │ ├── ConfiguredFilterTestOtherRoot.json │ ├── application.json │ ├── application.yml │ ├── development.json │ ├── development.yml │ ├── developmentOverriding.json │ ├── developmentOverriding.yml │ ├── feature.json │ ├── feature.yml │ ├── geometryFilter.josm.osm │ ├── keywordOverridingApplication.json │ ├── keywordOverridingApplication.yml │ ├── yaml_dot_compressed.yml │ └── yaml_dot_expanded.yml ├── filters/ │ ├── includeExcludePolygonArrangements.osm │ ├── multiPolygons.osm │ ├── testCounts.osm │ └── testForms.osm └── testing/ ├── josmOsmFile.osm ├── osmFile.osm └── test.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ # Java Gradle CircleCI 2.0 configuration file # # Check https://circleci.com/docs/2.0/language-java/ for more details # version: 2 general: branches: only: - main - dev jobs: build: docker: - image: circleci/openjdk:11-jdk working_directory: ~/repo environment: JVM_OPTS: -Xmx3200m TERM: dumb steps: - checkout # - run: if [ -z "$SONAR_TOKEN" ]; then echo "SONAR_TOKEN missing"; else echo "SONAR_TOKEN is here"; fi # - run: echo $SONAR_TOKEN # Run jar first, as the checkstyle plugin depends on atlas itself here. - run: name: Quality checks (No tests) command: ./gradlew jar check -x test -x integrationTest - run: name: Tests command: ./gradlew check no_output_timeout: 30m # - run: .circleci/sonar.sh # - run: ./gradlew cleanPyatlas buildPyatlas ================================================ FILE: .circleci/sonar.sh ================================================ #!/usr/bin/env sh if [ -z "$CIRCLE_PR_NUMBER" ]; then echo "Running sonarqube in a regular build" echo "sonar.branch.name=$CIRCLE_BRANCH" #TODO: Remove echo below echo ./gradlew jacocoTestReport sonarqube \ -Dsonar.branch.name=$CIRCLE_BRANCH \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml else CIRCLE_PR_BRANCH=`curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER | jq -r '.head.ref'` CIRCLE_PR_TARGET_BRANCH=`curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER | jq -r '.base.ref'` SONAR_PULLREQUEST_BRANCH="$CIRCLE_PR_USERNAME/$CIRCLE_PR_BRANCH" echo "Running sonarqube in Pull Request $CIRCLE_PR_NUMBER" echo "sonar.pullrequest.key=$CIRCLE_PR_NUMBER" echo "sonar.pullrequest.branch=$SONAR_PULLREQUEST_BRANCH" echo "sonar.pullrequest.base=$CIRCLE_PR_TARGET_BRANCH" #TODO: Remove echo below echo ./gradlew jacocoTestReport sonarqube \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml \ -Dsonar.pullrequest.key=$CIRCLE_PR_NUMBER \ -Dsonar.pullrequest.branch=$SONAR_PULLREQUEST_BRANCH \ -Dsonar.pullrequest.base=$CIRCLE_PR_TARGET_BRANCH \ -Dsonar.pullrequest.provider=github \ -Dsonar.pullrequest.github.repository=osmlab/atlas \ -Dsonar.pullrequest.github.endpoint=https://api.github.com/ \ -Dsonar.pullrequest.github.token.secured=$SONAR_PR_DECORATION_GITHUB_TOKEN fi ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ### Description: Add description here. ### Potential Impact: List potential impact to downstream libraries here (ex. atlas-checks, atlas-generator) ### Unit Test Approach: Describe unit tests being added with that PR. ### Test Results: Describe other (non-unit) test results here. ------ In doubt: [Contributing Guidelines](CONTRIBUTING.md) ================================================ FILE: .github/workflow_scripts/decrypt_gpg_key.sh ================================================ #!/bin/sh # Decrypt the file if [ "$MANUAL_RELEASE_TRIGGERED" = "true" ]; then # --batch to prevent interactive command # --yes to assume "yes" for questions gpg --quiet --batch --yes --decrypt --passphrase="$GPG_AES256_PASSPHRASE" \ --output "$GPG_KEY_LOCATION" .github/workflow_data/secret.gpg.aes256 chmod 700 "$GPG_KEY_LOCATION" else echo "Not decrypting key, since MANUAL_RELEASE_TRIGGERED=$MANUAL_RELEASE_TRIGGERED" fi ================================================ FILE: .github/workflow_scripts/deploy.sh ================================================ #!/bin/sh CURRENT_BRANCH=$(echo "$GITHUB_REF" | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "Criterion for Publishing artifacts to Maven Central:" echo "Current Branch (Should be main): $CURRENT_BRANCH" echo "Pull Request (Should be empty): $GITHUB_HEAD_REF" echo "Manual Release Triggered (Should be true): $MANUAL_RELEASE_TRIGGERED" if [ "$CURRENT_BRANCH" = "main" ] && [ -z "$GITHUB_HEAD_REF" ]; then echo "On branch main, not in a Pull Request" if [ "$MANUAL_RELEASE_TRIGGERED" = "true" ]; then echo "Sign, Upload archives to local repo, Upload archives to Sonatype, Close and release repository." ./gradlew publish publishToNexusAndClose #python -m pip install --user --upgrade twine #twine upload ./pyatlas/dist/* else echo "Not publishing artifacts, since MANUAL_RELEASE_TRIGGERED=$MANUAL_RELEASE_TRIGGERED" fi else echo "Not publishing artifacts, since not on branch main, or in a Pull Request" fi ================================================ FILE: .github/workflow_scripts/merge-dev-to-main.sh ================================================ #!/usr/bin/env sh GITHUB_REPO="osmlab/atlas" MERGE_BRANCH=main SOURCE_BRANCH=dev FUNCTION_NAME="merge-$SOURCE_BRANCH-to-$MERGE_BRANCH" CURRENT_BRANCH=$(echo "$GITHUB_REF" | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "$FUNCTION_NAME: $GITHUB_REPO" echo "$FUNCTION_NAME: CURRENT_BRANCH = $CURRENT_BRANCH" echo "$FUNCTION_NAME: GITHUB_HEAD_REF = $GITHUB_HEAD_REF" if [ "$CURRENT_BRANCH" != "$SOURCE_BRANCH" ]; then echo "$FUNCTION_NAME: Exiting! Branch is not $SOURCE_BRANCH: ($CURRENT_BRANCH)" exit 0; fi if [ -n "$GITHUB_HEAD_REF" ]; then PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "$FUNCTION_NAME: Exiting! This is a Pull Request: $PULL_REQUEST_NUMBER" exit 0; fi : ${MERGE_TAG_GITHUB_SECRET_TOKEN:?"MERGE_TAG_GITHUB_SECRET_TOKEN needs to be set in the workflow yml file!"} : ${GITHUB_SHA:?"GITHUB_SHA needs to be available to tag the right commit!"} TEMPORARY_REPOSITORY=$(mktemp -d) git clone "https://github.com/$GITHUB_REPO" "$TEMPORARY_REPOSITORY" cd "$TEMPORARY_REPOSITORY" echo "Checking out $SOURCE_BRANCH" git checkout $SOURCE_BRANCH git checkout -b tomerge "$GITHUB_SHA" echo "Checking out $MERGE_BRANCH" git checkout "$MERGE_BRANCH" echo "Merging temporary branch tomerge ($GITHUB_SHA) from $SOURCE_BRANCH into $MERGE_BRANCH" git merge --ff-only "tomerge" echo "Pushing to $GITHUB_REPO" # Redirect to /dev/null to avoid secret leakage git push "https://$MERGE_TAG_GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" "$MERGE_BRANCH" > /dev/null 2>&1 ================================================ FILE: .github/workflow_scripts/sonar.sh ================================================ #!/usr/bin/env sh if [ -n "$GITHUB_HEAD_REF" ]; then PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "Running sonarqube in Pull Request $PULL_REQUEST_NUMBER" echo "sonar.pullrequest.key=$PULL_REQUEST_NUMBER" echo "sonar.pullrequest.branch=$GITHUB_HEAD_REF" echo "sonar.pullrequest.base=$GITHUB_BASE_REF" ./gradlew jacocoTestReport sonarqube \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login="$SONAR_TOKEN" \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml \ -Dsonar.pullrequest.key="$PULL_REQUEST_NUMBER" \ -Dsonar.pullrequest.branch="$GITHUB_HEAD_REF" \ -Dsonar.pullrequest.base="$GITHUB_BASE_REF" \ -Dsonar.pullrequest.provider=github \ -Dsonar.pullrequest.github.repository=osmlab/atlas \ -Dsonar.pullrequest.github.endpoint=https://api.github.com/ \ -Dsonar.pullrequest.github.token.secured="$SONAR_PR_DECORATION_GITHUB_TOKEN" else CURRENT_BRANCH=$(echo "$GITHUB_REF" | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "Running sonarqube in branch $CURRENT_BRANCH" ./gradlew jacocoTestReport sonarqube \ -Dsonar.branch.name="$CURRENT_BRANCH" \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login="$SONAR_TOKEN" \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml fi ================================================ FILE: .github/workflow_scripts/tag-main.sh ================================================ #!/usr/bin/env sh GITHUB_REPO="osmlab/atlas" RELEASE_BRANCH=main FUNCTION_NAME="tag-$RELEASE_BRANCH" CURRENT_BRANCH=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "$FUNCTION_NAME: $GITHUB_REPO" echo "$FUNCTION_NAME: CURRENT_BRANCH = $CURRENT_BRANCH" echo "$FUNCTION_NAME: GITHUB_HEAD_REF = $GITHUB_HEAD_REF" if [ "$CURRENT_BRANCH" != "$RELEASE_BRANCH" ]; then echo "$FUNCTION_NAME: Exiting! Branch is not $RELEASE_BRANCH: ($CURRENT_BRANCH)" exit 0; fi if [ -n "$GITHUB_HEAD_REF" ]; then PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "$FUNCTION_NAME: Exiting! This is a Pull Request: $PULL_REQUEST_NUMBER" exit 0; fi if [ "$MANUAL_RELEASE_TRIGGERED" != "true" ]; then echo "$FUNCTION_NAME: Exiting! This is not a release build." exit 0; fi : ${MERGE_TAG_GITHUB_SECRET_TOKEN:?"MERGE_TAG_GITHUB_SECRET_TOKEN needs to be set in the workflow yml file!"} : ${GITHUB_SHA:?"GITHUB_SHA needs to be available to tag the right commit!"} export GIT_COMMITTER_EMAIL="github-actions@github.com" export GIT_COMMITTER_NAME="Github Actions CI" TEMPORARY_REPOSITORY=$(mktemp -d) git clone "https://github.com/$GITHUB_REPO" "$TEMPORARY_REPOSITORY" cd "$TEMPORARY_REPOSITORY" echo "Checking out $RELEASE_BRANCH (commit $GITHUB_SHA)" git checkout $GITHUB_SHA PROJECT_VERSION=$(cat gradle.properties | grep "\-SNAPSHOT" | awk -F '=' '{print $2}' | awk -F '-' '{print $1}') : ${PROJECT_VERSION:?"PROJECT_VERSION could not be found."} echo "Tagging $RELEASE_BRANCH (commit $GITHUB_SHA) at version $PROJECT_VERSION" git tag -a $PROJECT_VERSION -m "Release $PROJECT_VERSION" echo "Pushing tag $PROJECT_VERSION to $GITHUB_REPO" # Redirect to /dev/null to avoid secret leakage git push "https://$MERGE_TAG_GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" "$PROJECT_VERSION" > /dev/null 2>&1 ================================================ FILE: .github/workflow_scripts/update_project_version.sh ================================================ #!/bin/sh CURRENT_BRANCH=$(echo "$GITHUB_REF" | awk 'BEGIN { FS = "/" } ; { print $3 }') echo "Current Branch: $CURRENT_BRANCH" if [ "$CURRENT_BRANCH" = "main" ] && [ "$MANUAL_RELEASE_TRIGGERED" = "true" ]; then echo "This is a manual release, change the version locally to remove the -SNAPSHOT" sed -i "s/-SNAPSHOT//g" gradle.properties else echo "Not a manual release, keeping -SNAPSHOT" fi ================================================ FILE: .github/workflows/ci.yml ================================================ name: Continuous Integration on: push: branches: [ main, dev ] pull_request: branches: [ dev ] workflow_dispatch: inputs: MANUAL_RELEASE_TRIGGERED: description: "Environment Variable used to trigger a Maven Central release" required: false default: "true" jobs: build: runs-on: ubuntu-latest steps: # Setup - uses: actions/checkout@v2 - name: Set up JDK 11 uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: 11 # - name: Set up Python # uses: actions/setup-python@v2 # with: # python-version: 3.7 # - name: Install Virtualenv # run: pip install virtualenv - name: Install GEOS run: sudo apt-get install libgeos-dev - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Grant execute permission for Github Actions Workflows run: chmod -R ug+x .github/workflow_scripts - name: For release builds, remove -SNAPSHOT env: MANUAL_RELEASE_TRIGGERED: ${{ github.event.inputs.MANUAL_RELEASE_TRIGGERED }} run: .github/workflow_scripts/update_project_version.sh # Quality Checks # - name: ShellCheck PyAtlas # run: shellcheck pyatlas/*.sh # - name: Build PyAtlas # run: ./gradlew cleanPyatlas buildPyatlas - name: Quality checks (No tests) run: ./gradlew jar check -x test -x integrationTest - name: Tests run: ./gradlew check build # - name: Sonar # env: # SONAR_TOKEN: 374d4e512b90257ba50c21c37202ee01af40c6a0 # SONAR_PR_DECORATION_GITHUB_TOKEN: ${{ secrets.SONAR_PR_DECORATION_GITHUB_TOKEN }} # run: .github/workflow_scripts/sonar.sh # Merge to Main - name: Merge dev to main env: MERGE_TAG_GITHUB_SECRET_TOKEN: ${{ secrets.MERGE_TAG_GITHUB_SECRET_TOKEN }} run: .github/workflow_scripts/merge-dev-to-main.sh # Sign and Publish - name: Decrypt GPG key (To sign artifacts) env: GPG_KEY_LOCATION: secring.gpg GPG_AES256_PASSPHRASE: ${{ secrets.GPG_AES256_PASSPHRASE }} MANUAL_RELEASE_TRIGGERED: ${{ github.event.inputs.MANUAL_RELEASE_TRIGGERED }} run: .github/workflow_scripts/decrypt_gpg_key.sh - name: Sign and Upload Archives env: GPG_KEY_LOCATION: secring.gpg GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} MANUAL_RELEASE_TRIGGERED: ${{ github.event.inputs.MANUAL_RELEASE_TRIGGERED }} run: .github/workflow_scripts/deploy.sh - name: Tag main branch env: MANUAL_RELEASE_TRIGGERED: ${{ github.event.inputs.MANUAL_RELEASE_TRIGGERED }} MERGE_TAG_GITHUB_SECRET_TOKEN: ${{ secrets.MERGE_TAG_GITHUB_SECRET_TOKEN }} run: .github/workflow_scripts/tag-main.sh ================================================ FILE: .gitignore ================================================ **/build/ **/bin/ .classpath .project .settings .checkstyle .gradle .out .dist *.ipr *.iml *.iws *.ws~ .idea/ /tags /build/ .DS_Store **/.DS_Store **/out/ /bin/ **/ignored/ src/generated/ **/gradle/ gradlew* # pyatlas stuff *_pb2.py *.pyc pyatlas/dist/ pyatlas/pyatlas.egg-info/ __*venv__ pyatlas/protoc pyatlas/doc/*.html pyatlas/LICENSE ================================================ FILE: .travis/build-pyatlas-gate.sh ================================================ #!/usr/bin/env sh if [ $TRAVIS_TEST_RESULT -eq 0 ]; then ./gradlew cleanPyatlas buildPyatlas RETURN_VALUE=$? if [ "$RETURN_VALUE" != "0" ]; then exit $RETURN_VALUE fi fi ================================================ FILE: .travis/build.sh ================================================ #!/usr/bin/env sh chmod u+x gradlew if [ "$MANUAL_RELEASE_TRIGGERED" = "true" ]; then # This is a release job, triggered manually # Change the version locally to remove the -SNAPSHOT sed -i "s/-SNAPSHOT//g" gradle.properties echo "This is a manual release!" else echo "Not a manual release" fi if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then ./gradlew clean build else ./gradlew clean build fi ================================================ FILE: .travis/deploy-gate.sh ================================================ #!/usr/bin/env sh if [ $TRAVIS_TEST_RESULT -eq 0 ]; then .travis/deploy.sh RETURN_VALUE=$? if [ "$RETURN_VALUE" != "0" ]; then exit $RETURN_VALUE fi fi ================================================ FILE: .travis/deploy.sh ================================================ #!/usr/bin/env sh if [ "$TRAVIS_BRANCH" = "main" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then if [ "$MANUAL_RELEASE_TRIGGERED" = "true" ]; then echo "Sign, Upload archives to local repo, Upload archives to Sonatype, Close and release repository." ./gradlew uploadArchives publishToNexusAndClose #python -m pip install --user --upgrade twine #twine upload ./pyatlas/dist/* fi fi ================================================ FILE: .travis/install.sh ================================================ #!/usr/bin/env sh if [ "$TRAVIS_BRANCH" = "main" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then openssl aes-256-cbc -K $encrypted_7b323cc104d6_key -iv $encrypted_7b323cc104d6_iv -in $ENCRYPTED_GPG_KEY_LOCATION -out $GPG_KEY_LOCATION -d fi ================================================ FILE: .travis/merge-dev-to-main-gate.sh ================================================ #!/usr/bin/env sh if [ $TRAVIS_TEST_RESULT -eq 0 ]; then .travis/merge-dev-to-main.sh RETURN_VALUE=$? if [ "$RETURN_VALUE" != "0" ]; then exit $RETURN_VALUE fi fi ================================================ FILE: .travis/merge-dev-to-main.sh ================================================ #!/usr/bin/env sh GITHUB_REPO="osmlab/atlas" MERGE_BRANCH=main SOURCE_BRANCH=dev FUNCTION_NAME="merge-$SOURCE_BRANCH-to-$MERGE_BRANCH" echo "$FUNCTION_NAME: $GITHUB_REPO" echo "$FUNCTION_NAME: TRAVIS_BRANCH = $TRAVIS_BRANCH" echo "$FUNCTION_NAME: TRAVIS_PULL_REQUEST = $TRAVIS_PULL_REQUEST" if [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then echo "$FUNCTION_NAME: Exiting! Branch is not $SOURCE_BRANCH: ($TRAVIS_BRANCH)" exit 0; fi if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then echo "$FUNCTION_NAME: Exiting! This is a Pull Request: $TRAVIS_PULL_REQUEST" exit 0; fi : ${GITHUB_SECRET_TOKEN:?"GITHUB_SECRET_TOKEN needs to be set in .travis.yml!"} : ${TRAVIS_COMMIT:?"TRAVIS_COMMIT needs to be available to merge the right commit to main!"} TEMPORARY_REPOSITORY=$(mktemp -d) git clone "https://github.com/$GITHUB_REPO" "$TEMPORARY_REPOSITORY" cd $TEMPORARY_REPOSITORY echo "Checking out $SOURCE_BRANCH" git checkout $SOURCE_BRANCH git checkout -b tomerge $TRAVIS_COMMIT echo "Checking out $MERGE_BRANCH" git checkout $MERGE_BRANCH echo "Merging temporary branch tomerge ($TRAVIS_COMMIT) from $SOURCE_BRANCH into $MERGE_BRANCH" git merge --ff-only "tomerge" echo "Pushing to $GITHUB_REPO" # Redirect to /dev/null to avoid secret leakage git push "https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" $MERGE_BRANCH > /dev/null 2>&1 ================================================ FILE: .travis/sonar-gate.sh ================================================ #!/usr/bin/env sh if [ $TRAVIS_TEST_RESULT -eq 0 ]; then .travis/sonar.sh RETURN_VALUE=$? if [ "$RETURN_VALUE" != "0" ]; then exit $RETURN_VALUE fi fi ================================================ FILE: .travis/sonar.sh ================================================ #!/usr/bin/env sh if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then SONAR_PULLREQUEST_BRANCH="$(echo $TRAVIS_PULL_REQUEST_SLUG | awk '{split($0,a,"/"); print a[1]}')/$TRAVIS_PULL_REQUEST_BRANCH" echo "Running sonarqube in Pull Request $TRAVIS_PULL_REQUEST" echo "sonar.pullrequest.key=$TRAVIS_PULL_REQUEST" echo "sonar.pullrequest.branch=$SONAR_PULLREQUEST_BRANCH" echo "sonar.pullrequest.base=$TRAVIS_BRANCH" ./gradlew jacocoTestReport sonarqube \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml \ -Dsonar.pullrequest.key=$TRAVIS_PULL_REQUEST \ -Dsonar.pullrequest.branch=$SONAR_PULLREQUEST_BRANCH \ -Dsonar.pullrequest.base=$TRAVIS_BRANCH \ -Dsonar.pullrequest.provider=github \ -Dsonar.pullrequest.github.repository=osmlab/atlas \ -Dsonar.pullrequest.github.endpoint=https://api.github.com/ \ -Dsonar.pullrequest.github.token.secured=$SONAR_PR_DECORATION_GITHUB_TOKEN else echo "Running sonarqube in a regular build" ./gradlew jacocoTestReport sonarqube \ -Dsonar.branch.name=$TRAVIS_BRANCH \ -Dsonar.organization=osmlab \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.junit.reportPaths=build/test-results/test \ -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml fi ================================================ FILE: .travis/tag-main-gate.sh ================================================ #!/usr/bin/env sh if [ $TRAVIS_TEST_RESULT -eq 0 ]; then .travis/tag-main.sh RETURN_VALUE=$? if [ "$RETURN_VALUE" != "0" ]; then exit $RETURN_VALUE fi fi ================================================ FILE: .travis/tag-main.sh ================================================ #!/usr/bin/env sh GITHUB_REPO="osmlab/atlas" RELEASE_BRANCH=main FUNCTION_NAME="tag-$RELEASE_BRANCH" echo "$FUNCTION_NAME: $GITHUB_REPO" echo "$FUNCTION_NAME: TRAVIS_BRANCH = $TRAVIS_BRANCH" echo "$FUNCTION_NAME: TRAVIS_PULL_REQUEST = $TRAVIS_PULL_REQUEST" if [ "$TRAVIS_BRANCH" != "$RELEASE_BRANCH" ]; then echo "$FUNCTION_NAME: Exiting! Branch is not $RELEASE_BRANCH: ($TRAVIS_BRANCH)" exit 0; fi if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then echo "$FUNCTION_NAME: Exiting! This is a Pull Request: $TRAVIS_PULL_REQUEST" exit 0; fi if [ "$MANUAL_RELEASE_TRIGGERED" != "true" ]; then echo "$FUNCTION_NAME: Exiting! This is not a release build." exit 0; fi : ${GITHUB_SECRET_TOKEN:?"GITHUB_SECRET_TOKEN needs to be set in .travis.yml!"} : ${TRAVIS_COMMIT:?"TRAVIS_COMMIT needs to be available to tag the right commit!"} export GIT_COMMITTER_EMAIL="travis@travis.org" export GIT_COMMITTER_NAME="Travis CI" TEMPORARY_REPOSITORY=$(mktemp -d) git clone "https://github.com/$GITHUB_REPO" "$TEMPORARY_REPOSITORY" cd $TEMPORARY_REPOSITORY echo "Checking out $RELEASE_BRANCH (commit $TRAVIS_COMMIT)" git checkout $TRAVIS_COMMIT PROJECT_VERSION=$(cat gradle.properties | grep "\-SNAPSHOT" | awk -F '=' '{print $2}' | awk -F '-' '{print $1}') : ${PROJECT_VERSION:?"PROJECT_VERSION could not be found."} echo "Tagging $RELEASE_BRANCH (commit $TRAVIS_COMMIT) at version $PROJECT_VERSION" git tag -a $PROJECT_VERSION -m "Release $PROJECT_VERSION" echo "Pushing tag $PROJECT_VERSION to $GITHUB_REPO" # Redirect to /dev/null to avoid secret leakage git push "https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" $PROJECT_VERSION > /dev/null 2>&1 ================================================ FILE: .travis/trigger-release.sh ================================================ #!/usr/bin/env sh # Use Travis to trigger a release from Main GITHUB_ORGANIZATION=osmlab GITHUB_REPOSITORY_NAME=atlas # Assumptions # - This is called from the root of the project # - The travis client is installed: gem install travis # - travis login --org has been called to authenticate TRAVIS_PERSONAL_TOKEN=$(travis token) : ${TRAVIS_PERSONAL_TOKEN:?"TRAVIS_PERSONAL_TOKEN needs to be set to access the Travis API to trigger the build"} body=' { "request": { "branch": "main", "config": { "before_script": "export MANUAL_RELEASE_TRIGGERED=true" } } }' curl -s -X POST \ -H "Content-Type: application/json" \ -H "Accept: application/json" \ -H "Travis-API-Version: 3" \ -H "Authorization: token $TRAVIS_PERSONAL_TOKEN" \ -d "$body" \ https://api.travis-ci.org/repo/$GITHUB_ORGANIZATION%2F$GITHUB_REPOSITORY_NAME/requests ================================================ FILE: .travis.yml ================================================ env: global: - # GITHUB_SECRET_TOKEN - secure: "hYLV25stwJ4GX1BXlr7AOdeLHDCE7z395ibQAo0NDCnCqDZdoD3ySZAJteWxuBu3kpWQ1QUsCmOMn3A8m3DzqZnWQB4Hrp0aY/d5oC50LFFBbuBvuj8gqcog1t80fLAKDNNGlSxjYDPOlBVkHwez706zun0frmy3zHJ3TnfhM66CUnZ0UlKDzHwpXSplEHXJKARIX94A4O0aBsBusoHpbLYC1jqhQJWjH1D/gEcxqd2CNDRDge/xK4BZp9YiiQ1b9JSmw5D2fc1F2rXFv7FWWc9YoJt0HFGxJezVw+kmLEEJH3rk6TNxYPobsTYkwsoie6yKhwWN9xCBVmIXAzhFxjmdd02hlyAuNuK49qDKzFcFxlie3cgjxHFBdfKI/FY5jyRhcvS+IiV02qdNb+g9+Mfwz4SXFbJGUwiSM7VG/ezKxxMi9vjSisfeHSPrVNNDEbS2Nqdu/9zF1IjTsYTY6jTOMBM7hx+u0UpDEA09S280w0wZw+lym9RO3HGQvyZOEFYOQB1/IBvzDvOmWs+IrRFxgKytiZGoHMAZBL9uhYq0arqIs7GcD0ULtWRAba4X7Z1ll2kuIfUe1MruLpEGdABVLxUOwUdFLQwOl6WGL+paCSqXMghpXxP0Lfo8onjI7hLhu1bWdYQrcvnZnyQ1AwxilYjKkZc8wf9KsKeZrpk=" - # SONATYPE_USERNAME - secure: "E4anu/qe0DoKRrC6eHqYu6jYb69N5xgGnsb/yq4Fjb/6MPH3Hqe3VIny+iC/TmmG5qSeQR8FG5FM/Q9CAi0lb1kT8/UgRJdy81cfsmq1kIqhv7I1xxmwIHXLwtc14Bfjw2yhKobjCSQ4f3BhCX7H1q0GLTQRalg7PDUXLIP7Za5CWtKv7XH7Uywfz0f2pOHQ72Leqb/k93ZHGAUNHWVce++CzP+MupVQqTkR15I3DRcxH3IVj/pKGDg5u7xyDQ34lXQNSn6O19Ahbp/a2EnHjoIGSzt5pUjGtijVUqbUG++d0F9avqXhrvknADbSziUn/WaxUGypwfBXFYQ7pzZq1Wb41dngMmyszuSFuDYkLshm9aggJY+/fxSa1jKzWxi/0ArMA3dbbEHNEPjoE7Sp2bsK2yFJrx6R5F9qlNW8D7dvrMU4mx9ykJAce/6aD3l4QdZbmOYlaxdtMaf7DsRxM86LAzEiu7c+FrWqwdroR9TvWq6YYLb4Lwv5LotmhN9i68CPEq/ohiPRHhoD66WXJlEYirVd5o26z02IFWwxCvYP5yZM+3e7pXPM7WGvBwkvqM1kMmJ0NLSZAZr4gxv+AS3t4fvwrxWEVOMC9N7DVr3m9KiwD92OEP0tKY8N727/fr+ybfvPM6bgqGFKu4WDHZWSz27PWvckLhZ+PLBUoMM=" - # SONATYPE_PASSWORD - secure: "sZ3l0cznOkcMF+z7TE0SELgXNmF+RSPl+HX/kNEvVU3vDNNztbGTuaZsGZIQEYvw3FpTzGawMQB+yU+HCVlMOgllvQM7H5X/GYRrWlUNomlx8Flpc0GImz8HCUiJLtZyYO/1nFetmZmPDkMY0MgPstlduNz//+K/c+RzhGIEmNZ0FzAzdNArlgjcWMWKUARcUTf2uwb4rVvkYOhX2RcfcgZ+BJMPYR4svHV8F0U8PiHqn8DOcYAeLmlMVOZ3f3fpYFHQN9tWGoVtdMEMCPBlUcVq3/L9gp4mhj3YE7OTOAYtjfl14WktIGZ2wK2nvNHwsYqx7DeEYg/Gswap0/8xEJhf5gW5Gd6sTIVfpeLojaYacHEcLK/olqDevqKaFxxnN3hUlS6M7k8D9n6j0ZxMYKFDMja8/OduOccs69wz7/tN2RPiqWH10wh+wKDJx/vjvyq5ozmKfh3ebZn87l9hWtDGi53ZRpfqv3GFce/29kOyj0YwTYuIPG10/w6wsSoRqVl4t/WXgAPWMmodLbo9PqvAjVJuFxbfWv60UICg8zmPadc0+t4GkfE7VXI+46jU9ZGoWAWuf4CzmGhsuDkbBh1rhg1R6Xdg8nupWGuDE1rEfEHr6yhq1hcXxxnEXoZZChbRqabjlUDnXtBpCjjimjq4a+LEgWyufCG95EaWthc=" - # GPG_KEY_ID - secure: "ow4WEQXjjCi8hBfLNnhtuOorOn0k//K3QBGALcwynDpJIOxwvrV2jJPcNjoQ4aNmeE5a+zl/JomtUBz0mASe9j/JTGhhqrYC1Tpgoo3naEvFdOaucaTG2RV3u2jv0UdQm+wU4APf9t2HbzrZaa+SDMJwF/qpgH2gUE6v0wEQQyZJvuk9EKDsZL+dZkM5UlSGFJ1bUzgvHPEVXrj/gsh0NOQ7BUdjwfIpxKytUXLmQWs8NkP5ptygeifpaJ+D7IQ5HTJCthZ/MuInAIkD0bDPc0nbjhZSriZzCOUlhLDxb7F/iHUomQhm/EvcvMOmCtyDuaEZZxOEK2+lpOcoaMvRj/cXqY2tLQJgjgC95Ka3GvB7m7UY1QCoRWePUrHAbcZYCugD4vx5y8GJqxdBgtOMbI5blzSsaY7D/knXv2lAp8JBPylrNQJQPty1gl7Hp6CFwa1WkKh7C3mBl2a8oyYXlKIqd+NSGQGbWoy+ULv77tH15L2KwTdxYllQY2P1xj+k6CsEmRfTK8lW7c7AhbmHsW5gqGjK3rFThTPurwaJ92rsc4AusiW3uor3/z13zuHH+kgH58dc6h21lmouchvw4177xxj5aPjA09puRikGhEr2SyVwu2h1HU5+l/YwVOEMJZ6oDfYwug4ktmPjFRvF+PEo/Dw6kCBcCerVh7A0EL8=" - # GPG_PASSPHRASE - secure: "gGAS4pDztf35LwEAj7tuA5rYuh9F3lFqUJ/jW+AkIeS4yaQVB5xvj3Wx6pqFpQyGvr0XYmUYDKTfPxHyXI6Rsr2JEtYHRcYhJXWYmTmfX/rzflN7yvmbkq1Io9UA7q28ZqCirrEoigT+sI6jA5hOdvmYX+kUAju5U1moMDRRn9GGOnoE1rB74mnpo4Pl1OvMk0XTZSMMAnTG2FmpQMBVJkJFz81e+xUGcUa0X25U3nLVvnVf2gwbC0Mhi87ITlH4twzsEBr5nJJNrlidDca6OCWv96pKB1VI1/dgVOBUmQudMKxeAYgwxPepyC1jpMyABC+JWQAvUFu275c/hNPTs5pUdvw6FSEgELjkPwoLjamSLCO8NIfymIXurm7c+3z8FZeA+ny3XWvC7loREh8XpzUoAxzbZP3h9GqDhLljtIdquZp2sEVMRfMCGoe5Gb1YPRlDYPQxgTk2SlZMwjPQitv9wF3QFWSbL3p/f/0qYkOSYYb0XshMu9Q+66kkpLflY9XIXEqAylODXTAjfIOIXQxO+rwBapEDNAhtFK2gSdjztwZDm3cF+V4HNonLstvSHGKUl5ki2f7+/SdzizwLY6RQ1ufwuhRt0hkeUHK+AYOaq2J0yduEJKzMr4PJUPb2r/F6ecplE0vPOXu93E63+E57NbeoB6ESV7YTx1sCnEY=" - # TWINE_USERNAME - secure: "IP7AYNIUI5BtUF5I2/8I+JaAERe9UKUEKrxB8BTvndpC3wF+FlZyzbESl2DUgGV/8JiIAbpHnuAlR0Tm/R0NdFMtqy+QwBAuVkGCI3UPrZ4uWiQGWqnZ/wqH+uX0zZAXNvikvfz2lJfJyHpggIQWegn1MN75ypqvr1MgzpOKHF1RPcFL+MkW8GmfO6S1YMm0+FVffv5DRagZ8ouzS0HwW8t1UfRhVFCanUafgdDt8s3yvhKCMnjq2Q5of+yr6Vcr3wH3IUWF+6rMI8fLo9yr3bG3gP2GnVfxq7JtvgZH7GpZ+5PH9D9Cg7HoWb2Ggt2edFyCRCTYKgqivYUruzd8V9k/a6+JmWByi5QqZSCSQ/ZZOxF7yHz+5vKOwYHJjBg+4qq+CIN+o3cKorqLQ3UFPA8Oaat8WPdB5inAyzfVha5RXP5pucwYTFtnZAojTImYEXKGCK9x7oIjJFoLlVkZO8Wa7GuguG0IloM6tACL4uWognEUMHXADKmjWohcgcwa71PA3I2ZIbdIkoVjjD9RofWfImG/vVu/im3z2BDiuYYbg0kCXn23PD9aPTlJWsJ2XPLARjQlBWrSkvbaX233VLzKKQH3wHq6O1+qBYEFqILT1kBUJfEBseJfMBCwwJajGGDxswZ63n2sKC3UI+WBRZynNgkxhDLqwZKjR9VpxAk=" - # TWINE_PASSWORD - secure: "Nymg9duf3qwC6BAZqGTtoxmYWa5qy/FbTk8Rqy3GAa/YxtfyOHqClwCzY454cO5pIs5K1Z72Y8C/EpzsK3OHVDRQDNYpkJbhvYrHh7lCfl5EsuGQxp8eieXdAlwaDcDCjHIwcZXMaxZqQTKfMoJHzNQf3ktzjBXnUS6d7MygPIVcfCCIpFM40K/qVGvVY+nF2nakd/uveEifEENnts43PfMUr7aXo4WtrCNTOq9OFATnAOMNn/dt0Qx8Rx5HBvfLeXDBUUCHY2Bnc33WTCQWu2uP6KceGtKotWnFhuVxMNbxBPzRmGmiBYzoFydlnYB/t7UgLUJK1r9y+F0SB7mlK1kU6jNW1VYm/1gkNujzxQhKr8Z5CsB+K3ytxFG1W5OMGCfiZYVtz6C2scbEW6HJaogS/7yTPd4/tahxE/7Pf9Aa6RppcZsrz88kHKThkPxzAbgslMLieCIJFI+kkIrr02om3kSSSJq9XfW33YC5FdUpZbm/1DqkFxvm5sawwu8u1I951OgsYbsJWRuquWedeDnxen7fijeDizYsL9J7PcpgsPXr8gNOFbyTD8dCP2fpLi35iRqvjJOqspQ5wiLprUB5Hoj/La/XloCqws+8cjGw4IP/6dmRrimyuEs6g5RsIMAWUhpdhZm95sfe09sSKtXmMj/g/eYrTgxZbED7Hd8=" - # SONAR_TOKEN - secure: "F7d2WgJMrUe6GMGZaob3wBxXC4mSRUq23M18L7bGzrjFhF2eIGqwGBIfpzVuNO1lq3J6AVHwNsal1RiEg+beIsvXC1fOPJwa/FY1QcXWyi7pvdDAPsaRDzBpdZU4sKLbs6bnotiHAx8jMD6T1JFi87Qf2wit2UG/HSjSCOVF8AvONwFaU2Hv4nisvE6gdOcXEo5y3kUruJ3DKbnHyGKtuYacYkmsn90wroNJCSiGtJ3lACyxy5oqq5QX9JU971et3wQtfy1wwTovCsZcSBEg+XnHGlxEDacVb0/m5pbvLep8LKOaDrsxGrMYerJiO2NBVA5b+xI+esVBk8LhhL9BtMXWKMBQ0I/+GhfdThrws97nJhC7DdEh1xhW/bcVg9mkuTXekNgucXCor43xKohUGEwa7efe2upiNpRZMelMgTJzb/dyC/BhNkU3YGZaNuPWWPp4j9aORJ/ul4aMOtPcFW/YX/ZLSszd6Y57B9yGfGlCyYKeFIAabCOi1RXp03eqssAfb+WLNW3AAnRji9NGXIx0LZSAL+Q5V2KJE+t4+45VHJbdUD1Rvu31bEtCg8Qy6X8ySZRxWLmPHiEVHzi7ZAOQ46JaVLRrZfaCYT6TpT6cQ7hUkeWBOEdnEQbpuJaI2SfwQIo/1wfqfJSP2EAr+iQEAYBd/kfooECQK9pozrs=" - # SONAR_PR_DECORATION_GITHUB_TOKEN - secure: "tf1efkXd1zFeIb0CZimlpO39ECIl7yZz5fbO0+Fmx9NTptyguOQOZwEecgkoaIEyCSM25lXeUiGsPbheHWRF5ifs7+2axqM2BeMPX9Yd1frYaQ5yNBcqTmrMKYGkXsovMSZU3qDiQcDTzFotHydsO+KPGIs7LhPYLTw7Y4dDEMUqNAvzBLHUf0reMm5h+0CBIxOfMgSjW73wxBp1baqVvqpj5qM9R07e3EdpnMPszX5HZTvOAWqjLrh0re20wU0aNrMld5xfkP7qkSMO1zC68fzRWnFcwGCIt2FP9Vy7AgV8eq1dIqk+EdnilfY4Gq7KhwML+k0b3qmYWDS4NCc5j4EXrAWZLffoY4dYZp4hnv0JLCI+lq1fSn8Je0+AadmW9NjOKZyWg4iNTaHcBVA52OlWxw7KE4cTccuOwpbQXeatR8ayrYALV0UEc+F14bLZAkswbCA10XV4ghe1gLeCCjtqde1D/oltumR6lcXxsRsOGzAdf9DO0fuIJyvhFhKbSp51MFn0Fo5/SQMuZTO6P/YBnVIuzEZDo8xkhMczUxqB+kc2tlFns+BWt0J1H39G9WEGtgn28bUIDZA5CaY2WAwuFRSy95GFSYXkyNi6cm8Iu9xJ/PjZcLCyeQpviLE8am3oc9Mhjx6RMnTh1nc/B8TtuWTVU+PwuVnJzqo+F44=" - GPG_KEY_LOCATION=".travis/secring.gpg" - ENCRYPTED_GPG_KEY_LOCATION=".travis/secring.gpg.enc" branches: only: - main - dev language: java dist: xenial jdk: - openjdk11 before_install: - chmod -R ug+x .travis - .travis/install.sh script: - chmod -R ug+x .travis - set -e - shellcheck pyatlas/*.sh - set +e - .travis/build.sh - .travis/build-pyatlas-gate.sh # - .travis/sonar-gate.sh # - .travis/merge-dev-to-main-gate.sh - .travis/deploy-gate.sh - .travis/tag-main-gate.sh ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to Atlas Thanks for taking the time to contribute! ## Where to ask a question This project uses [StackOverflow](https://stackoverflow.com/) to handle all software related questions. If unsure about an issue, please first ask a question on StackOverflow, and then file an issue if necessary. Do not ask questions in GitHub issues as those will be closed and redirected to StackOverflow. ## Suggestions and bug reports ### Reporting bugs If you have found a bug to report, that is great! Please search for similar bugs in GitHub issues, as your bug might have been filed already. If not, filing a GitHub issue is the next thing to do! ### Filing a Github issue When submitting an issue, here is the information to include: * Title: Use a title that is as self explanatory as possible * Summary: Detailed description of the issue, including screenshots and/or stack traces * Steps to reproduce: Do not forget to include links to data samples that can help in reproducing the issue * Actual vs. Expected: Describe the results and how they differ from the expected behavior * Workaround: If you have found a temporary workaround the issue, please also include it there! ### Suggesting enhancements Enhancements are also handled with GitHub issues. Make sure to include the following: * Title: Use a title that is as self explanatory as possible * Summary: Use-case description of the proposed enhancement * Desired: Describe the desired behavior of the proposed enhancement * Implementation proposal: If you have an idea of how to implement the enhancement ## Submitting code ### Requirements * OpenJDK 11 * Gradle ### First contribution The first step would be to fork the project in your own github account, then clone the project locally using `git clone`. Then use gradle to build the code, and run tests: ``` cd ./gradlew build ``` Atlas is serialized using protobuf. The proto-generated sources are not checked-in to the repository, so any IDE setup requires to run `./gradlew build` to generate the proto-generated sources in the project. At that point, the IDE of your choice will be able to resolve all the necessary classes. To start contributing to your fork, the best way is to make sure that your IDE is setup to follow the [code formatting template](config/format/code_format.xml). Once you have fixed an issue or added a new feature, it is time to submit [a pull request](#pull-request-guidelines)! ### Building You can build a shaded JAR that will allow you to execute atlas. #### Log4j Properties Make sure you first have a `log4j.properties` file in `src/main/resources`. Alternatively, you can have as a VM parameter: ``` -Dlog4j.configuration=file:// ``` https://github.com/osmlab/atlas-checks/blob/dev/config/log4j/log4j.properties Then, you can build your shaded JAR with: ``` ./gradlew shaded ``` From there, you can run command line tools in atlas, like the following: ``` java -cp atlas-5.1.8-SNAPSHOT-shaded.jar org.openstreetmap.atlas.geography.atlas.delta.AtlasDeltaGenerator ``` #### IntelliJ Setup IntellJ IDEA works pretty much out of the box. However, you still need to mess with log4j. First, add a `log4j.properties` file to your project or your VM Options. Also, you need to manually add [slf4j-simple.jar](https://mvnrepository.com/artifact/org.slf4j/slf4j-simple/1.7.25) to your Project Module Dependencies: File > Project Structure > Modules > Dependencies In `atlas_main` add the the JAR that you downloaded. ![slf4j in Intellij](images/slf4j.png) ### Code formatting The project's code is checked by Checkstyle as part of the `gradle check` step. There also is an eclipse code formatting template [here](config/format/code_format.xml) that is used by [Spotless](https://github.com/diffplug/spotless) to check that the formatting rules are being followed. In case those are not, the `gradle spotlessCheck` step will fail and the build will not pass. Spotless provides an easy fix though, with `gradle spotlessApply` which will refactor your code so it follows the formatting guidelines. ### Testing The codebase contains an extensive range of unit tests, and integration tests. Unit tests are supposed to run fairly fast. If the test takes a long time to run, we put it in the integrationTest repository, to allow users to run them only when wanted. All the tests will be run for every pull request build, though! When contributing new code, make sure to not break existing tests (or modify them and explain why the modification is needed) and to add new tests for new features. ### Pull Request Guidelines Pull requests comments should follow the template below: * An as extensive as reasonable description of the change proposed, in easy to read [Markdown](https://guides.github.com/features/mastering-markdown/), with as many code snippet examples, screen captures links and diagrams as possible * A Benefit/Drawback analysis: what does this improve, and at what cost? Is the performance impacted or improved? * Label: If applicable, apply one of the available labels to the pull request ================================================ FILE: LICENSE ================================================ Copyright (c) 2015-2024, Apple Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: README.md ================================================ # Atlas ![Continuous Integration](https://github.com/osmlab/atlas/workflows/Continuous%20Integration/badge.svg?branch=main) [![quality gate](https://sonarcloud.io/api/project_badges/measure?project=org.openstreetmap.atlas%3Aatlas&metric=alert_status)](https://sonarcloud.io/dashboard?id=org.openstreetmap.atlas%3Aatlas) [![Maven Central](https://img.shields.io/maven-central/v/org.openstreetmap.atlas/atlas.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.openstreetmap.atlas%22%20AND%20a:%22atlas%22) [![CircleCI](https://circleci.com/gh/osmlab/atlas/tree/main.svg?style=shield)](https://circleci.com/gh/osmlab/atlas/tree/main) --- [`Atlas`](src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java) is a way to efficiently represent [OpenStreetMap](http://www.openstreetmap.org/) data in memory. A subset of the data is in a "navigable network" form, meaning anything that is assumed to be navigable will be in a form of `Node`s and `Edge`s in a way a routing algorithm could traverse it. It also provides easy to use APIs to access geographical data. On top of it all, it is easy to shard and re-stitch, making it perfect for distributed processing! Projects using Atlas: * [atlas-generator](https://github.com/osmlab/atlas-generator): A Spark job to distribute Atlas shards generation * [atlas-checks](https://github.com/osmlab/atlas-checks): A suite of tools to check OSM data integrity using Atlas, and Spark. * [josm-atlas](https://github.com/osmlab/josm-atlas): A JOSM plugin to visualize Atlas data. # Getting started For build instructions and to contribute, please see the [contributing guidelines](CONTRIBUTING.md). Start playing with Atlas directly with [this sample project](/sample)! # APIs Language|Level ---|--- [Java](/src/main/java/org/openstreetmap/atlas/geography/atlas#using-atlas)|Full feature [Python](/pyatlas#pyatlas)|Basic # What's in it? * A uni-directional navigable network ([`Edge`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Edge.java), [`Node`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Node.java)) * Non-navigable features ([`Area`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Area.java), [`Line`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Line.java), [`Point`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Point.java), [`Relation`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Relation.java)) * All tags As well as other handy tools: * [Create it from `.osm.pbf`](/src/main/java/org/openstreetmap/atlas/geography/atlas#building-an-atlas-from-an-osmpbf-file) * [Sharding](/src/main/java/org/openstreetmap/atlas/geography/sharding#sharding) * [Shard Stitching](/src/main/java/org/openstreetmap/atlas/geography/atlas/multi#multiatlas) * [Shard Exploration](/src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic#dynamicatlas) * [Tag Filtering](/src/main/java/org/openstreetmap/atlas/tags/filters#tag-filtering) * [Atlas Filtering](/src/main/java/org/openstreetmap/atlas/geography/atlas#filtering-an-atlas) * [PBF Ingest](/src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation) * [Country Slicing](/src/main/java/org/openstreetmap/atlas/geography/atlas/raw/slicing) * [Way Sectioning](/src/main/java/org/openstreetmap/atlas/geography/atlas/raw/sectioning) * [Cutting](/src/main/java/org/openstreetmap/atlas/geography/atlas#country-slicing) * [Routing](/src/main/java/org/openstreetmap/atlas/geography/atlas/routing#routing) * [Higher-level entities](/src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex#complex-entities) * [Saving](/src/main/java/org/openstreetmap/atlas/geography/atlas#saving-an-atlas) / [Loading](/src/main/java/org/openstreetmap/atlas/geography/atlas#using-atlas) * [GeoJSON Parser](/src/main/java/org/openstreetmap/atlas/geography/geojson/parser) * [Command Line Tools](atlas-shell-tools) * [Atlas Query Language i.e. AQL](/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl) # Community For more information, please contact our community projects lead [Andrew Wiseman](https://github.com/awisemanapple). ================================================ FILE: atlas-shell-tools/.atlas-shell-tools-integrity-file ================================================ # This file is used by Atlas Shell Tools to ensure that ATLAS_SHELL_TOOLS_HOME # is set correctly. ================================================ FILE: atlas-shell-tools/README.md ================================================ # Atlas Shell Tools A command line interface for [`osmlab/atlas`](https://github.com/osmlab/atlas) (and downstream repositories) tools. ##### Table of Contents 1. [What are the Atlas Shell Tools?](#what-are-the-atlas-shell-tools) 2. [Installation](#installation) * [Auto-install for `bash` users](#auto-install-for-bash-users) * [Auto-install for `zsh` users](#auto-install-for-zsh-users) * [Manual install](#manual-install) 3. [Managing Your Installation](#managing-your-installation) * [Installing A New Module From A Repo](#installing-a-new-module-from-a-repo) * [Switching The Active Module](#switching-the-active-module) * [Viewing Command Documentation](#viewing-command-documentation) * [Saving Command Option Presets](#saving-command-option-presets) * [And Much More](#and-much-more) 4. [Creating Your Own Command](#creating-your-own-command) 5. [Updating The Commands And Tools](#updating-the-commands-and-tools) ## What are the Atlas Shell Tools? Atlas Shell Tools is a command line interface for executing commands defined in [`osmlab/atlas`](https://github.com/osmlab/atlas) and its downstream repositories (like [`osmlab/atlas-generator`](https://github.com/osmlab/atlas-generator)). It provides Unix-like option parsing, autocomplete functionality, a feature-ful option preset system (for commands that need lots of options), module/repository management, and much more. To get a basic installation running, see the [**Installation**](#install) section. You can manage your installation with the `atlas-config(1)` command. Management features include: 1. Installing modules (JARs containing command classes) from various repositories 2. Switching the active module 3. Viewing command documentation 4. Saving command option presets 5. And much more... See the [**Managing Your Installation**](#managing) section for more information. To build a command, all you need to do is subclass `AbstractAtlasShellToolsCommand`(or one of its further subclasses) - then your command will be automatically integrated into the tools! For more information on this, see the [**Creating Your Own Command**](#creating) section. ## Installation Atlas Shell Tools comes with some quick install scripts for users of select shells. Note that the quick install scripts prompt to modify your shell's startup file(s) with some code Atlas Shell Tools needs to run (`.bash_profile` and `~/.bashrc` for `~/bash`, `~/.zshrc` and `~/.zshenv` for `zsh`). If you do not want this behaviour and just want to configure your startup files yourself, select 'n' at the appropriate prompts. #### Auto-install for `bash` users: ``` $ curl -O https://raw.githubusercontent.com/osmlab/atlas/main/atlas-shell-tools/quick_install_bash.sh # Inspect the downloaded file and ensure you are satisfied it is safe to run: $ vim quick_install_bash.sh $ sh quick_install_bash.sh # Answer the prompts, and restart your terminal once this finishes to get started! ``` #### Auto-install for `zsh` users: ``` $ curl -O https://raw.githubusercontent.com/osmlab/atlas/main/atlas-shell-tools/quick_install_zsh.sh # Inspect the downloaded file and ensure you are satisfied it is safe to run: $ vim quick_install_zsh.sh $ sh quick_install_zsh.sh # Answer the prompts, and restart your terminal once this finishes to get started! ``` #### Manual install: If you are not running one of the supported shells, or you want to manually install Atlas Shell Tools, please follow these steps: ##### Building and Installing ``` $ cd /path/to/desired/install/location $ git clone https://github.com/osmlab/atlas.git atlas-shell-tools $ cd atlas-shell-tools $ git checkout main $ ./gradlew clean shaded -x check -x javadoc $ chmod +x ./atlas-shell-tools/scripts/atlas ./atlas-shell-tools/scripts/atlas-config $ ./atlas-shell-tools/scripts/atlas-config repo add atlas https://github.com/osmlab/atlas.git $ ./atlas-shell-tools/scripts/atlas-config repo install atlas ``` ##### Setting up your shell startup file Next, open whichever configuration file your shell uses to set environment variables (e.g. `~/.bash_profile` for `bash`, or `~/.zshenv` for `zsh`). Export the following environment variables using your shell's equivalent `export` statement. ``` # In bash/zsh we can use 'export', other shells may use different syntax # Point ATLAS_SHELL_TOOLS_HOME at the 'atlas-shell-tools' subfolder within your 'atlas-shell-tools' installation export ATLAS_SHELL_TOOLS_HOME=/path/to/atlas-shell-tools/atlas-shell-tools # configure your PATH export PATH="$PATH:$ATLAS_SHELL_TOOLS_HOME/scripts" ``` ##### Autocomplete support Additionally, Atlas Shell Tools supports autocomplete for `bash` and `zsh` through [ast_completions.bash](https://github.com/osmlab/atlas/blob/main/atlas-shell-tools/ast_completions.bash) and [ast_completions.zsh](https://github.com/osmlab/atlas/blob/main/atlas-shell-tools/ast_completions.zsh), respectively. To get these set up, you'll need to source them in your shell's appropriate startup file (`~/.bashrc` for `bash` or `~/.zshrc` for `zsh`). An example for `bash`: ``` ##### ~/.bashrc file ##### # # other stuff here.... # source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.bash" ``` An example for `zsh`: ``` ##### ~/.zshrc file ##### # # other stuff here.... # source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.zsh" ``` ## Managing Your Installation Both `atlas(1)` and `atlas-config(1)` provide numerous ways to manage your installation. Some common operations include: #### Installing A New Module From A Repo Suppose the git repository `me/my-repo` depends on [`osmlab/atlas`](https://github.com/osmlab/atlas) and contains a command implementation you would like to run from the CLI. First, you can save the `my-repo` information with the `repo` subcommand of `atlas-config(1)`: ``` $ atlas-config repo add my-repo 'https://github.com/me/my-repo' ``` Then, to install a new module based on `my-repo`'s `main` branch, simply run: ``` $ atlas-config repo install my-repo ``` See the `atlas-config-repo(1)` man page for more information about repos. #### Switching The Active Module After running the command from `me/my-repo`, you may want to switch back to another repo, like `me/my-other-repo`. Assuming you have a module from `me/my-other-repo` installed, this is straightforward. First, you can list your installed modules with: ``` $ atlas-config list ``` Then, assuming you see the module you want - for sake of example say it's called `my-other-repo-ef3381a` - run: ``` $ atlas-config activate my-other-repo-ef3381a ``` You can check to see that it worked with `$ atlas-config list` again. #### Viewing Command Documentation Atlas Shell Tools comes with extensive documentation, both in the form of man pages as well as subcommand documentation. To see all available man pages, check the Atlas Shell Tools index man page: `atlas-shell-tools(7)`: ``` $ man atlas-shell-tools ``` From there you can jump to whichever page interests you. Subcommand implementations also provide their own documentation, part of which is auto-generated and part of which can be optionally supplied by the command author. To view this documentation, try the subcommand's `--help` option (every subcommand automatically responds to `--help`): ``` $ atlas some-command --help ``` OR ``` $ atlas --help some-command ``` #### Saving Command Option Presets Some commands require many options, and option presets can make repeated use much easier. Assuming your command is called `my-command`, you might save an option preset like: ``` $ atlas --save-preset my-command-preset-1 my-command arg1 --opt1=optarg1 --opt2 ``` You could then run ``` $ atlas --preset my-command-preset-1 my-command arg1 ``` and Atlas Shell Tools would fill in `--opt1=optarg1 --opt2` for you automatically. There is much more to be said about the presets feature. Please see the `atlas-presets(7)`, `atlas-config-preset(1)`, and `atlas(1)` man pages for all the details. #### And Much More... There are many more Atlas Shell Tools features just waiting to be found. Feel free to peruse all the man pages available in `atlas-shell-tools(7)`. ## Creating Your Own Command To create a new command for Atlas Shell Tools, simply create a class that `extends` [AbstractAtlasShellToolsCommand](https://github.com/osmlab/atlas/blob/main/src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/AbstractAtlasShellToolsCommand.java). Once you fill in the abstract methods appropriately (and add a main method), you should build a fat JAR file containing your command, and install it with: ``` $ atlas-config install /path/to/JARfile.jar --symlink ``` This will install the JAR file to the module workspace using a symlink, so iterative changes to the JAR will be automatically picked up by Atlas Shell Tools. For a comprehensive example of the `AbstractAtlasShellToolsCommand` API, check out the demo class [DemoSubcommand](https://github.com/osmlab/atlas/blob/main/src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasShellToolsDemoCommand.java). This class demonstrates how to implement the abstract methods, as well as how to structure the main method. ## Updating The Commands And Tools If you just want to quickly update everything, go ahead and run: ``` $ atlas-config update ``` This will update the toolkit, which includes the code for `atlas(1)`, `atlas-config(1)`, and the various man pages (see the `atlas-glossary(7)` man page for a definition of the toolkit). Then, to update the basic commands by installing a new module from the default [`osmlab/atlas`](https://github.com/osmlab/atlas) repo, run: ``` $ atlas-config repo install atlas ``` If you have any other repos you'd like to update, you can run a `repo install` on those as well to get the latest versions of any commands. ================================================ FILE: atlas-shell-tools/ast_completions.bash ================================================ # What is this script for? # # This script provides autocomplete functionality for Atlas Shell Tools in the # bash shell. # # How do I use it? # # Run the following command: # $ source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.bash" # Then add 'source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.bash"' to your '~/.bashrc' # file to pick up the completions in every new shell! # Return "true" if bash major version is greater than or equal to 4. # Bash4 has a better auto-complete API, so using bash4 if possible is better. is_bash_at_least_version_4 () { local version=$BASH_VERSION local major=$(echo "$version" | cut -d. -f1) if [ "$major" -ge "4" ]; then echo "true" else echo "false" fi } _complete_atlas_shell_tools () { local completion_mode="default"; if [ "$1" = "atlas" ]; then local completion_mode="__completion_atlas__" elif [ "$1" = "atlas-config" ]; then local completion_mode="__completion_atlascfg__" fi if [ "$completion_mode" = "default" ]; then echo "complete error: ${completion_mode} was still default" return 1 fi # disable readline default autocompletion, we are going to customize if [ "$(is_bash_at_least_version_4)" = "true" ]; then compopt +o default fi local reply=$(atlas-config "${completion_mode}" "${COMP_CWORD}" "${COMP_WORDS[@]}"); if [ "$reply" = "__atlas-shell-tools_sentinel_complete_filenames__" ]; then # re-enable bash default completion for filenames if [ "$(is_bash_at_least_version_4)" = "true" ]; then compopt -o default COMPREPLY=() else local cur=${COMP_WORDS[COMP_CWORD]} # We must locally set IFS to '\n' in case there are filenames with whitespace. # Without this, a filename like "file with spaces" would present itself # as 3 discrete completion options, "file", "with", and "spaces". local IFS=$'\n' COMPREPLY=($(compgen -o filenames -f -- "$cur")) fi return 0 else COMPREPLY=(${reply}) fi } complete -o filenames -o bashdefault -F _complete_atlas_shell_tools atlas complete -o filenames -o bashdefault -F _complete_atlas_shell_tools atlas-config ================================================ FILE: atlas-shell-tools/ast_completions.zsh ================================================ # What is this script for? # # This script provides autocomplete functionality for Atlas Shell Tools in the # zsh shell. # # How do I use it? # # Run the following command: # $ source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.zsh" # Then add 'source "$ATLAS_SHELL_TOOLS_HOME/ast_completions.zsh"' to your '~/.zshrc' # file to pick up the completions in every new shell! _complete_atlas_shell_tools_zsh () { local completion_mode="default"; if [ "$1" = "atlas" ]; then local completion_mode="__completion_atlas_zsh__" elif [ "$1" = "atlas-config" ]; then local completion_mode="__completion_atlascfg_zsh__" fi if [ "$completion_mode" = "default" ]; then echo "complete error: ${completion_mode} was still default" return 1 fi local reply=$(atlas-config "${completion_mode}" "${COMP_CWORD}" "${COMP_WORDS[@]}"); if [ "$reply" = "__atlas-shell-tools_sentinel_complete_filenames__" ]; then local cur=${COMP_WORDS[COMP_CWORD]} # We must locally set IFS to '\n' in case there are filenames with whitespace. # Without this, a filename like "file with spaces" would present itself # as 3 discrete completion options, "file", "with", and "spaces". local IFS=$'\n' COMPREPLY=($(compgen -o filenames -f -- "$cur")) return 0 else COMPREPLY=(${reply}) fi } autoload compinit compinit autoload bashcompinit bashcompinit complete -o filenames -o bashdefault -F _complete_atlas_shell_tools_zsh atlas complete -o filenames -o bashdefault -F _complete_atlas_shell_tools_zsh atlas-config ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-activate.1 ================================================ .\" Title: atlas-config-activate .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-ACTIVATE" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-activate \-\- Activate an installed Atlas Shell Tools module .SH "SYNOPSIS" .sp .nf \fIatlas-config activate\fR \-\-help \fIatlas-config activate\fR <\fImodule\fR> .fi .SH "DESCRIPTION" .sp Activate the provided <\fImodule\fR>. Commands defined in an activated module become available for execution by the \fBatlas\fR tool. Activating a module deactivates any other active modules. .SH "OPTIONS" .sp .PP <\fImodule\fR> .RS 4 This indicates a module. The module should be referred to using its name, as reported by \fBatlas-config-list\fR(1). .RE .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-config\-deactivate\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-deactivate.1 ================================================ .\" Title: atlas-config-deactivate .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-DEACTIVATE" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-deactivate \-\- Deactivate an installed Atlas Shell Tools module .SH "SYNOPSIS" .sp .nf \fIatlas-config deactivate\fR \-\-help \fIatlas-config deactivate\fR <\fImodule\fR> .fi .SH "DESCRIPTION" .sp Deactivate the provided <\fImodule\fR>. Commands defined in a deactivated module cannot be run by the \fBatlas\fR tool until the module is reactivated. Deactivating a module will also prevent the commands from showing in the \fBatlas\fR tool command list. .SH "OPTIONS" .sp .PP <\fImodule\fR> .RS 4 This indicates a module. The module should be referred to using its name, as reported by \fBatlas-config-list\fR(1). .RE .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-config\-activate\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-install.1 ================================================ .\" Title: atlas-config-install .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-INSTALL" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-install \-\- Install a new Atlas Shell Tools module from a local JAR .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIinstall\fR \-\-help \fIatlas\-config\fR \fIinstall\fR [\-\-symlink] [\-\-force] [\-\-skip] [\-\-deactivated] [\-\-name=<\fIname\fR>] <\fIJARfile\fR> .fi .SH "DESCRIPTION" .sp Installs and activates a given <\fIJARfile\fR> as a new \fBatlas\fR module. This will deactivate any currently activated module. If you are trying to install a module from a specific repo, please see \fBatlas-config-repo\fR(1) for more information. When installing directly from a <\fIJARfile\fR>, the \fB\-\-symlink\fR option can be enabled to install using a symbolic link. This is great for cases where the underlying <\fIJARfile\fR> may change dynamically (e.g. when incrementally rebuilding the JAR during development/testing of a new command). Note that once a module is installed, it should be referred to by its module name and not by the <\fIJARfile\fR> name. The module names are displayed by \fBatlas-config-list\fR(1). Unless the \fB\-\-name\fR option is used, the module name will be the same as the <\fIJARfile\fR> name but without the '.jar' extension. .SH "OPTIONS" .sp .PP <\fIJARfile\fR> .RS 4 The path to a JARfile containing \fBatlas\fR commands. Once installed, this JARfile is referred to as a module. .RE .PP \fB\-\-deactivated\fR .RS 4 Install a module without actually activating it. .RE .PP \fB\-\-force\fR .RS 4 Force overwrite when re-installing an existing module. .RE .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .PP \fB\-\-name\fR= .RS 4 Install a module using a custom name. .RE .PP \fB\-\-skip\fR .RS 4 Skip installation when re-installing an existing module. .RE .PP \fB\-\-symlink\fR, \fB-s\fR .RS 4 Install a module using a symlink as opposed to a copy. .RE .SH "EXAMPLES" .sp Install a module with custom name 'MyModule': .sp .RS 4 $ atlas\-config install \-\-name=MyModule ~/Desktop/example.jar .RE .sp Install a symlinked module but do not activate it: .sp .RS 4 $ atlas\-config install \-\-symlink \-\-deactivated ~/example.jar .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-config\-uninstall\fR(1), \fBatlas\-config\-activate\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-list.1 ================================================ .\" Title: atlas-config-list .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-LIST" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-list \-\- List installed Atlas Shell Tools modules and module statuses .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIlist\fR \-\-help \fIatlas\-config\fR \fIlist\fR [\-\-current] [\-\-one\-line] .fi .SH "DESCRIPTION" .sp List installed modules with additional contextual information. This listing is also known as the module workspace. The \fB\-\-current\fR option can be used to view only the current activated module, if there is one. The \fB\-\-one\-line\fR option can be used to suppress metadata output. A \fB*\fR next to a module denotes that the module is activated. Symlinked modules display their targets using the familiar arrow notation (\fB\->\fR). If a symlink is broken, the warning \fB(BROKEN SYMLINK)\fR is displayed next to the module name. .SH "OPTIONS" .sp .PP \fB\-\-current\fR, \fB-c\fR .RS 4 Show only the currently activated module, if one is activated. .RE .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .PP \fB\-\-one\-line\fR, \fB-1\fR .RS 4 Suppress the module metadata output, instead displaying each module on a single line. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-log.1 ================================================ .\" Title: atlas-config-log .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-LOG" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-log \-\- Check or set Atlas Shell Tools log parameters .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIlog\fR \-\-help \fIatlas\-config\fR \fIlog\fR reset \fIatlas\-config\fR \fIlog\fR set\-level \fIatlas\-config\fR \fIlog\fR set\-stream \fIatlas\-config\fR \fIlog\fR show .fi .SH "DESCRIPTION" .sp Use this command to check or set the current log parameters. Under the hood, this command is a shortcut for modifying the \fBatlas\fR log4j configuration file. It expects that file to follow a well-defined format. For best results, avoid manually updating the configuration file. .SH "OPTIONS" .sp .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "DIRECTIVES" \fBatlas\-config\-log\fR(1) exposes its interface through various directives, detailed below. .sp .PP \fIreset\fR .RS 4 Reset the log parameters to default, i.e. level=\fBERROR\fR and stream=\fBstderr\fR. This is equivalent to running \fBatlas\-config reset log\fR. .RE .PP \fIset\-level\fR .RS 4 Set the current logging level to . Valid log levels: \fBALL\fR, \fBTRACE\fR, \fBDEBUG\fR, \fBINFO\fR, \fBWARN\fR, \fBERROR\fR, \fBFATAL\fR, and \fBOFF\fR. .RE .PP \fIset\-stream\fR .RS 4 Set the current logging stream to . Valid stream settings: \fBstdout\fR and \fBstderr\fR. .RE .PP \fIshow\fR .RS 4 Show the current log settings. .RE .SH "EXAMPLES" .sp Set the log level to \fBINFO\fR: .sp .RS 4 $ atlas\-config log set\-level INFO .RE .sp Show the log parameters: .sp .RS 4 $ atlas\-config log show .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-preset.1 ================================================ .\" Title: atlas-config-preset .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-PRESET" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-preset \-\- Create and manage Atlas Shell Tools command presets .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIpreset\fR \-\-help \fIatlas\-config\fR \fIpreset\fR \fIcopy\fR \fIatlas\-config\fR \fIpreset\fR \fIcopy\-global\fR \fIatlas\-config\fR \fIpreset\fR \fIedit\fR \fIatlas\-config\fR \fIpreset\fR \fIedit\-global\fR \fIatlas\-config\fR \fIpreset\fR \fIlist\fR [command [preset]] \fIatlas\-config\fR \fIpreset\fR \fIlist\-global\fR [preset] \fIatlas\-config\fR \fIpreset\fR \fInamespace\fR [namespace] \fIatlas\-config\fR \fIpreset\fR \fIremove\fR [preset] \fIatlas\-config\fR \fIpreset\fR \fIremove\-global\fR [preset] \fIatlas\-config\fR \fIpreset\fR \fIsave\fR \fIatlas\-config\fR \fIpreset\fR \fIsave\-global\fR .fi .SH "DESCRIPTION" .sp \fBatlas\-config\-preset\fR(1) provides a more granular preset management interface than the basic \fBatlas\fR(1) options (\fB\-\-preset\fR, \fB\-\-save\-preset\fR, \fB\-\-save\-global\-preset\fR). It can handle more complex tasks like editing a preset in place, or managing the preset namespaces. For full details on the workings of \fBatlas\-config\-preset\fR(1), see the \fBTier 2\fR and \fBTier 3\fR sections in \fBatlas\-presets\fR(7). .SH "OPTIONS" .sp .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "DIRECTIVES" \fBatlas\-config\-preset\fR(1) exposes its interface through various directives, detailed below. Again, this information is presented in more detail in \fBatlas\-presets\fR(7). .sp .PP \fIcopy\fR .RS 4 For , copy preset into new preset . must not already exist, else the copy will fail. .RE .PP \fIcopy\-global\fR .RS 4 Copy global preset into new global preset . must not already exist, else the copy will fail. .RE .PP \fIedit\fR .RS 4 Edit preset for . If does not exist, then it will be created when the edit is successfully saved. The default preset editor is \fBvim\fR, but this can be changed by setting the \fBATLAS_SHELL_TOOLS_EDITOR\fR environment variable. .RE .PP \fIedit\-global\fR .RS 4 Edit global preset . If does not exist, then it will be created when the edit is successfully saved. The default preset editor is \fBvim\fR, but this can be changed by setting the \fBATLAS_SHELL_TOOLS_EDITOR\fR environment variable. .RE .PP \fIlist\fR .RS 4 List all available presets (including globals), or list all presets for a given [command], or list contents of preset [preset] for [command]. .RE .PP \fIlist\-global\fR .RS 4 List all available global presets, or list contents of global preset [preset]. .RE .PP \fInamespace\fR .RS 4 Execute a on a given preset [namespace]. Available subdirectives are \fBcreate\fR, \fBlist\fR, \fBremove\fR, and \fBuse\fR. See \fBTier 3\fR in \fBatlas\-presets\fR(7) for more details. .RE .PP \fIremove\fR .RS 4 Remove all presets for a given , or remove the preset [name] for . .RE .PP \fIremove\-global\fR .RS 4 Remove all global presets, or remove the global preset [name]. .RE .PP \fIsave\fR .RS 4 Save a preset for without actually running the command. is a sequence of options to be saved in the preset. Again, recall that you must use the long option '=' syntax for specifying option arguments when saving a preset (e.g. '--opt=arg' and \fInot\fR '--opt arg'). .RE .PP \fIsave\-global\fR .RS 4 Save a global preset . is a sequence of options to be saved in the preset. Again, recall that you must use the long option '=' syntax for specifying option arguments when saving a preset (e.g. '--opt=arg' and \fInot\fR '--opt arg'). .RE .SH "EXAMPLES" .sp Edit preset "p1" for command "MyCommand": .sp .RS 4 $ atlas\-config preset edit MyCommand p1 .RE .sp Create a namespace called "namespace1": .sp .RS 4 $ atlas\-config preset namespace create namespace1 .RE .sp Save a global preset "p1": .sp .RS 4 $ atlas\-config preset save\-global p1 --opt1 --opt2=opt2Arg .RE .sp For more examples, see \fBatlas\-presets\fR(7). .SH "SEE ALSO" .sp \fBatlas\-presets\fR(7) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-repo.1 ================================================ .\" Title: atlas-config-repo .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-REPO" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-repo \-\- Register and manage Atlas Shell Tools repositories .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIrepo\fR \-\-help \fIatlas\-config\fR \fIrepo\fR \fIadd\fR [ref] \fIatlas\-config\fR \fIrepo\fR \fIadd\-gradle\-exclude\fR \fIatlas\-config\fR \fIrepo\fR \fIadd\-gradle\-skip\fR \fIatlas\-config\fR \fIrepo\fR \fIedit\fR \fIatlas\-config\fR \fIrepo\fR \fIinstall\fR [\-\-ref=ref] \fIatlas\-config\fR \fIrepo\fR \fIlist\fR [repo\-name] \fIatlas\-config\fR \fIrepo\fR \fIremove\fR .fi .SH "DESCRIPTION" .sp \fBatlas\-config\-repo\fR(1) provides an interface for managing repo objects (By repo, we simply mean any \fBgit\fR(1) repository that contains Atlas Shell Tools command implementations). Once a new repo object has been registered using \fIadd\fR, new modules can be installed from the repo using the \fIinstall\fR directive. This makes it easy to fetch the newest versions of Atlas Shell Tools commands and manage their multiple sources. Additional directives facilitate more advanced use cases. See the \fBDIRECTIVES\fR section for details. .sp For module installation, \fBatlas\-config\-repo\fR(1) uses the default \fBbuild.gradle\fR of the referenced repo, but with a special injected configuration designed to build a fat JAR that will work with Atlas Shell Tools. See the \fIinstall\fR directive for more details on this process. Additionally, \fBatlas\-config\-repo\fR(1) provides some other directives to manipulate this configuration. .sp Also note that registering new repos with \fIadd\fR does not actually save the referenced repository on your machine. Rather it simply stores some metadata (which we refer to as a repo object) that is used to clone the referenced repo at install\-time. The clone is saved in a temporary location and is removed when module installation completes. .SH "DIRECTIVES" \fBatlas\-config\-repo\fR(1) exposes its interface through various directives, detailed below. .sp .PP \fIadd\fR .RS 4 Add a new repo object with a given , \fBgit\fR(1) remote , and an optional default [ref]. Here, [ref] is a \fBgit\fR(1) ref (e.g. a branch name, a tag, or even a commit hash). If no explicit [ref] is provided, \fIadd\fR defaults to 'main'. The ref associated with a repo object is used by \fIinstall\fR as the default. See \fIinstall\fR for more information. Note that for , you can use any git\-clonable URL, i.e. an https:// URL, a file:// URL, or even a local path like /Users/you/somerepo. .RE .PP \fIadd\-gradle\-exclude\fR .RS 4 Add an excluded to repo . When the \fIinstall\fR directive builds a module, it is using Gradle (with a custom Atlas Shell Tools configuration) to create a fat JAR file for that module. Excluded packages are removed from the Atlas Shell Tools Gradle configuration using Gradle's 'exclude group:' directive. The 'exclude module:' directive is not currently supported. .RE .PP \fIadd\-gradle\-skip\fR .RS 4 Add a skipped to repo . When the \fIinstall\fR directive builds a module, it is using Gradle (with a custom Atlas Shell Tools configuration) to create a fat JAR file for that module. By default, \fIinstall\fR is running a \fB./gradlew clean build\fR. Adding skipped tasks effectively adds \fB\-x \fR arguments to the above command. .RE .PP \fIedit\fR .RS 4 Edit the configuration for repo if it exists. The default repo editor is \fBvim\fR, but this can be changed by setting the \fBATLAS_SHELL_TOOLS_EDITOR\fR environment variable. .RE .PP \fIinstall\fR .RS 4 Install a module from a repo object with name . This command will clone the repo referenced by 's URL to a temporary location. Next, it will check out the ref specified in 's config or use the override provided by \-\-ref if present. It will then initiate a Gradle build, using the repo's \fBbuild.gradle\fR file with a custom injected Atlas Shell Tools configuration. Finally, it will install and activate the freshly built module. When the install finishes, the cloned repo will be removed. .RE .PP \fIlist\fR .RS 4 Display all registered repo objects, along with their URLs and default refs. If a [repo\-name] is provided, then display the full configuration of the given repo object. .RE .PP \fIremove\fR .RS 4 Remove the registration for repo object , if it exists. .RE .SH "EXAMPLES" .sp Register a new repo object with name \fBmy\-repo\fR and default branch \fBmain\fR: .sp .RS 4 $ atlas\-config repo add my\-repo https://github.com/me/my\-repo.git .RE .sp Register a new repo object with name \fBmy\-repo\fR and default branch \fBmy\-branch\fR: .sp .RS 4 $ atlas\-config repo add my\-repo https://github.com/me/my\-repo.git my\-branch .RE .sp Skip Gradle task 'integrationTest' when installing from \fBmy\-repo\fR: .sp .RS 4 $ atlas\-config repo add\-gradle\-skip my\-repo integrationTest .RE .sp Install a new module from repo \fBmy\-repo\fR, but override the default ref with a specific release: .sp .RS 4 $ atlas\-config repo install my\-repo \-\-ref=1.3.9 .RE .sp .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-glossary\fR(7) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-reset.1 ================================================ .\" Title: atlas-config-reset .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-RESET" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-reset \-\- Reset the Atlas Shell Tools installation .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIreset\fR \-\-help \fIatlas\-config\fR \fIreset\fR all \fIatlas\-config\fR \fIreset\fR index \fIatlas\-config\fR \fIreset\fR log \fIatlas\-config\fR \fIreset\fR modules \fIatlas\-config\fR \fIreset\fR presets \fIatlas\-config\fR \fIreset\fR repos .fi .SH "DESCRIPTION" .sp Completely reset the \fBatlas\fR installation. A full reset (using \fIall\fR) consists of the following steps: .RS 4 \fB1)\fR Uninstalling all installed modules \fB2)\fR Removing all presets \fB3)\fR Setting the log level back to \fBERROR\fR \fB4)\fR Setting the log stream back to \fBstderr\fR \fB5)\fR Deleting the active command index \fB6)\fR Removing all saved repos .RE You can use various directives to only run some subset of these steps. See the \fBDIRECTIVES\fR section for more details. .SH "OPTIONS" .sp .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "DIRECTIVES" \fBatlas\-config\-reset\fR(1) exposes its interface through various directives, detailed below. .sp .PP \fIall\fR .RS 4 Perform all the reset steps, fully resetting the environment to the state it was in at install time. .RE .PP \fIindex\fR .RS 4 Reset the active module index. .RE .PP \fIlog\fR .RS 4 Reset the log parameters to default, i.e. level=\fBERROR\fR and stream=\fBstderr\fR. This is equivalent to running \fBatlas\-config log reset\fR. .RE .PP \fImodules\fR .RS 4 Uninstall all installed modules. This is equivalent to running \fBatlas\-config uninstall \-\-all\fR. .RE .PP \fIpresets\fR .RS 4 Remove all saved presets in all namespaces. Also resets the current namespace to 'default'. .RE .PP \fIrepos\fR .RS 4 Remove all saved repos. .RE .SH "EXAMPLES" .sp Reset everything: .sp .RS 4 $ atlas\-config reset all .RE .sp Reset only the repos: .sp .RS 4 $ atlas\-config reset repos .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-sync.1 ================================================ .\" Title: atlas-config-sync .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-SYNC" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-sync \-\- Refresh the Atlas Shell Tools installation based on the active module .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIsync\fR \-\-help \fIatlas\-config\fR \fIsync\fR .fi .SH "DESCRIPTION" .sp Refresh the command index for the current active module. Additionally, this will perform some runtime checks to validate that all commands in the active module registered their documentation and options/arguments correctly. This command is useful when a module JAR is installed using a symlink, and has been updated at the other end of the link. If you made changes to a command name or description, running this command will reflect those changes in \fBatlas \-\-list\fR. If you altered some option/argument or documentation registration, running \fBsync\fR will perform a quick validation \- preventing you from having to test each altered command individually. .SH "OPTIONS" .sp .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-uninstall.1 ================================================ .\" Title: atlas-config-uninstall .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-UNINSTALL" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-uninstall \-\- Uninstall Atlas Shell Tools module(s) .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIuninstall\fR \-\-help \fIatlas\-config\fR \fIuninstall\fR \-\-all \fIatlas\-config\fR \fIuninstall\fR [\-\-force] <\fImodules...\fR> .fi .SH "DESCRIPTION" .sp Remove installed module(s). If the module is a symlink, this will only remove the link. It will \fBnot\fR delete the linked\-to file. By default, this command will not uninstall an activated module. This can be overridden with the \fB\-\-force\fR or \fB\-\-all\fR options. .SH "OPTIONS" .sp .PP <\fImodules...\fR> .RS 4 This indicates one or more modules. The modules should be referred to using their names as reported by \fBatlas-config-list\fR(1). .RE .PP \fB\-\-all\fR, \fB\-a\fR .RS 4 Uninstall all modules including the activated module, ignoring the supplied <\fImodule\fR> argument if present. This is equivalent to running \fBatlas\-config reset \-\-modules\fR. .RE .PP \fB\-\-force\fR .RS 4 Force uninstall an activated module. .RE .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-config\-install\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config-update.1 ================================================ .\" Title: atlas-config-update .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG-UPDATE" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config\-update \-\- Update the Atlas Shell Tools core toolkit .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR \fIupdate\fR \-\-help \fIatlas\-config\fR \fIupdate\fR .fi .SH "DESCRIPTION" .sp Update the \fBatlas\-shell\-tools\fR(7) core toolkit using \fBgit\fR(1). This will not actually update any modules. To update modules using your repos, see \fBatlas\-config\-repo\fR(1). .SH "OPTIONS" .sp .PP \fB\-\-help\fR .RS 4 Show this help menu. .RE .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-config\-repo\fR(1) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas-config.1 ================================================ .\" Title: atlas-config .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CONFIG" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-config \-\- Configure the Atlas Shell Tools installation .SH "SYNOPSIS" .sp .nf \fIatlas\-config\fR [\-\-no-pager] \-\-help[=<\fIcommand\fR>] \fIatlas\-config\fR \-\-version \fIatlas\-config\fR [\-\-no-pager] [\-\-quiet] <\fIcommand\fR> [arg...] .fi .SH "DESCRIPTION" .sp \fBatlas\-config\fR exposes an interface for configuring the Atlas Shell Tools installation. It provides a way to install new modules, change the currently activated module, reset the installation, and more. \fBatlas\-config\fR operates using a subcommand interface, where each subcommand performs a different configuration action. See the \fBATLAS\-CONFIG COMMANDS\fR section for more information. .sp For running actual Atlas Shell Tools commands, see \fBatlas\fR(1). .sp For a glossary of Atlas Shell Tools terms, see \fBatlas\-glossary\fR(7). .SH "OPTIONS" .PP <\fIcommand\fR> .RS 4 This indicates a command. See the \fBATLAS\-CONFIG COMMANDS\fR section for a list of available commands. .RE .PP \fB\-\-help\fR[=<\fIcommand\fR>]\fR .RS 4 Show the help menu for a given command. If no command is given, show a general help menu and then exit. .RE .PP \fB\-\-no\-pager\fR .RS 4 Disable pagination for all documentation. .RE .PP \fB\-\-quiet\fR, \fB\-q\fR .RS 4 Suppress non-essential output. .RE .PP \fB\-\-version\fR, \fB-V\fR .RS 4 Show the \fBatlas\-config\fR version info and then exit. .RE .sp .SH "ATLAS-CONFIG COMMANDS" .sp \fBatlas-config\fR uses a set of subcommands to present a high\-level interface for installation configuration. \fBatlas\-config\-activate\fR(1) .RS 4 Activate an installed module. .RE \fBatlas\-config\-deactivate\fR(1) .RS 4 Deactivate an installed module. .RE \fBatlas\-config\-install\fR(1) .RS 4 Install a new module. .RE \fBatlas\-config\-list\fR(1) .RS 4 List all installed modules and module statuses. .RE \fBatlas\-config\-log\fR(1) .RS 4 Check or set log parameters. .RE \fBatlas\-config\-preset\fR(1) .RS 4 Create and manage Atlas Shell Tools command presets. .RE \fBatlas\-config\-repo\fR(1) .RS 4 Register and manage Atlas Shell Tools repositories. .RE \fBatlas\-config\-reset\fR(1) .RS 4 Reset the installation. .RE \fBatlas\-config\-sync\fR(1) .RS 4 Refresh the installation based on the active module. .RE \fBatlas\-config\-uninstall\fR(1) .RS 4 Uninstall a module. .RE \fBatlas\-config\-update\fR(1) .RS 4 Update the Atlas Shell Tools core toolkit. .RE .SH "TERMINAL AND ENVIRONMENT" .sp \fBatlas\-config\fR displays its various help and manual pages using \fBman\fR, which is paged by default. To disable paged output for all documentation, try the \fB\-\-no\-pager\fR option. .sp \fBatlas-config\fR uses formatted output when appropriate. To change this behavior, \fBatlas-config\fR checks for existence of the following environment variables: .sp .RS 4 \fBNO_COLOR\fR \- Disable all special formatted output. Other popular CLI tools also respect this variable. See . .sp \fBATLAS_SHELL_TOOLS_NO_COLOR\fR \- Disable special formatted output for \fBatlas\fR and \fBatlas-config\fR only. .sp \fBATLAS_SHELL_TOOLS_USE_COLOR\fR \- Enable special formatted output. Overrides the setting of \fBNO_COLOR\fR and \fBATLAS_SHELL_TOOLS_NO_COLOR\fR. .sp .RE \fBatlas-config\fR stores program data in compliance with the XDG Base Directory specification, i.e. at $HOME/.local/share/atlas\-shell\-tools. It also respects the \fBXDG_DATA_HOME\fR environment variable \- if set, \fBatlas-config\fR will store program data at the base path specified by that variable. See \fBatlas\-plumbing\fR(5) for more information. .sp For more details about all the environment variables used by \fBatlas\-shell\-tools\fR(7), see the \fBatlas\-environment\fR(7) manpage. .SH "SEE ALSO" .sp \fBatlas\fR(1), \fBatlas\-plumbing\fR(5), \fBatlas\-presets\fR(7), \fBatlas\-environment\fR(7) .SH "AUTHOR" .sp This program was written by Lucas Cram . Please report any bugs you find. .SH "BUGS" .sp If you run the command: .sp .RS 4 $ atlas\-config install .RE .sp where is the path to the currently active module JARfile, this will start the installation process but then delete the JARfile. .sp Please report any bugs you find to the \fBAUTHOR\fR. .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man1/atlas.1 ================================================ .\" Title: atlas .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS" "1" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas \-\- Run an Atlas Shell Tools command .SH "SYNOPSIS" .sp .nf \fIatlas\fR [\-\-no-pager] \-\-help[=<\fIcommand\fR>] \fIatlas\fR \-\-version \fIatlas\fR [\-\-no-pager] \-\-list \fIatlas\fR \-\-class\-of=<\fIcommand\fR> \fIatlas\fR [\-\-no\-pager] [\-\-debug] [\-\-memory=] [\-\-preset=] [\-\-save\-preset=] [\-\-save\-global\-preset=] <\fIcommand\fR> [arg...] .fi .SH "DESCRIPTION" .sp \fBatlas\fR is an easy\-to\-use wrapper for launching Atlas Shell Tools commands. It provides a simple interface to find and run any commands defined in the currently activated Atlas Shell Tools module. Atlas Shell Tools also implements a robust preset management system, which can save you tons of typing for commands that require a lot of options. See \fBatlas-presets\fR(7) section for more on this topic. .sp For advanced module management capabilites and other installation configuration commands, see \fBatlas\-config\fR(1). .sp For a glossary of Atlas Shell Tools terms, see \fBatlas-glossary\fR(7). .SH "OPTIONS" .PP <\fIcommand\fR> .RS 4 This indicates a command. A list of available commands can be seen with the \fB--list\fR option. .RE .PP \fB\-\-class\-of\fR=<\fIcommand\fR> .RS 4 Show the full classname of a given command and then exit. .RE .PP \fB\-\-debug\fR .RS 4 Run \fBatlas\fR in debug mode. While in debug mode, \fBatlas\fR will not actually run any commands. Instead, it will print debug information and exit. Use this option to see the full JVM launch command, preset debug diagnostics, and more. .RE .PP \fB\-\-help\fR[=<\fIcommand\fR>]\fR .RS 4 Show the help menu for a given command. If no command is given, show a general help menu and then exit. .RE .PP \fB\-\-list\fR, \fB-l\fR .RS 4 Display a list of all available commands and exit. Command name collision is resolved when the active module index is generated. The resolution exists at the index level only and is not reflected in the actual command class files. .RE .PP \fB\-\-memory\fR=, \fB\-m\fR .RS 4 Set the maximum memory pool size for the JVM that runs your command. should be specified in bytes. \fB--memory\fR also understands shorthand for metric prefixes, e.g. 1024K, 512M, 4G, etc. If this option is not supplied, the memory pool defaults to 8G. .RE .PP \fB\-\-no\-pager\fR .RS 4 Disable pagination for all documentation. .RE .PP \fB\-\-preset\fR=, \fB\-p\fR .RS 4 Apply the given preset(s) before running <\fIcommand\fR>. A list of presets can be provided by separating preset names with either a colon or a comma. See \fBatlas-presets\fR(7) for more information. .RE .PP \fB\-\-quiet\fR, \fB\-q\fR .RS 4 Suppress non-essential \fBatlas\fR output. .RE .PP \fB\-\-save\-preset\fR= .RS 4 Save the command ARGV to before running <\fIcommand\fR>. If \fB\-\-preset\fR is also supplied, the save operation will occur after the argument to \fB\-\-preset\fR is applied. So will include ARGV elements inserted by the \fB\-\-preset\fR application in addition to those directly supplied by the user. See \fBatlas-presets\fR(7) for more information. .RE .PP \fB\-\-save\-global\-preset\fR= .RS 4 This option is the same as \fB\-\-save\-preset\fR, except it will instead save to the global presets instead of the command\-specific presets. Note that it can be used in conjuction with \fB\-\-save\-preset\fR to save both a global and command preset at the same time. Also like \fB\-\-save\-preset\fR, \fB\-\-save\-global\-preset\fR saves ARGV after the the application of \fB\-\-preset\fR if present. See \fBatlas-presets\fR(7) for more information. .RE .PP \fB\-\-version\fR, \fB-V\fR .RS 4 Show the \fBatlas\fR version info and then exit. .RE .sp .SH "EXAMPLES" .sp Let's break down how to run a command called MyCommand with some args and options. Here, '--opt1' is an option that takes no arguments. '--opt2' is an option that takes an optional argument, so we use the '=' operator to disambiguate 'optarg2' from a regular argument. '--opt3' is an option that takes a required argument, so \- if desired \- we can omit the '=' when specifying 'optarg3'. Finally, 'arg1' and 'arg2' are regular program arguments: .sp .RS 4 $ atlas MyCommand \-\-opt1 arg1 \-\-opt2=optarg2 arg2 --opt3 optarg3 .RE .sp See the manual page for MyCommand: .sp .RS 4 $ atlas \-\-help MyCommand .RE .sp See a list of all available commands: .sp .RS 4 $ atlas \-\-list .RE .sp Save preset 'p1' for MyCommand and then run: .sp .RS 4 $ atlas \-\-save\-preset p1 MyCommand arg1 \-\-opt1=opt1arg \-\-opt2=opt2arg .RE .sp Run MyCommand with preset 'p1': .sp .RS 4 $ atlas \-\-preset p1 MyCommand arg1 \-\-opt2=overrideOpt2Arg .RE .sp Run MyCommand with preset 'p1', while also saving new preset 'p2': .sp .RS 4 $ atlas \-\-preset p1 \-\-save\-preset p2 MyCommand arg1 \-\-opt2=overrideOpt2Arg .RE .sp .SH "TERMINAL AND ENVIRONMENT" .sp \fBatlas\fR pages the output of the various help messages using a combination of \fBless\fR and \fBman\fR. Subcommand help pages are piped through \fBless\fR by default, but this can be overridden with the \fBATLAS_SHELL_TOOLS_PAGER\fR environment variable. The actual \fBatlas\fR manual page (which you are currently reading) is displayed using \fBman\fR. To disable paged output for all documentation, try the \fB\-\-no\-pager\fR option. .sp \fBatlas\fR uses formatted output when appropriate. To change this behavior, \fBatlas\fR checks for existence of the following environment variables: .sp .RS 4 \fBNO_COLOR\fR \- Disable all special formatted output. Other popular CLI tools also respect this variable. See . .sp \fBATLAS_SHELL_TOOLS_NO_COLOR\fR \- Disable special formatted output for \fBatlas\fR and \fBatlas-config\fR only. .sp \fBATLAS_SHELL_TOOLS_USE_COLOR\fR \- Enable special formatted output. Overrides the setting of \fBNO_COLOR\fR and \fBATLAS_SHELL_TOOLS_NO_COLOR\fR. .sp .RE \fBatlas\fR stores program data in compliance with the XDG Base Directory specification, i.e. at $HOME/.local/share/atlas\-shell\-tools. It also respects the \fBXDG_DATA_HOME\fR environment variable \- if set, \fBatlas\fR will store program data at the base path specified by that variable. See \fBatlas\-plumbing\fR(5) for more information. .sp For more details about all the environment variables used by \fBatlas\-shell\-tools\fR(7), see the \fBatlas\-environment\fR(7) manpage. .SH "SEE ALSO" .sp \fBatlas\-config\fR(1), \fBatlas\-presets\fR(7), \fBatlas\-environment\fR(7) .SH "AUTHOR" .sp This program was written by Lucas Cram . .SH "BUGS" .sp Please report any bugs you find to the \fBAUTHOR\fR. .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man5/atlas-plumbing.5 ================================================ .\" Title: atlas-plumbing .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-PLUMBING" "5" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-plumbing \-\- Details about how Atlas Shell Tools stores toolkit program data .SH "SYNOPSIS" $HOME/.local/share/atlas-shell-tools/* .SH "DESCRIPTION" .sp Unless otherwise specified via the \fBXDG_DATA_HOME\fR environment variable, \fBatlas\-shell-tools\fR(7) will store data at $HOME/.local/share/atlas-shell-tools. This data directory itself contains four other directories, each of which is detailed in its own section below. These directories store all user data for the toolkit (the toolkit location is specified by the \fBATLAS_SHELL_TOOLS_HOME\fR environment variable). .SH "LOG DIRECTORY" $HOME/.local/share/atlas-shell-tools/log4j This directory stores the default log4j configuration file for the toolkit. The parameters in this file can be edited with \fBatlas\-config\-log\fR(1) \- they should \fBnot\fR be edited manually. This directory also stores a backup log4j file that can be restored using the aforementioned \fBatlas\-config\-log\fR(1) command should the default file become corrupted. .SH "MODULE DIRECTORY" $HOME/.local/share/atlas-shell-tools/modules Also known as the module workspace, this directory stores modules (fat JARs), module metadata files, and the active module index. Any JAR that is not the current active module will have a '.deactivated' file extension. Files with a '.metadata' extension store additional information about the JAR file with the same name. The active module index file, '.active_module_index', stores data about the commands found in the current activated module and is used by \fBatlas\fR(1) to look up command classes and provide the command list for the '\-\-list' option. The active module index stores each command name, class, and description line\-wise with the fields separated by an ASCII record separator character. This index can be refreshed with \fBatlas\-config\-sync\fR(1). .SH "PRESET DIRECTORY" $HOME/.local/share/atlas-shell-tools/presets This directory stores all preset data for the toolkit. At root it contains a directory for each preset namespace, using the 'default' namespace by default. The current active namespace is stored in a file called '.current_namespace'. Within each namespace is 1) a directory called '.global' which stores the global presets for the namespace and 2) directories for any command that has saved presets under that namespace. The actual preset data files are found within the '.global' and command directories. Preset data is stored line\-wise exactly as each element will appear in ARGV, e.g. the line "--option=argument" will become a single ARGV element "--option=argument". The data in this folder can be managed with \fBatlas\-config\-preset\fR(1). .SH "REPO DIRECTORY" $HOME/.local/share/atlas-shell-tools/repos This directory stores the repo data for all the registered repos. Each registered repo has its own directory that contains the repo data file 'repo_config'. Repo data is stored as line\-wise "key = value" pairs. The data in this folder can be managed with \fBatlas\-config\-repo\fR(1). .SH "SEE ALSO" .sp \fBatlas\-environment\fR(7) .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-cli.7 ================================================ .\" Title: atlas-cli .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-CLI" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-cli \-\- Atlas Shell Tools command line interface and conventions .SH "SYNOPSIS" * .SH "DESCRIPTION" .sp \fBatlas\-shell\-tools\fR(7) makes use of various command line conventions and utilities to provide a comprehensive user experience. The \fBShells\fR section discusses the various shells supported by \fBatlas\-shell\-tools\fR(7). The \fBOptions\fR section explains the various ways \fBatlas\-shell\-tools\fR(7) parses command line options. The \fBTAB Complete\fR section shows the various ways that TAB completions can make command line interaction easier. .SH "SHELLS" .sp While \fBatlas\-shell\-tools\fR(7) is compatible with any command shell, \fBbash\fR(1) and \fBzsh\fR(1) users can take advantage of both the quick installation scripts and advanced TAB completion features. For \fBbash\fR(1) users, note that while \fBatlas\-shell\-tools\fR(7) works best with \fBbash\fR(1) version 4, version 3 can still utilize the full TAB complete feature set. .SH "OPTIONS" .sp \fBatlas\-shell\-tools\fR(7) has two ways of dealing with options: .sp 1. Perl's Getopt::Long semantics are used when parsing global options suppled to the \fBatlas\fR(1) and \fBatlas\-config\fR(1) scripts. In the following command line, the '\-\-debug' option is a global option: .sp .RS 4 $ atlas \-\-debug my\-command \-\-opt1 .RE .sp 2. Subcommand options are parsed with the SimpleOptionAndArgumentParser class, found in the Atlas repo . Please check the source of that class for javadoc documentation comments detailing the parsing semantics. They are similar to the GNU semantics found here: . More links are provided in the class documentation. In the following command line, the '\-\-opt1' option is a subcommand option: .sp .RS 4 $ atlas \-\-debug my\-command \-\-opt1 .RE .sp Also note that both the Perl option parsing and the SimpleOptionAndArgumentParser class recognize options using a shortest non-ambiguous prefix. For example, suppose 'my\-command' takes two options '\-\-large\-option' and '\-\-lazy\-option'. You could specify the '\-\-lazy\-option' by simply supplying '\-\-lazy' at the command line. However, supplying '\-\-la' would cause an error since 'la' is an ambiguous prefix between those two options. .SH "TAB COMPLETE" .sp \fBatlas\-shell\-tools\fR(7) supports various TAB completions for the \fBbash\fR(1) and \fBzsh\fR(1) shells. \fBatlas\fR(1) will contextually complete subcommand names in the current activated module, the '\-\-preset' option with various available presets, and local files. \fBatlas\-config\fR(1) will contextually complete subcommands as well as relevant arguments to whatever subcommand is provided on the command line. .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-cookbook.7 ================================================ .\" Title: atlas-cookbook .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-GLOSSARY" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-cookbook \-\- Atlas Shell Tools command cookbook .SH "SYNOPSIS" * .SH "RECIPES" .sp \fBUpdate everything from the default installation to latest:\fR .RS 4 .sp # First, update the toolkit .sp $ atlas\-config update .sp # Now, update the default atlas commands using the atlas repo .sp $ atlas\-config repo install atlas .sp # You can also update any other registered repos in a similar way .RE .sp \fBSee a list of all installed modules, activate , then list its commands:\fR .RS 4 .sp $ atlas\-config list .sp $ atlas\-config activate .sp $ atlas \-\-list .sp # You can also use the abbreviated \-l flag .sp $ atlas \-l .RE .sp \fBAdd a new repo, check your registered repos, and then install a module from it:\fR .sp .RS 4 .sp $ atlas\-config repo add my\-new\-repo https://github.com/me/my\-repo.git .sp $ atlas\-config repo list .sp $ atlas\-config repo install my\-new\-repo .RE .sp \fBBuild a fat JAR and then symlink install for local testing:\fR .sp .RS 4 .sp $ cd /path/to/my/project .sp $ ./gradlew clean fat \-x check .sp $ atlas\-config install \-\-symlink .sp # Now you can make some changes in your code, then simply rebuild to test them! .sp $ vim src/MySourceFile.java .sp $ ./gradlew clean fat \-x check .RE .sp \fBLower the log level to DEBUG and run a command, then reset everything to default:\fR .sp .RS 4 .sp $ atlas\-config log set\-level DEBUG .sp $ atlas my\-command .sp $ atlas\-config log reset .sp # Now show the log settings to check that it reset .sp $ atlas\-config log show .RE .sp \fBSave a global preset called 'foo' for some common options, then use it:\fR .sp .RS 4 .sp $ atlas\-config preset save\-global foo \-\-foo=bar \-\-baz=bat .sp $ atlas \-\-preset foo my\-command .sp $ atlas \-\-preset foo another\-command .RE .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-environment.7 ================================================ .\" Title: atlas-environment .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-ENVIRONMENT" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-environment \-\- Atlas Shell Tools environment variables .SH "SYNOPSIS" * .SH "DESCRIPTION" .sp The following are all environment variables utilized in some way by \fBatlas\-shell\-tools\fR(7). .sp \fBATLAS_SHELL_TOOLS_EDITOR\fR .sp .RS 4 Use the specified editor when performing tasks that require an editor. This value must either be a full path to an editor executable or an executable name that is found on the system path. \fBatlas\-shell\-tools\fR(7) will split the value of this variable on whitespace to create ARGV, thus you can include editor flags and options in the standard way. \fBatlas\-shell\-tools\fR(7) respects this variable first. If it is not set, it will try the system\-global \fBEDITOR\fR variable. If that is also not set, it will use a default editor. .RE .sp \fBATLAS_SHELL_TOOLS_HOME\fR .sp .RS 4 This variable should point to the location of the current \fBatlas\-shell\-tools\fR(7) installation. Provided you installed \fBatlas\-shell\-tools\fR(7) in the usual way (using the setup scripts), this variable should be automatically set for you in your shell startup file. You can temporarily switch to an alternate installation (e.g. for debugging purposes) by resetting this variable. You can also specify a new data directory by resetting \fBXDG_DATA_HOME\fR. .RE .sp \fBATLAS_SHELL_TOOLS_NO_COLOR\fR .sp .RS 4 Disable special formatted output for \fBatlas\-shell\-tools\fR(7) only. See info on \fBNO_COLOR\fR for contrast. .RE .sp \fBATLAS_SHELL_TOOLS_PAGER\fR .sp .RS 4 Use the specified pager when performing tasks that require a pager. This value must either be a full path to a pager executable or an executable name that is found on the system path. \fBatlas\-shell\-tools\fR(7) will split the value of this variable on whitespace to create ARGV, thus you can include pager flags and options in the standard way. \fBatlas\-shell\-tools\fR(7) respects this variable first. If it is not set, it will try the system\-global \fBPAGER\fR variable. If that is also not set, it will use a default pager. .RE .sp \fBATLAS_SHELL_TOOLS_USE_COLOR\fR .sp .RS 4 Enable special formatted output. Overrides the settings of \fBNO_COLOR\fR and \fBATLAS_SHELL_TOOLS_NO_COLOR\fR. .RE .sp \fBEDITOR\fR .sp .RS 4 Use the specified editor when performing tasks that require an editor. This value must either be a full path to an editor executable or an executable name that is found on the system path. \fBatlas\-shell\-tools\fR(7) will split the value of this variable on whitespace to create ARGV, thus you can include editor flags and options in the standard way. \fBatlas\-shell\-tools\fR(7) respects the system's \fBEDITOR\fR variable in the event that \fBATLAS_SHELL_TOOLS_EDITOR\fR is not set. If neither of these are set, it will use a default editor. .RE .sp \fBNO_COLOR\fR .sp .RS 4 Disable all special formatted output. \fBatlas\-shell\-tools\fR(7) respects this variable, as do other popular CLI tools. See . .RE .sp \fBPAGER\fR .sp .RS 4 Use the specified pager when performing tasks that require a pager. This value must either be a full path to a pager executable or an executable name that is found on the system path. \fBatlas\-shell\-tools\fR(7) will split the value of this variable on whitespace to create ARGV, thus you can include pager flags and options in the standard way. \fBatlas\-shell\-tools\fR(7) respects the system's \fBPAGER\fR variable in the event that \fBATLAS_SHELL_TOOLS_PAGER\fR is not set. If neither of these are set, it will use a default pager. .RE .sp \fBXDG_DATA_HOME\fR .sp .RS 4 \fBatlas\-shell\-tools\fR(7) stores program data in compliance with the XDG Base Directory specification, i.e. at $HOME/.local/share/atlas\-shell\-tools. Accordingly, it respects the \fBXDG_DATA_HOME\fR environment variable \- if set, \fBatlas\-shell\-tools\fR(7) will store program data at the base path specified by that variable. See \fBatlas\-plumbing\fR(5) for more information. .RE .sp .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-glossary.7 ================================================ .\" Title: atlas-glossary .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-GLOSSARY" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-glossary \-\- A glossary of Atlas Shell Tools terms and concepts .SH "SYNOPSIS" * .SH "DESCRIPTION" .sp \fBactive module index\fR .RS 4 The active module index is a file containing metadata about the commands in the presently activated module. The metadata includes the command names as well as the full classnames of the command implementations. A new index is generated each time a new module is activated. The index can also be regenerated with \fBatlas-config-sync\fR(1). See \fBatlas\-plumbing\fR(5) for info on where the index is stored. .RE .sp \fBARGV\fR .RS 4 Throughout the Atlas Shell Tools manual, ARGV refers to the argument vector from the command line. In the context of \fBatlas\fR(1), it generally refers to the actual arguments passed to the command. \fBatlas\fR(1) converts your command line like "$ atlas \-\-globalOpt MyCommand arg1 \-\-opt1" into a JVM call, so here ARGV refers to just ["arg1", "\-\-opt1"], the actual arguments given to the JVM. In the aforementioned example, '\-\-globalOpt' is referred to as a "global" option. See \fBoption\fR. .RE .sp \fBcommand\fR .RS 4 A command generally refers to a Java class that implements an Atlas Shell Tools command. In order to implement an Atlas Shell Tools command, the class must be a subclass of \fBAbstractAtlasShellToolsCommand\fR, found in the Atlas project: see . .RE .sp \fBmodule\fR .RS 4 Atlas Shell Tools lingo for a JARfile that contains implementations of commands (see "command" in this glossary). Once a JARfile has been installed into Atlas Shell Tools using \fBatlas\-config\fR(1), it is known as a module and is referred to using its module name, as reported by \fBatlas\-config\-list\fR(1). .sp See \fBmodule workspace\fR. .RE .sp \fBmodule workspace\fR .RS 4 The module workspace is where \fBatlas\-shell\-tools\fR(7) stores installed modules. The workspace can be displayed with \fBatlas\-config\-list\fR(1). More info about the workspace structure is detailed in \fBatlas\-plumbing\fR(5). .sp See \fBmodule\fR. .RE .sp \fBoption\fR .RS 4 An option refers to a special command line argument prefixed with a "\-" (a.k.a. a short option) or with a "\-\-" (a.k.a. a long option). Options can be either global (i.e. they come before the subcommand on the command line), or they can be tied to the subcommand itself. For example, in the command "atlas \-\-opt1 MyCommand \-\-opt2", "\-\-opt1" is a global option whereas "\-\-opt2" is an option of the MyCommand subcommand. Options can also take their own additional optional or required arguments. For more details on the intricacies of option syntax, see \fBatlas\-cli\fR(7). .sp Finally it should be noted that the option prefix dashes without attached options have special meanings. "\-" on its own is treated as a regular argument, and "\-\-" on its own is treated as the end\-of\-options marker. Again, see \fBatlas\-cli\fR(7) for more information. .RE .sp \fBpreset\fR .RS 4 A preset is a list of saved options for a given command. Presets have a name, but they are meaningless without their associated command context \- i.e. the preset "p1" is not meaningful unless you say the preset "p1" for command "MyCommand". Technically speaking, presets are additionally qualified by a namespace, but this can be effectively ignored unless that behavior is desired. Presets and their namespaces are manipulated using the \fBatlas\fR(1) and \fBatlas\-config\fR(1) programs. See \fBatlas\-presets\fR(7) for more information. .RE .sp \fBrepo\fR .RS 4 A repo is a remote \fBgit\fR(1) repository containing Atlas Shell Tools command implementations. The repo management interface is exposed by \fBatlas\-config\-repo\fR(1), and repo installation is performed using the \fBinstall\fR directive of \fBatlas\-config\-repo\fR(1). The repo object itself simply consists of some metadata \- specifically a remote URL, a ref name (could be a branch, commit, tag, etc.), and optional Gradle configuration data. More info on repo objects can be found in \fBatlas\-plumbing\fR(5). .RE .sp \fBtoolkit\fR .RS 4 Throughout the Atlas Shell Tools documentation, toolkit generally refers to the \fBatlas\fR(1) and \fBatlas\-config\fR(1) programs, the manpages included in the \fBatlas\-shell\-tools\fR(7) suite, and anything else stored in the directory pointed to by the \fBATLAS_SHELL_TOOLS_HOME\fR environment variable. The toolkit does \fBnot\fR refer to installed modules, repos, or any saved presets. Those are stored in the toolkit's data directory. See \fBatlas\-plumbing\fR(5) for more on the data directory. .RE .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-presets.7 ================================================ .\" Title: atlas-presets .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-PRESETS" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-presets \-\- Atlas Shell Tools preset interface .SH "SYNOPSIS" * .SH "DESCRIPTION" .sp Atlas Shell Tools features a robust preset management system. Presets provide a way to save command options (and long option arguments) for future application or modification. For more information about how Atlas Shell Tools saves and manipulates presets under the hood, see \fBatlas\-plumbing\fR(5). .sp The preset interface is divided into 3 tiers. Each tier supports an increasing number of potential use cases at the cost of added complexity for the user. The tiers are designed such that you can stop at any tier you feel comfortable using \- without needing to understand how the next tier up works. Each tier builds on concepts introduced in the previous tiers. The three interface tiers are: .sp .RS 4 \fBTier 1)\fR Basic preset creation and use with \fBatlas\fR(1) options \fB\-\-save\-preset\fR and \fB\-\-preset\fR. .RS 8 \fBGlobal Presets)\fR A brief addendum to Tier 1 about global preset saving. .RE \fBTier 2)\fR More precise preset management with \fBatlas-config-preset\fR(1), including editing, copying, etc. \fBTier 3)\fR Preset namespace management with the \fBatlas-config-preset\fR(1) \fBnamespace\fR directive. .RE .SH "TIER 1" Tier 1 provides a simple, lightweight interface for preset usage through the \fBatlas\fR(1) options \fB\-\-preset\fR=\fInames\fR and \fB\-\-save\-preset\fR=\fInew\-name\fR, where \fInames\fR are the name(s) of the preset(s) you would like to apply and \fInew\-name\fR is the name of the preset you would like to create. .sp When running a command with the \fB\-\-preset\fR=\fInames\fR option, \fBatlas\fR(1) checks the list of saved presets associated with that command. If one of those presets matches with one of the given \fInames\fR, \fBatlas\fR(1) will apply that preset. It will perform this check and application for each preset provided in \fInames\fR (which should be comma or colon separated). If one of the given \fInames\fR does not refer to a saved preset, \fBatlas\fR(1) displays the list of presets and exits with an error. .sp When running a command with the \fB\-\-save\-preset\fR=\fInew\-name\fR option, \fBatlas\fR attempts to save the current ARGV to a new preset called \fInew\-name\fR. If \fInew\-name\fR already exists, \fBatlas\fR(1) will exit with an error. .sp If both \fB\-\-preset\fR=\fInames\fR and \fB\-\-save\-preset\fR=\fInew\-name\fR are applied at the same time, \fBatlas\fR(1) will attempt to apply \fInames\fR before saving \fInew\-name\fR. This allows you to easily save new presets that iterate on previously created presets. .sp Note that preset names are bound to the target command at save\-time \- a preset name without its command context is meaningless. This means that preset "p1" for "ExampleCommand" and preset "p1" for "MyCommand" may contain completely different values. .sp Let's look at an example of the Tier 1 interface in action. We will run a command called "MyCommand" and assume preset "p1" does not yet exist: .sp .RS 4 $ atlas \-\-save\-preset=p1 MyCommand arg1 arg2 \-\-opt1 \-\-opt2=opt2Arg .RE .sp This will save a preset "p1" for "MyCommand" with contents ["\-\-opt1", "\-\-opt2=opt2Arg"] and then run the command. Note that the preset engine only saves options, and so will automatically discard any ARGV element that does not look like an option. For this reason, you \fImust\fR use the long option '=' syntax for specifying option arguments when saving a preset (e.g. '--opt=arg' and \fInot\fR '--opt arg'). .sp Now that you have saved preset "p1" for "MyCommand", you can apply it like: .sp .RS 4 $ atlas \-\-preset=p1 MyCommand arg1 arg2 .RE .sp This will run the exact same command as before, but it saves us from having to type out "\-\-opt1" and "\-\-opt2=opt2Arg". In addition to simply applying a preset verbatim, you can also override a saved option by specifying it again on the command line: .sp .RS 4 $ atlas \-\-preset=p1 MyCommand arg1 arg2 \-\-opt2=OverrideOpt2Arg .RE .sp This will run the same command as the above 2 examples, except it will override the preset value of \-\-opt2 with your new value "OverrideOpt2Arg". .sp You can extend presets by applying and saving at the same time. Here, we apply our preset "p1" while also saving a new preset called "p2", based off the contents of "p1": .sp .RS 4 $ atlas \-\-preset=p1 \-\-save\-preset=p2 MyCommand arg1 arg2 --opt2=OverrideOpt2Arg --opt3 .RE .sp The new preset "p2" will contain the following contents: ["\-\-opt1", "\-\-opt2=opt2Arg", "\-\-opt2=OverrideOpt2Arg", "\-\-opt3"]. Even though \-\-opt2 is repeated, this is OK. When multiple instances of the same option are supplied, the option parser will use ARGV's rightmost instance of that option. .sp Let's look at one final example where we apply two presets at once. Assuming you now have a preset "p3" with contents: ["\-\-opt3"], you could run: .sp .RS 4 $ atlas \-\-preset=p1,p3 MyCommand .RE .sp This would run MyCommand with ["\-\-opt1", "\-\-opt2=opt2Arg", "\-\-opt3"] as the effective ARGV. Note that the rightmost ARGV selection strategy for duplicate options applies here as well. Keep this in mind when applying multiple presets that contain the same option. .SS "Global Presets" In addition to the presets tied to each command, you can also save global presets using the \fBatlas\fR(1) option \fB\-\-save\-global\-preset\fR=\fInew\-name\fR. Global presets can be applied to any command using the \fB\-\-preset\fR=\fInames\fR option. When checking the \fInames\fR given to \fB\-\-preset\fR, the \fBatlas\fR(1) preset system will always attempt to use global presets as a fall back. This means, for example, that if you have a preset "p1" for "MyCommand" as well as global presets "p1" and "p2", supplying \fB\-\-preset=p1,p2\fR when running MyCommand will use the "p1" associated with "MyCommand", but it will use the global preset "p2". .sp For the following examples, let's assume we have a command "MyCommand" with a preset "p1". We also have a command "AnotherCommand" with no presets. Finally we have global presets "p1" and "p2". .sp The following will run "MyCommand" with its version of "p1" but with global "p2": .sp .RS 4 $ atlas \-\-preset=p1,p2 MyCommand .RE .sp This next command will run "AnotherCommand" with globals "p1" and "p2": .sp .RS 4 $ atlas \-\-preset=p1,p2 AnotherCommand .RE .sp Finally, this last command will fail since there is no preset "p3" associated with "MyCommand", nor is there a global definition of "p3": .sp .RS 4 $ atlas \-\-preset=p1,p2,p3 MyCommand .RE .sp .SH "TIER 2" Tier 2 provides more precise preset management using the \fBatlas\-config\fR(1) subcommand \fBpreset\fR. \fBatlas\-config\-preset\fR(1) takes a mandatory \fIdirective\fR, which is just a verb specifying a configuration action. Note that many of the directives require additional command context (recall that preset names are meaningless without an associated command). Below are the available directives: .sp .RS 4 \fBcopy\fR \fIcommand\fR .RS 4 Copy \fIcommand\fR preset into new preset . must not already exist, else the copy will fail. The following example copies the preset "p1" into new preset "p2", for command "MyCommand": .sp .RS 4 $ atlas\-config preset copy MyCommand p1 p2 .RE .sp The \fBcopy\fR directive is useful in combination with the \fBedit\fR directive \- when you want to make multiple versions of a long preset, each with some minor differences. .RE .RE .sp .RS 4 \fBcopy\-global\fR .RS 4 Copy global preset into new preset . must not already exist, else the copy will fail. The following example copies the global preset "p1" into new global preset "p2": .sp .RS 4 $ atlas\-config preset copy\-global p1 p2 .RE .sp Like the \fBcopy\fR directive, the \fBcopy\-global\fR is useful in combination with the \fBedit\-global\fR directive for quick preset iteration. .RE .RE .sp .RS 4 \fBedit\fR \fIcommand\fR .RS 4 Edit preset for \fIcommand\fR. If does not exist, then it will be created when the edit is successfully saved. The default preset editor is \fBvim\fR, but this can be changed by setting the \fBATLAS_SHELL_TOOLS_EDITOR\fR environment variable. The following example will edit preset "p1" for command "MyCommand": .sp .RS 4 $ atlas\-config preset edit MyCommand p1 .RE .RE .RE .sp .RS 4 \fBedit\-global\fR .RS 4 Edit global preset . If does not exist, then it will be created when the edit is successfully saved. The default preset editor is \fBvim\fR, but this can be changed by setting the \fBATLAS_SHELL_TOOLS_EDITOR\fR environment variable. The following example will edit global preset "p1": .sp .RS 4 $ atlas\-config preset edit\-global p1 .RE .RE .RE .sp .RS 4 \fBlist\fR [\fIcommand\fR [name]] .RS 4 List all available presets (including globals), or list all presets for a given [\fIcommand\fR], or list contents of preset [name] for [\fIcommand\fR]. The following example lists all available presets, then lists all presets for "MyCommand", and finally lists the contents of preset "p1" for command "MyCommand": .sp .RS 4 $ atlas\-config preset list $ atlas\-config preset list MyCommand $ atlas\-config preset list MyCommand p1 .RE .sp .RE .RE .sp .RS 4 \fBlist\-global\fR [name] .RS 4 List all available global presets, or list contents of global preset [name]. The following example lists all available global presets, then lists the contents of global preset "p1": .sp .RS 4 $ atlas\-config preset list\-global $ atlas\-config preset list\-global p1 .RE .sp .RE .RE .sp .RS 4 \fBnamespace\fR [namespace] .RS 4 Execute a on a given preset [namespace]. Available subdirectives are \fBlist\fR, \fBuse\fR, \fBcreate\fR, and \fBremove\fR. Preset namespaces \- and the \fBnamespace\fR directive \- are explained in more detail in the \fBTIER 3\fR section found below. .RE .RE .sp .sp .RS 4 \fBremove\fR \fIcommand\fR [name] .RS 4 Remove all presets for a given \fIcommand\fR, or remove the preset [name] for \fIcommand\fR. The following example removes all presets for command "MyCommand", then removes preset "p1" for command "AnotherCommand": .sp .RS 4 $ atlas\-config preset remove MyCommand $ atlas\-config preset remove AnotherCommand p1 .RE .sp .RE .RE .sp .RS 4 \fBremove\-global\fR [name] .RS 4 Remove all global presets , or remove the global preset [name]. The following example removes all global presets, then removes globa preset "p1": .sp .RS 4 $ atlas\-config preset remove\-global $ atlas\-config preset remove\-global p1 .RE .sp .RE .RE .sp .RS 4 \fBsave\fR \fIcommand\fR .RS 4 Save a preset for \fIcommand\fR without actually running the command. is a sequence of options to be saved in the preset. Again, recall that you must use the long option '=' syntax for specifying option arguments when saving a preset (e.g. '--opt=arg' and \fInot\fR '--opt arg'). The following example saves preset "p1" to command "MyCommand" with some options --opt1 and --opt2=opt2Arg: .sp .RS 4 $ atlas\-config preset save MyCommand p1 --opt1 --opt2=opt2Arg .RE .sp .RE .RE .sp .RS 4 \fBsave\-global\fR .RS 4 Save a global preset . is a sequence of options to be savedin the preset. Again, recall that you must use the long option '=' syntax for specifying option arguments when saving a preset (e.g. '--opt=arg' and \fInot\fR '--opt arg'). The following example saves global preset "p1" with some options --opt1 and --opt2=opt2Arg: .sp .RS 4 $ atlas\-config preset save\-global p1 --opt1 --opt2=opt2Arg .RE .sp .RE .RE .SH "TIER 3" Tier 3 provides preset namespaces. A namespace creates an enclosing scope for the presets associated with each command. For example, preset "p1" for command "MyCommand" under "namespace1" and preset "p1" for command "MyCommand" under "namespace2" may have completely different contents. Up to this point, we have been working under the default namespace, appropriately called "default". You can create and manage namespaces using the \fBatlas\-config\-preset\fR(1) \fBnamespace\fR directive, which takes a subdirective to denote the desired action. The available subdirectives are below: .sp .RS 4 \fBcreate\fR .RS 4 Create a new , throwing an error if already exists. This will not actually switch to the new namespace. The following example creates a namespace called "namespace1": .sp .RS 4 $ atlas\-config preset namespace create namespace1 .RE .RE .RE .RS 4 \fBlist\fR [namespace] .RS 4 List all namespaces while highlighting the current namespace with a "*", or list the contents of namespace [namespace]. The following example lists all namespaces, then lists the contents of namespace "namespace1". .sp .RS 4 $ atlas\-config preset namespace list $ atlas\-config preset namespace list namespace1 .RE .RE .RE .RS 4 \fBremove\fR .RS 4 Delete a , including all associated presets. The \fBremove\fR will fail if does not exist, if is currently in-use, or if is the default namespace. The following example removes a namespace called "namespace1": .sp .RS 4 $ atlas\-config preset namespace remove namespace1 .RE .RE .RE .RS 4 \fBuse\fR .RS 4 Switch to , throwing an error if does not exist. Any new presets you create will now be saved under , and presets you apply will be sourced from . The following example switches to a namespace called "namespace1": .sp .RS 4 $ atlas\-config preset namespace use namespace1 .RE .RE .RE .sp .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/man/man7/atlas-shell-tools.7 ================================================ .\" Title: atlas-shell-tools .\" Author: Lucas Cram .\" Source: atlas-shell-tools 1.0.0 .\" Language: English .\" .TH "ATLAS-SHELL-TOOLS" "7" "28 September 2020" "atlas\-shell\-tools 1\&.0\&.0" "Atlas Shell Tools Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" .sp atlas\-shell\-tools \-\- A command line toolkit for Atlas and related projects .SH "SYNOPSIS" * .SH "DESCRIPTION" The Atlas Shell Tools are a command line toolkit for running commands defined in Atlas and related projects. The main tool for running commands is \fBatlas\fR(1), and the installation can be configured with \fBatlas\-config\fR(1) and its various subcommands. .SH "MANUAL PAGES" .RS 4 \fBatlas\fR(1) .RS 4 Run an Atlas Shell Tools command .RE \fBatlas\-config\fR(1) .RS 4 Configure the Atlas Shell Tools installation .RE \fBatlas\-config\-activate\fR(1) .RS 4 Activate an installed module .RE \fBatlas\-config\-deactivate\fR(1) .RS 4 Deactivate an installed module .RE \fBatlas\-config\-install\fR(1) .RS 4 Install a new Atlas Shell Tools module from a local JAR .RE \fBatlas\-config\-list\fR(1) .RS 4 List installed modules and module statuses .RE \fBatlas\-config\-log\fR(1) .RS 4 Check or set log parameters .RE \fBatlas\-config\-preset\fR(1) .RS 4 Create and manage Atlas Shell Tools command presets .RE \fBatlas\-config\-repo\fR(1) .RS 4 Register and manage Atlas Shell Tools repositories .RE \fBatlas\-config\-reset\fR(1) .RS 4 Reset the installation .RE \fBatlas\-config\-sync\fR(1) .RS 4 Refresh the installation based on the active module .RE \fBatlas\-config\-uninstall\fR(1) .RS 4 Uninstall Atlas Shell Tools module(s) .RE \fBatlas\-config\-update\fR(1) .RS 4 Update the Atlas Shell Tools core toolkit .RE \fBatlas\-plumbing\fR(5) .RS 4 Details about how Atlas Shell Tools stores toolkit program data .RE \fBatlas\-cli\fR(7) .RS 4 Atlas Shell Tools command line interface and conventions .RE \fBatlas\-cookbook\fR(7) .RS 4 Atlas Shell Tools command cookbook .RE \fBatlas\-environment\fR(7) .RS 4 Atlas Shell Tools environment variables .RE \fBatlas\-glossary\fR(7) .RS 4 A glossary of Atlas Shell Tools terms and concepts .RE \fBatlas\-presets\fR(7) .RS 4 Atlas Shell Tools preset interface .RE .RE .SH "ATLAS SHELL TOOLS" .sp Part of the \fBatlas\-shell\-tools\fR(7) suite ================================================ FILE: atlas-shell-tools/quick_install_bash.sh ================================================ #!/bin/sh # Define a prompt function for re-use prompt_yn_was_yes() { prompt=$1 while true; do echo "$prompt" # handle EOF case if ! read -r answer; then exit 0 fi if [ "$answer" != "${answer#[Yy]}" ]; then # case user entered 'y' return 0 elif [ "$answer" != "${answer#[Nn]}" ]; then # case user entered 'n' return 1 fi echo "Please enter 'y' or 'n'..." done } # Utilize POSIX sh features ONLY for installation. Also, we exit the script # on any error. set -e # Grab the install location from first argument if present. install_location=$1 # Verify that you have programs needed by atlas-shell-tools if ! command -v less > /dev/null; then echo "Error: atlas-shell-tools requires the 'less' paging program." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 35-42." exit 1 fi if ! command -v man > /dev/null; then echo "Error: atlas-shell-tools requires the 'man' program." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 44-51." exit 1 fi if ! command -v vim > /dev/null; then echo "Error: atlas-shell-tools requires the 'vim' editor." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 53-60." exit 1 fi # Check the install location. If blank, we will use $HOME as default after # prompting the user for confirmation. if [ -z "$install_location" ]; then echo "Using the value of \$HOME ($HOME) as install location." echo "If an alternative location is desired, select 'n' and try:" echo echo " \$ sh quick_install_bash.sh /install/path" echo if prompt_yn_was_yes "OK to continue (y/n)?"; then install_location="$HOME" else exit 0 fi fi if [ ! -d "$install_location" ]; then echo "Error: $install_location does not exist" exit 1 fi if [ ! -w "$install_location" ]; then echo "Error: you do not have write permissions for $install_location" exit 1 fi base_folder="atlas-shell-tools" full_installation_path="$install_location/$base_folder" if [ -e "$full_installation_path" ]; then echo "Error: $full_installation_path already exists" exit 1 fi git clone https://github.com/osmlab/atlas.git "$full_installation_path" cd "$full_installation_path" git checkout main chmod +x atlas-shell-tools/scripts/atlas atlas-shell-tools/scripts/atlas-config export ATLAS_SHELL_TOOLS_HOME="$full_installation_path/atlas-shell-tools" export PATH="$PATH:$ATLAS_SHELL_TOOLS_HOME/scripts" # Install the core Atlas module using a repo ./atlas-shell-tools/scripts/atlas-config repo add atlas https://github.com/osmlab/atlas.git main ./atlas-shell-tools/scripts/atlas-config repo install atlas # Modify the bash startup files with appropriate settings start_startup_line="# atlas-shell-tools startup: added automatically by quick_install_bash.sh" end_startup_line="# END atlas-shell-tools startup" export_home_line="export ATLAS_SHELL_TOOLS_HOME=\"$full_installation_path/atlas-shell-tools\"" export_path_line="export PATH=\"\$PATH:\$ATLAS_SHELL_TOOLS_HOME/scripts\"" source_line="source \"\$ATLAS_SHELL_TOOLS_HOME/ast_completions.bash\"" echo echo "About to append ~/.bash_profile with:" echo echo " $start_startup_line"; echo " $export_home_line"; echo " $export_path_line"; echo " $source_line"; echo " $end_startup_line"; echo if prompt_yn_was_yes "Is this OK (y/n)?"; then { echo; echo "$start_startup_line"; echo "$export_home_line"; echo "$export_path_line"; echo "$source_line"; echo "$end_startup_line"; echo; } >> "$HOME/.bash_profile" fi echo echo "About to append ~/.bashrc with:" echo echo " $start_startup_line"; echo " $source_line"; echo " $end_startup_line"; echo if prompt_yn_was_yes "Is this OK (y/n)?"; then { echo; echo "$start_startup_line"; echo "$source_line"; echo "$end_startup_line"; echo; } >> "$HOME/.bashrc" fi # Complete the installation echo echo "Installation complete." echo "Restart your terminal and try 'man atlas-shell-tools' to get started." ================================================ FILE: atlas-shell-tools/quick_install_zsh.sh ================================================ #!/bin/sh # Define a prompt function for re-use prompt_yn_was_yes() { prompt=$1 while true; do echo "$prompt" # handle EOF case if ! read -r answer; then exit 0 fi if [ "$answer" != "${answer#[Yy]}" ]; then # case user entered 'y' return 0 elif [ "$answer" != "${answer#[Nn]}" ]; then # case user entered 'n' return 1 fi echo "Please enter 'y' or 'n'..." done } # Utilize POSIX sh features ONLY for installation. Also, we exit the script # on any error. set -e # Grab the install location from first argument if present. install_location=$1 # Verify that you have programs needed by atlas-shell-tools if ! command -v less > /dev/null; then echo "Error: atlas-shell-tools requires the 'less' paging program." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 35-42." exit 1 fi if ! command -v man > /dev/null; then echo "Error: atlas-shell-tools requires the 'man' program." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 44-51." exit 1 fi if ! command -v vim > /dev/null; then echo "Error: atlas-shell-tools requires the 'vim' editor." # Unfortunately, we cannot use $LINENO in POSIX sh. Make sure to manually # maintain this line number. echo "To install anyway, open $0 and comment out check on lines 53-60." exit 1 fi # Check the install location. If blank, we will use $HOME as default after # prompting the user for confirmation. if [ -z "$install_location" ]; then echo "Using the value of \$HOME ($HOME) as install location." echo "If an alternative location is desired, select 'n' and try:" echo echo " \$ sh quick_install_bash.sh /install/path" echo if prompt_yn_was_yes "OK to continue (y/n)?"; then install_location="$HOME" else exit 0 fi fi if [ ! -d "$install_location" ]; then echo "Error: $install_location does not exist" exit 1 fi if [ ! -w "$install_location" ]; then echo "Error: you do not have write permissions for $install_location" exit 1 fi base_folder="atlas-shell-tools" full_installation_path="$install_location/$base_folder" if [ -e "$full_installation_path" ]; then echo "Error: $full_installation_path already exists" exit 1 fi git clone https://github.com/osmlab/atlas.git "$full_installation_path" cd "$full_installation_path" git checkout main chmod +x atlas-shell-tools/scripts/atlas atlas-shell-tools/scripts/atlas-config export ATLAS_SHELL_TOOLS_HOME="$full_installation_path/atlas-shell-tools" export PATH="$PATH:$ATLAS_SHELL_TOOLS_HOME/scripts" # Install the core Atlas module using a repo ./atlas-shell-tools/scripts/atlas-config repo add atlas https://github.com/osmlab/atlas.git main ./atlas-shell-tools/scripts/atlas-config repo install atlas # Modify the bash startup files with appropriate settings start_startup_line="# atlas-shell-tools startup: added automatically by quick_install_zsh.sh" end_startup_line="# END atlas-shell-tools startup" export_home_line="export ATLAS_SHELL_TOOLS_HOME=\"$full_installation_path/atlas-shell-tools\"" export_path_line="export PATH=\"\$PATH:\$ATLAS_SHELL_TOOLS_HOME/scripts\"" source_line="source \"\$ATLAS_SHELL_TOOLS_HOME/ast_completions.zsh\"" echo echo "About to append ~/.zshenv with:" echo echo " $start_startup_line"; echo " $export_home_line"; echo " $export_path_line"; echo " $source_line"; echo " $end_startup_line"; echo if prompt_yn_was_yes "Is this OK (y/n)?"; then { echo; echo "$start_startup_line"; echo "$export_home_line"; echo "$export_path_line"; echo "$source_line"; echo "$end_startup_line"; echo; } >> "$HOME/.zshenv" fi echo echo "About to append ~/.zshrc with:" echo echo " $start_startup_line"; echo " $source_line"; echo " $end_startup_line"; echo if prompt_yn_was_yes "Is this OK (y/n)?"; then { echo; echo "$start_startup_line"; echo "$source_line"; echo "$end_startup_line"; echo; } >> "$HOME/.zshrc" fi # Complete the installation echo echo "Installation complete." echo "Restart your terminal and try 'man atlas-shell-tools' to get started." ================================================ FILE: atlas-shell-tools/scripts/atlas ================================================ #!/usr/bin/env perl use warnings; use strict; use File::Spec; use Getopt::Long qw(GetOptions); use POSIX; # Pull in code from the common modules use FindBin; use lib "$FindBin::Bin/common"; use ast_log_subsystem; use ast_preset_subsystem; use ast_module_subsystem; use ast_tty; use ast_utilities; ## ORGANIZATION ## This script is organized into 3 sections: ## 1) GLOBAL INITIALIZATION - initialize some useful global constants ## 2) SUBROUTINES - subroutines used by the command logic ## 3) EXECUTION LOGIC - the actual command logic, ie. 'main' ########## BEGIN GLOBAL INITIALIZATION ########## my $ansi_red = ast_tty::ansi_red(); my $ansi_green = ast_tty::ansi_green(); my $ansi_magenta = ast_tty::ansi_magenta(); my $ansi_bold = ast_tty::ansi_bold(); my $ansi_reset = ast_tty::ansi_reset(); my $ansi_bunl = ast_tty::ansi_begin_underln(); my $ansi_eunl = ast_tty::ansi_end_underln(); my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : $ansi_red; my $green_stdout = $no_colors_stdout ? "" : $ansi_green; my $magenta_stdout = $no_colors_stdout ? "" : $ansi_magenta; my $bold_stdout = $no_colors_stdout ? "" : $ansi_bold; my $reset_stdout = $no_colors_stdout ? "" : $ansi_reset; my $bunl_stdout = $no_colors_stdout ? "" : $ansi_bunl; my $eunl_stdout = $no_colors_stdout ? "" : $ansi_eunl; my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : $ansi_red; my $green_stderr = $no_colors_stderr ? "" : $ansi_green; my $magenta_stderr = $no_colors_stderr ? "" : $ansi_magenta; my $bold_stderr = $no_colors_stderr ? "" : $ansi_bold; my $reset_stderr = $no_colors_stderr ? "" : $ansi_reset; my $bunl_stderr = $no_colors_stderr ? "" : $ansi_bunl; my $eunl_stderr = $no_colors_stderr ? "" : $ansi_eunl; my $ast_path; my $skip_paging; my $quiet; my $debug_flag; my $program_name = $ast_utilities::COMMAND_PROGRAM; my $program_version = "$ast_utilities::ATLAS_SHELL_TOOLS_VERSION ($program_name program)"; ########## END GLOBAL INITIALIZATION ########## ########## BEGIN SUBROUTINES ########## sub atlas_unrecognized_command_message_and_exit { my $command = shift; ast_utilities::error_output($program_name, "no such command ${bold_stderr}${command}${reset_stderr}"); print STDERR "Try '${bold_stderr}${program_name} --list${reset_stderr}' for a list of commands.\n"; print STDERR "Try '${bold_stderr}${program_name} --help${reset_stderr}' for more information.\n\n"; my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my @subcommands = keys %subcommand_classes; # Determine the most similar command using Levenshtein distance my $closest_command; my $min_distance = undef; foreach my $candidate_command (@subcommands) { my $distance = ast_utilities::levenshtein($command, $candidate_command); if (!defined $min_distance) { $closest_command = $candidate_command; $min_distance = $distance; } elsif ($distance < $min_distance) { $closest_command = $candidate_command; $min_distance = $distance; } } print STDERR "The most similar command is: ${bold_stderr}${closest_command}${reset_stderr}\n"; exit 127; } sub atlas_show_contextual_help_menu_and_exit { my $context = shift; my $skip_paging = shift; my $ast_path = shift; unless (defined $skip_paging) { $skip_paging = 0; } my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my $subcommand_class = $subcommand_classes{$context}; unless (defined $subcommand_class) { atlas_unrecognized_command_message_and_exit($context); } my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my @activated_modules = ast_module_subsystem::get_activated_modules(\%modules); my $module = $activated_modules[0]; my $full_path_to_modules_folder = File::Spec->catfile($ast_path, $ast_module_subsystem::MODULES_FOLDER, "$module" . $ast_module_subsystem::MODULE_SUFFIX); my $full_path_to_log4j = File::Spec->catfile($ast_path, $ast_log_subsystem::LOG4J_FILE_PATH); my @java_command = (); push @java_command, 'java'; push @java_command, '-Xms2G'; push @java_command, '-Xmx2G'; push @java_command, '-cp'; push @java_command, "${full_path_to_modules_folder}"; push @java_command, "-Dlog4j.configuration=file:${full_path_to_log4j}"; push @java_command, "${subcommand_class}"; push @java_command, "--help"; if (ast_tty::is_no_colors_stdout()) { push @java_command, "$ast_utilities::JAVA_NO_COLOR_STDOUT"; } else { push @java_command, "$ast_utilities::JAVA_COLOR_STDOUT"; } if (ast_tty::is_no_colors_stderr()) { push @java_command, "$ast_utilities::JAVA_NO_COLOR_STDERR"; } else { push @java_command, "$ast_utilities::JAVA_COLOR_STDERR"; } push @java_command, "$ast_utilities::JAVA_NO_USE_PAGER"; my $terminal_width = ast_tty::terminal_width(); push @java_command, "$terminal_width"; push @java_command, "$ast_utilities::JAVA_MARKER_SENTINEL"; my @pager_command = ast_utilities::get_pager(); if ($debug_flag) { print("Would execute JVM command:\n"); print("@java_command\n"); print("Then pipe into:\n"); print("@pager_command\n"); exit 0; } # NOTE: there is no easy way to prevent shell interference should the java # command array contain only one element. open JAVA, "-|", @java_command or die $!; my $output = ''; while () { # Not the most efficient way to do things. # Perhaps some kind of slurp is needed. File::Slurp could work but does # have an outstanding Unicode bug. Need to investigate more. $output = $output . $_; } close JAVA; if ($skip_paging) { print "$output"; } else { # NOTE: there is no easy way to prevent shell interference should the pager # command array contain only one element. open PAGER, "|-", @pager_command or die $!; print PAGER "$output"; close PAGER; } exit 0; } sub atlas_show_class_of_and_exit { my $ast_path = shift; my $class_of = shift; my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my $subcommand_class = $subcommand_classes{$class_of}; unless (defined $subcommand_class) { atlas_unrecognized_command_message_and_exit($class_of); } print "$subcommand_class\n"; exit 0; } sub atlas_list_subcommands_and_exit { my $ast_path = shift; my $skip_paging = shift; unless (defined $skip_paging) { $skip_paging = 0; } my %subcommand_desc = ast_module_subsystem::get_subcommand_to_description_hash($ast_path); my @pager_command = ast_utilities::get_pager(); if ($skip_paging) { print "\n"; print "${bold_stdout}AVAILABLE COMMANDS${reset_stdout}\n"; print "See the help page for a command with ${bold_stdout}${program_name} --help ${reset_stdout}.\n\n"; foreach my $subcommand (sort {lc $a cmp lc $b} keys %subcommand_desc) { print " ${bold_stdout}$subcommand${reset_stdout}\n"; print " $subcommand_desc{$subcommand}\n\n"; } print "\n"; } else { # NOTE: there is no easy way to prevent shell interference should the pager # command array contain only one element. open PAGER, "|-", @pager_command or die $!; print PAGER "${bold_stdout}AVAILABLE COMMANDS${reset_stdout}\n"; print PAGER "See the help page for a command with ${bold_stdout}${program_name} --help ${reset_stdout}.\n"; print PAGER "You can open a new page directly from this window with ${bold_stdout}!${program_name} --help ${reset_stdout}.\n\n"; foreach my $subcommand (sort {lc $a cmp lc $b} keys %subcommand_desc) { print PAGER " ${bold_stdout}$subcommand${reset_stdout}\n"; print PAGER " $subcommand_desc{$subcommand}\n\n"; } close PAGER; } exit 0; } ########## END SUBROUTINES ########## ########## BEGIN EXECUTION LOGIC ########## ast_utilities::verify_environment_or_exit(); $ast_path = ast_utilities::create_data_directory(); # Handle atlas global options. Global options are options that come before the # supplied subcommand. Subcommand options are handled by the command implementation. my $memory = '8G'; my $help_argument; my $show_list; my $class_of; my $save_preset; my $save_global_preset; my $use_preset; my $remove_preset; my $all_presets; my $show_preset; my $edit_preset; my $allow_run_as_root; Getopt::Long::Configure(qw(no_ignore_case_always)); GetOptions( "no-pager" => \$skip_paging, "memory|m=s" => \$memory, "help|h:s" => \$help_argument, "version|V" => sub { print "$program_version\n"; exit 0; }, "quiet|q" => \$quiet, "list|l" => \$show_list, "class-of=s" => \$class_of, "preset|p=s" => \$use_preset, "save-preset=s" => \$save_preset, "save-global-preset=s" => \$save_global_preset, "debug" => \$debug_flag, "allow-run-as-root" => \$allow_run_as_root, # This callback occurs the first time we see a non-option argument. # In our case, this will be the subcommand. "<>" => sub { my ($arg) = @_; if ($arg =~ m{^-}) { unless ($arg eq '-') { die "FATAL error: unhandled global option $arg"; } } # add the subcommand to the front of ARGV unshift @ARGV, $arg; die "!FINISH"; } ) or ast_utilities::getopt_failure_and_exit($program_name); if (geteuid() == 0) { unless (defined $allow_run_as_root) { print STDERR "For security reasons, you are highly discouraged from running atlas-shell-tools\n"; print STDERR "as the root user. Atlas-shell-tools cannot guarantee that modules installed from\n"; print STDERR "external repositories are safe to run with root privileges.\n\n"; print STDERR "To disregard this warning and run as root anyway, please use the option:\n"; print STDERR "--allow-run-as-root\n\n"; exit 1; } } # Handle the case where the user supplied a --help flag with no arg. # We can show this without doing any other verification. # Just display the man page and exit. if (defined $help_argument) { if ($help_argument eq '') { my @man_command = ast_utilities::get_man($skip_paging); if (scalar @man_command == 0) { ast_utilities::error_output($program_name, "could not obtain \'man\' command"); print STDERR "Please ensure a valid \'man\' command is on your path.\n"; exit 1; } my @command = (); push @command, @man_command; push @command, "$program_name"; system {$command[0]} @command; my $exitcode = $? >> 8; if ($exitcode != 0) { exit 1; } exit 0; } } my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my %modules_links = ast_module_subsystem::get_module_to_symlink_hash($ast_path); # If there are no modules, let's throw an error unless (keys %modules) { ast_utilities::error_output($program_name, 'found no installed modules'); print STDERR "Try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} install /path/to/module.jar${reset_stderr}' to install a module.\n"; exit 1; } my @activated_modules = ast_module_subsystem::get_activated_modules(\%modules); # If there are modules but none are active, warn the user if (scalar @activated_modules == 0) { ast_utilities::error_output($program_name, 'no activated module'); print STDERR "Try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} list${reset_stderr}' to see all installed modules.\n"; print STDERR "Then try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} activate ${reset_stderr}' to activate.\n"; exit 1; } # If the currently active module is a broken symlink, warn the user if ($modules_links{$activated_modules[0]} == $ast_module_subsystem::BROKEN_SYMLINK) { ast_utilities::error_output($program_name, 'current active module is a broken symlink'); print STDERR "To see the link value, try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} list${reset_stderr}'.\n"; print STDERR "Fix the link, then run '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} sync${reset_stderr}' to resolve.\n"; exit 1; } # If there is no active module index, warn the user my $index_path = File::Spec->catfile($ast_path, $ast_module_subsystem::ACTIVE_INDEX_PATH); unless (-f $index_path) { ast_utilities::error_output($program_name, 'could not find active module index'); print STDERR "Try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} sync${reset_stderr}' to resolve.\n"; exit 1; } if ($debug_flag) { print "---- ATLAS DEBUG MODE ----\n"; } # Handle case where user entered --help=TOPIC flag # We waited until after verifying that a command index exists if (defined $help_argument) { unless ($help_argument eq '') { atlas_show_contextual_help_menu_and_exit($help_argument, $skip_paging, $ast_path); } } my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); # Handle --class-of option if (defined $class_of) { atlas_show_class_of_and_exit($ast_path, $class_of); } # Handle --list option if (defined $show_list) { atlas_list_subcommands_and_exit($ast_path, $skip_paging); } # All global options have been processed, so shift the subcommand off of ARGV my $subcommand = shift @ARGV; unless (defined $subcommand) { ast_utilities::error_output($program_name, "missing required command or option"); print STDERR "Try '${bold_stderr}${program_name} --list${reset_stderr}' for a list of commands.\n"; print STDERR "Try '${bold_stderr}${program_name} --help${reset_stderr}' for more information.\n"; exit 1; } my $subcommand_class = $subcommand_classes{$subcommand}; # Case where the user entered an invalid subcommand unless (defined $subcommand_class) { atlas_unrecognized_command_message_and_exit($subcommand); } # Set @new_argv to @ARGV for now. @new_argv will get overriden by an applied # preset if the user supplied one. my @new_argv = @ARGV; my $current_namespace = ast_preset_subsystem::get_namespace($ast_path); # Check if a preset is being saved or used. We check for preset application # first, and apply it if necessary. We then check for a preset save. This allows # users to easily extend presets they have already created by applying and # saving in a single step. if (defined $use_preset) { @new_argv = ast_preset_subsystem::apply_preset_or_exit($ast_path, $program_name, $quiet, $use_preset, $subcommand, $current_namespace, \@ARGV); if ($debug_flag) { print "Applying preset(s) ${use_preset} for ${subcommand} in namespace ${current_namespace}\n"; } } if (defined $save_preset) { unless (ast_preset_subsystem::preset_regex_ok($save_preset)) { ast_utilities::error_output($program_name, "invalid preset name ${bold_stderr}${save_preset}${reset_stderr}"); print STDERR "Name must match regex: " . ast_preset_subsystem::preset_regex() . "\n"; exit 1; } if ($debug_flag) { print "Would save preset ${save_preset} for ${subcommand} to namespace ${current_namespace}\n"; print "Preset ARGV: \"@new_argv\"\n\n"; } else { my $success = ast_preset_subsystem::save_preset($ast_path, $program_name, $quiet, $save_preset, $subcommand, $current_namespace, \@new_argv); unless ($success) { exit 1; } print "Launching command ${bold_stdout}${subcommand}${reset_stdout}...\n\n"; } } if (defined $save_global_preset) { unless (ast_preset_subsystem::preset_regex_ok($save_global_preset)) { ast_utilities::error_output($program_name, "invalid preset name ${bold_stderr}${save_global_preset}${reset_stderr}"); print STDERR "Name must match regex: " . ast_preset_subsystem::preset_regex() . "\n"; exit 1; } if ($debug_flag) { print "Would save global preset ${save_global_preset} to namespace ${current_namespace}\n"; print "Preset ARGV: \"@new_argv\"\n\n"; } else { my $success = ast_preset_subsystem::save_preset($ast_path, $program_name, $quiet, $save_global_preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace, \@new_argv); unless ($success) { exit 1; } print "Launching command ${bold_stdout}${subcommand}${reset_stdout}...\n\n"; } } # Set up the subcommand to execute using the JVM my $module = $activated_modules[0]; my $full_path_to_modules_folder = File::Spec->catfile($ast_path, $ast_module_subsystem::MODULES_FOLDER, "$module" . $ast_module_subsystem::MODULE_SUFFIX); my $full_path_to_log4j = File::Spec->catfile($ast_path, $ast_log_subsystem::LOG4J_FILE_PATH); my @java_command = (); push @java_command, "java"; push @java_command, "-Xms$memory"; push @java_command, "-Xmx$memory"; push @java_command, "-cp"; push @java_command, "${full_path_to_modules_folder}"; push @java_command, "-Dlog4j.configuration=file:${full_path_to_log4j}"; push @java_command, "${subcommand_class}"; # Surround each arg in quotes in case it contains whitespace foreach my $arg (@new_argv) { push @java_command, "$arg"; } # Append the special TTY formatting sentinel arguments if (ast_tty::is_no_colors_stdout()) { push @java_command, "${ast_utilities::JAVA_NO_COLOR_STDOUT}"; } else { push @java_command, "${ast_utilities::JAVA_COLOR_STDOUT}"; } if (ast_tty::is_no_colors_stderr()) { push @java_command, "${ast_utilities::JAVA_NO_COLOR_STDERR}"; } else { push @java_command, "${ast_utilities::JAVA_COLOR_STDERR}"; } if ($skip_paging) { push @java_command, "${ast_utilities::JAVA_NO_USE_PAGER}"; } else { push @java_command, "${ast_utilities::JAVA_USE_PAGER}"; } my $terminal_width = ast_tty::terminal_width(); push @java_command, "${terminal_width}"; push @java_command, "${ast_utilities::JAVA_MARKER_SENTINEL}"; if ($debug_flag) { print("Would execute JVM command:\n"); print("@java_command\n"); exit 0; } # Here we use system's indirect object syntax to correctly handle all possible # edge cases regarding the configuration of @java_command. # See bottom of page: http://perldoc.perl.org/functions/exec.html system {$java_command[0]} @java_command; my $exitcode = $? >> 8; exit $exitcode; ########## END EXECUTION LOGIC ########## ================================================ FILE: atlas-shell-tools/scripts/atlas-config ================================================ #!/usr/bin/env perl use warnings; use strict; use Getopt::Long qw(GetOptions); use File::Basename; use File::Copy qw(copy); use File::Spec; use File::Path; use POSIX; # Pull in code from the common modules use FindBin; use lib "$FindBin::Bin/common"; use ast_completions; use ast_log_subsystem; use ast_module_subsystem; use ast_preset_subsystem; use ast_repo_subsystem; use ast_utilities; use ast_tty; ## ORGANIZATION ## This script is organized into 3 sections: ## 1) GLOBAL INITIALIZATION - initialize some useful global constants ## 2) SUBROUTINES - helper subroutines ## 3) COMMAND SUBROUTINES - subcommand execution subroutines ## 4) EXECUTION LOGIC - the actual command logic, ie. 'main' ########## BEGIN GLOBAL INITIALIZATION ########## my $ansi_red = ast_tty::ansi_red(); my $ansi_green = ast_tty::ansi_green(); my $ansi_magenta = ast_tty::ansi_magenta(); my $ansi_bold = ast_tty::ansi_bold(); my $ansi_blink = ast_tty::ansi_blink(); my $ansi_reset = ast_tty::ansi_reset(); my $ansi_bunl = ast_tty::ansi_begin_underln(); my $ansi_eunl = ast_tty::ansi_end_underln(); my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : $ansi_red; my $green_stdout = $no_colors_stdout ? "" : $ansi_green; my $magenta_stdout = $no_colors_stdout ? "" : $ansi_magenta; my $bold_stdout = $no_colors_stdout ? "" : $ansi_bold; my $blink_stdout = $no_colors_stdout ? "" : $ansi_blink; my $reset_stdout = $no_colors_stdout ? "" : $ansi_reset; my $bunl_stdout = $no_colors_stdout ? "" : $ansi_bunl; my $eunl_stdout = $no_colors_stdout ? "" : $ansi_eunl; my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : $ansi_red; my $green_stderr = $no_colors_stderr ? "" : $ansi_green; my $magenta_stderr = $no_colors_stderr ? "" : $ansi_magenta; my $bold_stderr = $no_colors_stderr ? "" : $ansi_bold; my $blink_stderr = $no_colors_stderr ? "" : $ansi_blink; my $reset_stderr = $no_colors_stderr ? "" : $ansi_reset; my $bunl_stderr = $no_colors_stderr ? "" : $ansi_bunl; my $eunl_stderr = $no_colors_stderr ? "" : $ansi_eunl; my $ast_path; my $skip_paging; my $quiet; my $program_name = $ast_utilities::CONFIG_PROGRAM; my $program_version = "$ast_utilities::ATLAS_SHELL_TOOLS_VERSION ($program_name program)"; # logging definitions my %valid_levels = ( "ALL" => 1, "TRACE" => 1, "DEBUG" => 1, "INFO" => 1, "WARN" => 1, "ERROR" => 1, "FATAL" => 1, "OFF" => 1 ); my @valid_level_keys = keys %valid_levels; my %valid_streams = ( "stdout" => 1, "stderr" => 1 ); my @valid_stream_keys = keys %valid_streams; ########## END GLOBAL INITIALIZATION ########## ########## BEGIN SUBROUTINES ########## sub show_submanpage_and_exit { my $subpage = shift; my @man_command = ast_utilities::get_man($skip_paging); if (scalar @man_command == 0) { ast_utilities::error_output($program_name, "could not obtain \'man\' command"); print STDERR "Please ensure a valid \'man\' command is on your path.\n"; exit 1; } my @command = (); push @command, @man_command; if ($subpage eq '') { push @command, "$program_name"; } else { push @command, "$program_name-$subpage"; } system {$command[0]} @command; exit 0; } # helper routine for the 'preset' subcommand sub atlas_cfgpreset_namespace { my $current_namespace = shift; my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my $success = 1; my $subdirective = shift @ARGV; unless (defined $subdirective) { ast_utilities::error_output($program_name, "${bold_stderr}${program_name} preset namespace${reset_stderr} requires a subdirective"); print STDERR "Try ${bold_stderr}create${reset_stderr}, ${bold_stderr}list${reset_stderr}, ${bold_stderr}remove${reset_stderr}, or ${bold_stderr}use${reset_stderr}.\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } if ($subdirective eq 'list') { my $namespace = shift @ARGV; if (defined $namespace) { $success = ast_preset_subsystem::all_presets($ast_path, $program_name, $quiet, $namespace); } else { $success = ast_preset_subsystem::all_namespaces($ast_path, $program_name, $quiet); } } elsif ($subdirective eq 'use') { my $namespace = shift @ARGV; unless (defined $namespace) { ast_utilities::error_output($program_name, "must specify a namespace"); print STDERR "Usage: ${bold_stderr}${program_name} preset namespace use ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::use_namespace($ast_path, $program_name, $quiet, $namespace); } elsif ($subdirective eq 'create') { my $new_namespace = shift @ARGV; unless (defined $new_namespace) { ast_utilities::error_output($program_name, "must specify a new namespace"); print STDERR "Usage: ${bold_stderr}${program_name} preset namespace create ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::create_namespace($ast_path, $program_name, $quiet, $new_namespace); } elsif ($subdirective eq 'remove') { my $namespace = shift @ARGV; unless (defined $namespace) { ast_utilities::error_output($program_name, "must specify a namespace"); print STDERR "Usage: ${bold_stderr}${program_name} preset namespace remove ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::remove_namespace($ast_path, $program_name, $quiet, $namespace); } else { ast_utilities::error_output($program_name, "unrecognized ${bold_stderr}${program_name} preset namespace${reset_stderr} subdirective '${bold_stderr}${subdirective}${reset_stderr}'"); print STDERR "Try ${bold_stderr}create${reset_stderr}, ${bold_stderr}list${reset_stderr}, ${bold_stderr}remove${reset_stderr}, or ${bold_stderr}use${reset_stderr}.\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } return 1; } ########## END SUBROUTINES ########## ########## BEGIN COMMAND SUBROUTINES ########## sub execute_command_activate { my $help_flag; GetOptions( 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "activate"); if (defined $help_flag) { show_submanpage_and_exit('activate'); } my $module_to_activate = shift @ARGV; unless (defined $module_to_activate) { ast_utilities::error_output($program_name . ": activate", "missing required argument"); ast_utilities::getopt_failure_and_exit($program_name, "activate"); } my %modules = get_module_to_status_hash($ast_path); unless (exists $modules{$module_to_activate}) { ast_utilities::error_output($program_name . ": activate", "no such module ${bold_stderr}${module_to_activate}${reset_stderr}"); return 0; } # deactivate all other modules foreach my $module (keys %modules) { if ($modules{$module} == $ast_module_subsystem::ACTIVATED) { ast_module_subsystem::perform_deactivate($module, $ast_path, $program_name, $quiet); } } my $success = ast_module_subsystem::perform_activate($module_to_activate, $ast_path, $program_name, $quiet); return $success; } sub execute_command_deactivate { my $help_flag; GetOptions( 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "deactivate"); if (defined $help_flag) { show_submanpage_and_exit('deactivate'); } my $module_to_deactivate = shift @ARGV; unless (defined $module_to_deactivate) { ast_utilities::error_output($program_name . ": deactivate", "missing required argument"); ast_utilities::getopt_failure_and_exit($program_name, "deactivate"); } my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); unless (exists $modules{$module_to_deactivate}) { ast_utilities::error_output($program_name . ": deactivate", "no such module ${bold_stderr}${module_to_deactivate}${reset_stderr}"); return 0; } my $success = ast_module_subsystem::perform_deactivate($module_to_deactivate, $ast_path, $program_name, $quiet); if ($success) { ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); } return $success; } sub execute_command_install { my $syminstall = 0; my $force_install = 0; my $skip_install = 0; my $install_deactivated = 0; my $alternate_name = ""; my $help_flag; GetOptions( 'symlink|s' => \$syminstall, 'deactivated' => \$install_deactivated, 'force' => \$force_install, 'skip' => \$skip_install, 'name=s' => \$alternate_name, 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "install"); if (defined $help_flag) { show_submanpage_and_exit('install'); } my $module_to_install = shift @ARGV; unless (defined $module_to_install) { ast_utilities::error_output($program_name . ": install", "missing required argument"); ast_utilities::getopt_failure_and_exit($program_name, "install"); } my %metadata; if ($syminstall) { $metadata{$ast_module_subsystem::SOURCE_KEY} = "symlink"; } else { $metadata{$ast_module_subsystem::SOURCE_KEY} = "local_file"; } my $uri_abs = Cwd::realpath($module_to_install); $metadata{$ast_module_subsystem::URI_KEY} = "file://" . $uri_abs; $metadata{$ast_module_subsystem::DATE_TIME_KEY} = POSIX::strftime("%Y-%m-%d %H:%M:%S UTC", gmtime(time)); my $success = ast_module_subsystem::perform_install($module_to_install, $ast_path, $program_name, $alternate_name, $syminstall, $skip_install, $force_install, $install_deactivated, \%metadata, $quiet); unless ($success) { return 0; } return 1; } sub execute_command_list { my $current = 0; my $help_flag; my $one_line = 0; GetOptions( 'current|c' => \$current, 'help|h' => \$help_flag, 'one-line|1' => \$one_line ) or ast_utilities::getopt_failure_and_exit($program_name, "list"); if (defined $help_flag) { show_submanpage_and_exit('list'); } my $modules_folder = File::Spec->catfile($ast_path, $ast_module_subsystem::MODULES_FOLDER); my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my %symlinks = ast_module_subsystem::get_module_to_symlink_hash($ast_path); my %targets = ast_module_subsystem::get_module_to_target_hash($ast_path); my %metadata = ast_module_subsystem::get_module_to_metadata_hash($ast_path); unless (keys %modules) { ast_utilities::error_output($program_name . ": list", "found no installed modules"); print STDERR "Try '${bold_stderr}${program_name} install /path/to/module.jar${reset_stderr}' to install a module.\n"; print STDERR "Or try '${bold_stderr}${program_name} repo install atlas${reset_stderr}' to install the commands from the atlas repo.\n"; return 0; } if ($current) { print "${bold_stdout}Currently active module:${reset_stdout}\n\n"; } else { print "${bold_stdout}Installed modules:${reset_stdout}\n\n"; } # Sort the module names alphabetically. We use 'lc' to convert them to # lowercase, since by default 'sort' uses ASCII ordering. foreach my $module (sort {lc $a cmp lc $b} keys %modules) { my $status = $modules{$module}; my $symlink = $symlinks{$module}; my $target = $targets{$module}; my %module_metadata; if (defined $metadata{$module}) { %module_metadata = %{$metadata{$module}}; } else { %module_metadata = (); } my $display = ' '; if ($current && $status != 1) { next; } # if activated, place a star next to the name if ($status == 1) { $display = $display . '*'; } else { $display = $display . ' '; } # choose an appropriate color for the display if ($status == $ast_module_subsystem::ACTIVATED && ($symlink == $ast_module_subsystem::REAL_FILE || $symlink == $ast_module_subsystem::GOOD_SYMLINK)) { $display = $display . "${green_stdout}${bold_stdout}"; } elsif ($symlink == $ast_module_subsystem::BROKEN_SYMLINK) { $display = $display . "${red_stdout}${bold_stdout}"; } # show the module name! $display = $display . " ${bold_stdout}${module}${reset_stdout}"; # show a big message if the symlink is broken, blink if also activated if ($symlink == $ast_module_subsystem::BROKEN_SYMLINK) { if ($status == $ast_module_subsystem::ACTIVATED) { $display = $display . " ${bold_stdout}${blink_stdout}(BROKEN SYMLINK)${reset_stdout}"; } else { $display = $display . " ${bold_stdout}(BROKEN SYMLINK)${reset_stdout}"; } } # if we were a symlink, show the target after the module name if ($symlink != $ast_module_subsystem::REAL_FILE) { $display = $display . " -> ${target}"; } print "$display\n"; unless ($one_line) { if (%module_metadata) { foreach my $metadata_key (sort {lc $a cmp lc $b} keys %module_metadata) { print " ${metadata_key}: $module_metadata{$metadata_key}\n"; } } print "\n"; } } if ($one_line) { print "\n"; } return 1; } sub execute_command_log { my $help_flag; GetOptions( 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "log"); if (defined $help_flag) { show_submanpage_and_exit('log'); } my $directive = shift @ARGV; my $success; unless (defined $directive) { ast_utilities::error_output($program_name . ": log", "missing required directive"); print STDERR "Usage: ${bold_stderr}${program_name} log ${reset_stderr}\n"; print STDERR "Available directives: ${bold_stderr}reset${reset_stderr}, ${bold_stderr}set-level${reset_stderr}, ${bold_stderr}set-stream${reset_stderr}, or ${bold_stderr}show${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } my $log_folder = File::Spec->catfile($ast_path, $ast_log_subsystem::LOG4J_FOLDER); my $logfile_path = File::Spec->catfile($log_folder, $ast_log_subsystem::LOG4J_FILE); my $current_level = ast_log_subsystem::read_loglevel_from_file($logfile_path); my $current_stream = ast_log_subsystem::read_logstream_from_file($logfile_path); if ($directive eq 'reset') { ast_log_subsystem::reset_log4j($ast_path); } elsif ($directive eq 'set-level') { my $new_level = shift @ARGV; unless (defined $new_level) { ast_utilities::error_output($program_name . ": log", "must specify a new level"); print STDERR "Usage: ${bold_stderr}${program_name} log set-level ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } if (exists($valid_levels{$new_level})) { ast_log_subsystem::replace_loglevel_in_file($logfile_path, $new_level); } else { ast_utilities::error_output($program_name . ": log", "unrecognized log level ${bold_stderr}${new_level}${reset_stderr}"); print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } } elsif ($directive eq 'set-stream') { my $new_stream = shift @ARGV; unless (defined $new_stream) { ast_utilities::error_output($program_name . ": log", "must specify a new stream"); print STDERR "Usage: ${bold_stderr}${program_name} log set-stream ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } if (exists($valid_streams{$new_stream})) { ast_log_subsystem::replace_logstream_in_file($logfile_path, $new_stream); } else { ast_utilities::error_output($program_name . ": log", "unrecognized log stream ${bold_stderr}${new_stream}${reset_stderr}"); print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } } elsif ($directive eq 'show') { $current_level = ast_log_subsystem::read_loglevel_from_file($logfile_path); $current_stream = ast_log_subsystem::read_logstream_from_file($logfile_path); print "Current log level: ${bold_stdout}${current_level}${reset_stdout}\n"; print "Current log stream: ${bold_stdout}${current_stream}${reset_stdout}\n"; } else { ast_utilities::error_output($program_name . ": log", "unrecognized ${bold_stderr}${program_name} log${reset_stderr} directive '${bold_stderr}${directive}${reset_stderr}'"); print STDERR "Available directives: ${bold_stderr}reset${reset_stderr}, ${bold_stderr}set-level${reset_stderr}, ${bold_stderr}set-stream${reset_stderr}, or ${bold_stderr}show${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} log --help${reset_stderr}\' for more information.\n"; return 0; } return 1; } sub execute_command_preset { my $help_flag; GetOptions( 'help|h' => \$help_flag, # This callback occurs the first time we see a non-option argument. # In our case, this will be the subcommand. "<>" => sub { my ($arg) = @_; if ($arg =~ m{^-}) { unless ($arg eq '-') { die "FATAL error: unhandled global option $arg"; } } # add the subcommand to the front of ARGV unshift @ARGV, $arg; die "!FINISH"; } ) or ast_utilities::getopt_failure_and_exit($program_name, "preset"); if (defined $help_flag) { show_submanpage_and_exit('preset'); } my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my $current_namespace = ast_preset_subsystem::get_namespace($ast_path); my $directive = shift @ARGV; my $success; unless (defined $directive) { ast_utilities::error_output($program_name . ": preset", "missing required directive"); print STDERR "Usage: ${bold_stderr}${program_name} preset ${reset_stderr}\n"; print STDERR "Available directives: ${bold_stderr}copy${reset_stderr}, ${bold_stderr}edit${reset_stderr}, ${bold_stderr}list${reset_stderr}, ${bold_stderr}namespace${reset_stderr}, ${bold_stderr}remove${reset_stderr}, or ${bold_stderr}save${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } if ($directive eq 'copy') { my $command_context = shift @ARGV; unless (defined $command_context) { ast_utilities::error_output($program_name . ": preset", "must specify a command"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $subcommand_classes{$command_context}) { ast_utilities::error_output($program_name . ": preset", "no such command ${bold_stderr}${command_context}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } my $src_preset = shift @ARGV; my $dest_preset = shift @ARGV; unless (defined $src_preset) { ast_utilities::error_output($program_name . ": preset", "must specify a source preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $dest_preset) { ast_utilities::error_output($program_name . ": preset", "must specify a destination preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::copy_preset($ast_path, $program_name, $quiet, $src_preset, $dest_preset, $command_context, $current_namespace); } elsif ($directive eq 'copy-global') { my $src_preset = shift @ARGV; my $dest_preset = shift @ARGV; unless (defined $src_preset) { ast_utilities::error_output($program_name . ": preset", "must specify a source preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy-global ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $dest_preset) { ast_utilities::error_output($program_name . ": preset", "must specify a destination preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset copy-global ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::copy_preset($ast_path, $program_name, $quiet, $src_preset, $dest_preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } elsif ($directive eq 'edit') { my $command_context = shift @ARGV; unless (defined $command_context) { ast_utilities::error_output($program_name . ": preset", "must specify a command"); print STDERR "Usage: ${bold_stderr}${program_name} preset edit ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $subcommand_classes{$command_context}) { ast_utilities::error_output($program_name . ": preset", "no such command ${bold_stderr}${command_context}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} preset edit ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } my $preset = shift @ARGV; unless (defined $preset) { ast_utilities::error_output($program_name . ": preset", "must specify a preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset edit ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::edit_preset($ast_path, $program_name, $quiet, $preset, $command_context, $current_namespace); } elsif ($directive eq 'edit-global') { my $preset = shift @ARGV; unless (defined $preset) { ast_utilities::error_output($program_name . ": preset", "must specify a preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset edit-global ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::edit_preset($ast_path, $program_name, $quiet, $preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } elsif ($directive eq 'list') { my $command_context = shift @ARGV; unless (defined $command_context) { return ast_preset_subsystem::all_presets($ast_path, $program_name, $quiet, $current_namespace); } unless (defined $subcommand_classes{$command_context}) { ast_utilities::error_output($program_name . ": preset", "no such command ${bold_stderr}${command_context}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} preset list [command [preset]]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } my $preset = shift @ARGV; if (defined $preset) { $success = ast_preset_subsystem::show_preset($ast_path, $program_name, $quiet, $preset, $command_context, $current_namespace); } else { $success = ast_preset_subsystem::all_presets_for_command($ast_path, $program_name, $quiet, $command_context, $current_namespace); } } elsif ($directive eq 'list-global') { my $preset = shift @ARGV; if (defined $preset) { $success = ast_preset_subsystem::show_preset($ast_path, $program_name, $quiet, $preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } else { $success = ast_preset_subsystem::all_presets_for_command($ast_path, $program_name, $quiet, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } } elsif ($directive eq 'namespace') { $success = atlas_cfgpreset_namespace($current_namespace); } elsif ($directive eq 'remove') { my $command_context = shift @ARGV; unless (defined $command_context) { ast_utilities::error_output($program_name . ": preset", "must specify a command"); print STDERR "Usage: ${bold_stderr}${program_name} preset remove [preset]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $subcommand_classes{$command_context}) { ast_utilities::error_output($program_name . ": preset", "no such command ${bold_stderr}${command_context}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} preset remove [preset]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } my $preset = shift @ARGV; if (defined $preset) { $success = ast_preset_subsystem::remove_preset($ast_path, $program_name, $quiet, $preset, $command_context, $current_namespace); } else { $success = ast_preset_subsystem::remove_all_presets_for_command($ast_path, $program_name, $quiet, $command_context, $current_namespace); } } elsif ($directive eq 'remove-global') { my $preset = shift @ARGV; if (defined $preset) { $success = ast_preset_subsystem::remove_preset($ast_path, $program_name, $quiet, $preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } else { $success = ast_preset_subsystem::remove_all_presets_for_command($ast_path, $program_name, $quiet, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace); } } elsif ($directive eq 'save') { my $command_context = shift @ARGV; unless (defined $command_context) { ast_utilities::error_output($program_name . ": preset", "must specify a command"); print STDERR "Usage: ${bold_stderr}${program_name} preset save ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $subcommand_classes{$command_context}) { ast_utilities::error_output($program_name . ": preset", "no such command ${bold_stderr}${command_context}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} preset save ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } my $preset = shift @ARGV; unless (defined $preset) { ast_utilities::error_output($program_name . ": preset", "must specify a preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset save ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (ast_preset_subsystem::preset_regex_ok($preset)) { ast_utilities::error_output($program_name . ": preset", "invalid preset name ${bold_stderr}${preset}${reset_stderr}"); print STDERR "Name must match regex: " . ast_preset_subsystem::preset_regex() . "\n"; print STDERR "Usage: ${bold_stderr}${program_name} preset save ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::save_preset($ast_path, $program_name, $quiet, $preset, $command_context, $current_namespace, \@ARGV); } elsif ($directive eq 'save-global') { my $preset = shift @ARGV; unless (defined $preset) { ast_utilities::error_output($program_name . ": preset", "must specify a preset"); print STDERR "Usage: ${bold_stderr}${program_name} preset save-global ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } unless (ast_preset_subsystem::preset_regex_ok($preset)) { ast_utilities::error_output($program_name . ": preset", "invalid preset name ${bold_stderr}${preset}${reset_stderr}"); print STDERR "Name must match regex: " . ast_preset_subsystem::preset_regex() . "\n"; print STDERR "Usage: ${bold_stderr}${program_name} preset save-global ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_preset_subsystem::save_preset($ast_path, $program_name, $quiet, $preset, $ast_preset_subsystem::GLOBAL_FOLDER, $current_namespace, \@ARGV); } else { ast_utilities::error_output($program_name . ": preset", "unrecognized ${bold_stderr}${program_name} preset${reset_stderr} directive '${bold_stderr}${directive}${reset_stderr}'"); print STDERR "Available directives: ${bold_stderr}copy[-global]${reset_stderr}, ${bold_stderr}edit[-global]${reset_stderr}, ${bold_stderr}list[-global]${reset_stderr}, ${bold_stderr}namespace${reset_stderr}, ${bold_stderr}remove[-global]${reset_stderr}, or ${bold_stderr}save[-global]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} preset --help${reset_stderr}\' for more information.\n"; return 0; } return 1; } sub execute_command_repo { my $help_flag; my $ref_flag = ''; GetOptions( 'help|h' => \$help_flag, 'ref=s' => \$ref_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "repo"); if (defined $help_flag) { show_submanpage_and_exit('repo'); } my $directive = shift @ARGV; my $success; unless (defined $directive) { ast_utilities::error_output($program_name . ": repo", "missing required directive"); print STDERR "Usage: ${bold_stderr}${program_name} repo ${reset_stderr}\n"; print STDERR "Available directives: ${bold_stderr}add${reset_stderr}, ${bold_stderr}add-gradle-exclude${reset_stderr}, ${bold_stderr}add-gradle-skip${reset_stderr}, ${bold_stderr}edit${reset_stderr}, ${bold_stderr}install${reset_stderr}, ${bold_stderr}list${reset_stderr}, or ${bold_stderr}remove${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } if ($directive eq 'add') { my $repo_name = shift @ARGV; my $repo_url = shift @ARGV; my $repo_ref = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo add [ref]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $repo_url) { ast_utilities::error_output($program_name . ": repo", "missing repo url"); print STDERR "Usage: ${bold_stderr}${program_name} repo add [ref]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } unless (ast_repo_subsystem::repo_regex_ok($repo_name)) { ast_utilities::error_output($program_name . ": repo", "invalid repo name ${bold_stderr}${repo_name}${reset_stderr}"); print STDERR "Usage: ${bold_stderr}${program_name} repo add [ref]${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $repo_ref) { $repo_ref = 'main'; } $success = ast_repo_subsystem::create_repo($ast_path, $program_name, $quiet, $repo_name, $repo_url, $repo_ref); } elsif ($directive eq 'add-gradle-exclude') { my $repo_name = shift @ARGV; my $exclude = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo add-gradle-exclude ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $exclude) { ast_utilities::error_output($program_name . ": repo", "missing exclude package"); print STDERR "Usage: ${bold_stderr}${program_name} repo add-gradle-exclude ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_repo_subsystem::add_exclude_variable($ast_path, $program_name, $quiet, $repo_name, $exclude); if ($success) { ast_repo_subsystem::print_repo_settings($ast_path, $program_name, $quiet, $repo_name); } } elsif ($directive eq 'add-gradle-skip') { my $repo_name = shift @ARGV; my $skip = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo add-gradle-skip ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } unless (defined $skip) { ast_utilities::error_output($program_name . ": repo", "missing skip task"); print STDERR "Usage: ${bold_stderr}${program_name} repo add-gradle-skip ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_repo_subsystem::add_skip_variable($ast_path, $program_name, $quiet, $repo_name, $skip); if ($success) { ast_repo_subsystem::print_repo_settings($ast_path, $program_name, $quiet, $repo_name); } } elsif ($directive eq 'edit') { my $repo_name = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo edit ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_repo_subsystem::edit_repo($ast_path, $program_name, $quiet, $repo_name); if ($success) { ast_repo_subsystem::print_repo_settings($ast_path, $program_name, $quiet, $repo_name); } } elsif ($directive eq 'install') { my $repo_name = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo install ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_repo_subsystem::install_repo($ast_path, $program_name, $quiet, $repo_name, $ref_flag); } elsif ($directive eq 'list') { my $repo_name = shift @ARGV; if (defined $repo_name) { $success = ast_repo_subsystem::print_repo_settings($ast_path, $program_name, $quiet, $repo_name); } else { $success = ast_repo_subsystem::list_repos($ast_path, $program_name, $quiet); } } elsif ($directive eq 'remove') { my $repo_name = shift @ARGV; unless (defined $repo_name) { ast_utilities::error_output($program_name . ": repo", "missing repo name"); print STDERR "Usage: ${bold_stderr}${program_name} repo remove ${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } $success = ast_repo_subsystem::remove_repo($ast_path, $program_name, $quiet, $repo_name); } else { ast_utilities::error_output($program_name . ": repo", "unrecognized directive '${bold_stderr}${directive}${reset_stderr}'"); print STDERR "Available directives: ${bold_stderr}add${reset_stderr}, ${bold_stderr}add-gradle-exclude${reset_stderr}, ${bold_stderr}add-gradle-skip${reset_stderr}, ${bold_stderr}edit${reset_stderr}, ${bold_stderr}install${reset_stderr}, ${bold_stderr}list${reset_stderr}, or ${bold_stderr}remove${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} repo --help${reset_stderr}\' for more information.\n"; return 0; } return 1; } sub execute_command_reset { my $help_flag; GetOptions( 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "reset"); if (defined $help_flag) { show_submanpage_and_exit('reset'); } my $directive = shift @ARGV; my $ran_at_least_one_directive = 0; unless (defined $directive) { ast_utilities::error_output($program_name . ": reset", "missing required directive"); print STDERR "Usage: ${bold_stderr}${program_name} reset ${reset_stderr}\n"; print STDERR "Available directives: ${bold_stderr}all${reset_stderr}, ${bold_stderr}index${reset_stderr}, ${bold_stderr}log${reset_stderr}, ${bold_stderr}modules${reset_stderr}, ${bold_stderr}presets${reset_stderr}, or ${bold_stderr}repos${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} reset --help${reset_stderr}\' for more information.\n"; return 0; } if ($directive eq 'index' || $directive eq 'all') { ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); $ran_at_least_one_directive = 1; } if ($directive eq 'log' || $directive eq 'all') { my $log_folder = File::Spec->catfile($ast_path, $ast_log_subsystem::LOG4J_FOLDER); my $logfile_path = File::Spec->catfile($log_folder, $ast_log_subsystem::LOG4J_FILE); ast_log_subsystem::replace_loglevel_in_file($logfile_path, 'ERROR'); ast_log_subsystem::replace_logstream_in_file($logfile_path, 'stderr'); unless ($quiet) { print "Reset log parameters.\n"; } $ran_at_least_one_directive = 1; } if ($directive eq 'modules' || $directive eq 'all') { my $modules_folder = File::Spec->catfile($ast_path, $ast_module_subsystem::MODULES_FOLDER); my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my $modules_length = keys %modules; if ($modules_length == 0) { ast_utilities::warn_output($program_name . ": reset", "found no modules to uninstall"); } else { foreach my $module (keys %modules) { ast_module_subsystem::perform_uninstall($module, $ast_path, $program_name, $quiet, 1); } ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); } $ran_at_least_one_directive = 1; } if ($directive eq 'presets' || $directive eq 'all') { my $presets_folder = File::Spec->catfile($ast_path, $ast_preset_subsystem::PRESETS_FOLDER); rmtree($presets_folder); unless ($quiet) { print "Cleared presets.\n"; } $ran_at_least_one_directive = 1; } if ($directive eq 'repos' || $directive eq 'all') { my $repos_folder = File::Spec->catfile($ast_path, $ast_repo_subsystem::REPOS_FOLDER); rmtree($repos_folder); unless ($quiet) { print "Cleared repos.\n"; } $ran_at_least_one_directive = 1; } if (!$ran_at_least_one_directive) { ast_utilities::error_output($program_name . ": reset", "unrecognized ${bold_stderr}${program_name} reset${reset_stderr} directive '${bold_stderr}${directive}${reset_stderr}'"); print STDERR "Available directives: ${bold_stderr}all${reset_stderr}, ${bold_stderr}index${reset_stderr}, ${bold_stderr}log${reset_stderr}, ${bold_stderr}modules${reset_stderr}, ${bold_stderr}presets${reset_stderr}, or ${bold_stderr}repos${reset_stderr}\n"; print STDERR "Try \'${bold_stderr}${program_name} reset --help${reset_stderr}\' for more information.\n"; return 0; } return 1; } sub execute_command_sync { my $help_flag; GetOptions( 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "sync"); if (defined $help_flag) { show_submanpage_and_exit('sync'); } my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my %modules_links = ast_module_subsystem::get_module_to_symlink_hash($ast_path); my @activated_modules = ast_module_subsystem::get_activated_modules(\%modules); if (scalar @activated_modules == 0) { ast_utilities::error_output($program_name . ": sync", 'found no activated module'); print STDERR "To see installed modules, try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} list${reset_stderr}'.\n"; print STDERR "Then run '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} activate ${reset_stderr}' to activate .\n"; return 0; } # If the currently active module is a broken symlink, warn the user if ($modules_links{$activated_modules[0]} == $ast_module_subsystem::BROKEN_SYMLINK) { ast_utilities::error_output($program_name . ": sync", 'current active module is a broken symlink'); print STDERR "To see link value, try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} list${reset_stderr}'.\n"; print STDERR "Fix the link, then run '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} sync${reset_stderr}' to resolve.\n"; return 1; } ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); ast_module_subsystem::generate_active_module_index($ast_path, $program_name, $quiet, 1); return 1; } sub execute_command_uninstall { my $allflag; my $forceflag = 0; my $help_flag; GetOptions( 'all|a' => \$allflag, 'force' => \$forceflag, 'help|h' => \$help_flag ) or ast_utilities::getopt_failure_and_exit($program_name, "uninstall"); if (defined $help_flag) { show_submanpage_and_exit('uninstall'); } if (defined $allflag) { my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); foreach my $module (keys %modules) { ast_module_subsystem::perform_uninstall($module, $ast_path, $program_name, $quiet, 1); } ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); return 1; } my $module_to_uninstall = shift @ARGV; unless (defined $module_to_uninstall) { ast_utilities::error_output($program_name . ": uninstall", "missing required argument"); ast_utilities::getopt_failure_and_exit($program_name, "uninstall"); } my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my @activated_modules = ast_module_subsystem::get_activated_modules(\%modules); my $activated_module = $activated_modules[0]; while (defined $module_to_uninstall) { my $success = ast_module_subsystem::perform_uninstall($module_to_uninstall, $ast_path, $program_name, $quiet, $forceflag); unless ($success) { ast_utilities::error_output($program_name . ": uninstall", "failed to uninstall ${bold_stderr}${module_to_uninstall}${reset_stderr}"); } elsif (defined $activated_module && $module_to_uninstall eq $activated_module) { ast_module_subsystem::remove_active_module_index($ast_path, $program_name, $quiet); } $module_to_uninstall = shift @ARGV; } return 1; } sub execute_command_update { my $help_flag; GetOptions( 'help|h' => \$help_flag, ) or ast_utilities::getopt_failure_and_exit($program_name, "update"); if (defined $help_flag) { show_submanpage_and_exit('update'); } chdir $ENV{ATLAS_SHELL_TOOLS_HOME} or die "$!"; my @command = (); push @command, "git"; push @command, "checkout"; push @command, "main"; my $success = system {$command[0]} @command; unless ($success == 0) { ast_utilities::error_output($program_name . ": update", "update operation failed"); return 1; } @command = (); push @command, "git"; push @command, "pull"; push @command, "origin"; push @command, "main"; $success = system {$command[0]} @command; unless ($success == 0) { ast_utilities::error_output($program_name . ": update", "update operation failed"); return 1; } return 1; } ########## END COMMAND SUBROUTINES ########## ########## BEGIN EXECUTION LOGIC ########## ast_utilities::verify_environment_or_exit(); $ast_path = ast_utilities::create_data_directory(); my $help_argument; my $allow_run_as_root; Getopt::Long::Configure(qw(no_ignore_case_always)); GetOptions( "no-pager" => \$skip_paging, "help|h:s" => \$help_argument, "version|V" => sub { print "$program_version\n"; exit 0; }, "quiet|q" => \$quiet, "allow-run-as-root" => \$allow_run_as_root, # This callback occurs the first time we see a non-option argument. # In our case, this will be the subcommand. "<>" => sub { my ($arg) = @_; if ($arg =~ m{^-}) { unless ($arg eq '-') { die "FATAL error: unhandled global option $arg"; } } # add the subcommand to the front of ARGV unshift @ARGV, $arg; die "!FINISH"; } ) or ast_utilities::getopt_failure_and_exit($program_name); if (geteuid() == 0) { unless (defined $allow_run_as_root) { print STDERR "For security reasons, you are highly discouraged from running atlas-shell-tools\n"; print STDERR "as the root user. Atlas-shell-tools cannot guarantee that modules installed from\n"; print STDERR "external repositories are safe to run with root privileges.\n\n"; print STDERR "To disregard this warning and run as root anyway, please use the option:\n"; print STDERR "--allow-run-as-root\n\n"; exit 1; } } # Handle the cases where the user supplied a --help flag # 1) --help -> print the default help menu and exit # 2) --help=TOPIC -> print the TOPIC help menu and exit if (defined $help_argument) { show_submanpage_and_exit($help_argument); } # All global options have been processed, so shift the subcommand off of ARGV my $subcommand = shift @ARGV; unless (defined $subcommand) { ast_utilities::error_output($program_name, "missing required command or option"); print STDERR "Try '${bold_stderr}${program_name} --help${reset_stderr}' for more information.\n"; exit 1; } my $success = 0; if ($subcommand eq "activate") { $success = execute_command_activate(); } elsif ($subcommand eq "deactivate") { $success = execute_command_deactivate(); } elsif ($subcommand eq "install") { $success = execute_command_install(); } elsif ($subcommand eq "list") { $success = execute_command_list(); } elsif ($subcommand eq "log") { $success = execute_command_log(); } elsif ($subcommand eq "preset") { $success = execute_command_preset(); } elsif ($subcommand eq "repo") { $success = execute_command_repo(); } elsif ($subcommand eq "reset") { $success = execute_command_reset(); } elsif ($subcommand eq "sync") { $success = execute_command_sync(); } elsif ($subcommand eq "uninstall") { $success = execute_command_uninstall(); } elsif ($subcommand eq "update") { $success = execute_command_update(); } # These subcommands are "hidden", they are used by the completion scripts # to generate autocomplete suggestions. elsif ($subcommand eq "__completion_atlas__") { $success = ast_completions::completion_atlas($ast_path, 0, \@ARGV); } elsif ($subcommand eq "__completion_atlascfg__") { $success = ast_completions::completion_atlascfg($ast_path, 0, \@ARGV); } elsif ($subcommand eq "__completion_atlas_zsh__") { $success = ast_completions::completion_atlas($ast_path, 1, \@ARGV); } elsif ($subcommand eq "__completion_atlascfg_zsh__") { $success = ast_completions::completion_atlascfg($ast_path, 1, \@ARGV); } else { ast_utilities::error_output($program_name, "no such command ${bold_stderr}${subcommand}${reset_stderr}"); print STDERR "Try '${bold_stderr}${program_name} --help${reset_stderr}' for more information.\n"; exit 127; } if ($success) { exit 0; } else { exit 1; } ########## END EXECUTION LOGIC ########## ================================================ FILE: atlas-shell-tools/scripts/common/ast_completions.pm ================================================ package ast_completions; use warnings; use strict; use Exporter qw(import); use ast_module_subsystem; use ast_preset_subsystem; use ast_utilities; use ast_tty; # Export symbols: variables and subroutines our @EXPORT = qw( completion_match_prefix completion_atlas completion_atlascfg ); my $FILE_COMPLETE_SENTINEL = "__atlas-shell-tools_sentinel_complete_filenames__"; my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); sub completion_match_prefix { my $prefix_to_complete = shift; my $possible_words_ref = shift; my @possible_words = @{$possible_words_ref}; if ($prefix_to_complete eq "") { return @possible_words; } my @matched_words = (); foreach my $word (@possible_words) { if (ast_utilities::string_starts_with($word, $prefix_to_complete)) { push @matched_words, $word; } } return @matched_words; } # TODO implement a function that takes in the current argv and returns the name # of the current subcommand (stripping away the global options) # TODO implement a function that takes in the current argv and returns an array # containing just the subcommand and its arguments # The above ideas would simplify all the 'rargv_mX' variables sub completion_atlas { my $ast_path = shift; my $zsh_mode = shift; my $argv_ref = shift; my @argv = @{$argv_ref}; my %argv_map = map { $_ => 1 } @argv; # Shift COMP_CWORD off the front of ARGV my $comp_cword = shift @argv; # Shift "atlas" off the front of ARGV shift @argv; if ($zsh_mode && $comp_cword == (scalar @argv + 1)) { push @argv, ""; } my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); my @commands = keys %subcommand_classes; # In the completion code, we use the following conventions to name variables # containing ARGV elements. Assume ARGV looks like the following: # # ARGV of length N: # ARGV[0] ... ARGV[N - K] ... ARGV[N - 3], ARGV[N - 2], ARGV[N - 1] # ^ rargv_m(k-1) ^ rargv_m2 ^ rargv_m1 ^ rargv # # Essentially, "rargv" means the "rightmost" ARGV element. Then "rargv_m1" # means the "rightmost" ARGV element minus 1, and so on. Since perl plays fast # and loose with array indexing, we can index into elements that may or may # not actually exist, and then check if they are defined before we actually # use them. The '-1' indexing syntax just indexes from the end of the array. # my $argv_len = scalar @argv; my $rargv = $argv[-1]; my $rargv_m1 = $argv[-2]; my $rargv_m2 = $argv[-3]; my $rargv_m3 = $argv[-4]; # Autocomplete the '--preset' and '--save-preset' flags, since they are probably the most used flags. # Since they are global options, we only complete them if we have not yet seen a command in ARGV. my $saw_command = 0; if (ast_utilities::string_starts_with($rargv, '-')) { foreach my $command (@commands) { if (exists($argv_map{$command})) { $saw_command = 1; } } unless ($saw_command) { my @flags = qw(--preset --save-preset --save-global-preset); my @completion_matches = completion_match_prefix($rargv, \@flags); print join("\n", @completion_matches) . "\n"; return 1; } } # Handle special case where user is applying a preset with "--preset" if (defined $rargv_m1) { if ($rargv_m1 eq '-p' || ast_utilities::string_starts_with($rargv_m1, '--p')) { my @presets = ast_preset_subsystem::get_all_presets_in_current_namespace($ast_path); my @completion_matches = completion_match_prefix($rargv, \@presets); print join("\n", @completion_matches) . "\n"; return 1; } } # If we see a command anywhere in ARGV, stop special completions and signal # the completion wrapper script to use its filename defaults. foreach my $command (@commands) { if (exists($argv_map{$command})) { print $FILE_COMPLETE_SENTINEL; return 1; } } # Default to completing available command names my @completion_matches = completion_match_prefix($rargv, \@commands); print join("\n", @completion_matches) . "\n"; return 1; } sub completion_atlascfg { my $ast_path = shift; my $zsh_mode = shift; my $argv_ref = shift; my @argv = @{$argv_ref}; # Shift COMP_CWORD off the front of ARGV my $comp_cword = shift @argv; # Shift "atlas-config" off the front of ARGV shift @argv; my %subcommand_classes = ast_module_subsystem::get_subcommand_to_class_hash($ast_path); # Shift global options off the front of ARGV foreach my $element (@argv) { if (ast_utilities::string_starts_with($element, '-')) { shift @argv; # We need to decrement $comp_cword, since it will contain an extra # count for each of the global options. $comp_cword = $comp_cword - 1; } } if ($zsh_mode && $comp_cword == (scalar @argv + 1)) { push @argv, ""; } # If no more ARGV is left, exit if (scalar @argv == 0) { return 1; } my @commands = (); my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); # In the completion code, we use the following conventions to name variables # containing ARGV elements. Assume ARGV looks like the following: # # ARGV of length N: # ARGV[0] ... ARGV[N - K] ... ARGV[N - 3], ARGV[N - 2], ARGV[N - 1] # ^ rargv_m(k-1) ^ rargv_m2 ^ rargv_m1 ^ rargv # # Essentially, "rargv" means the "rightmost" ARGV element. Then "rargv_m1" # means the "rightmost" ARGV element minus 1, and so on. Since perl plays fast # and loose with array indexing, we can index into elements that may or may # not actually exist, and then check if they are defined before we actually # use them. The '-1' indexing syntax just indexes from the end of the array. # my $rargv = $argv[-1]; my $rargv_m1 = $argv[-2]; my $rargv_m2 = $argv[-3]; my $rargv_m3 = $argv[-4]; unless (defined $argv[1]) { @commands = qw(activate deactivate install list log preset repo reset sync uninstall update); } # # This really long if-else handles all the possible command-lines # # 'atlas-config install' command will complete file names if ((defined $argv[0] && $argv[0] eq 'install') && (defined $rargv_m1 && $rargv_m1 eq 'install')) { print $FILE_COMPLETE_SENTINEL; return 1; } # 'atlas-config activate' command will complete deactivated modules elsif ((defined $argv[0] && $argv[0]) eq 'activate' && (defined $rargv_m1 && $rargv_m1 eq 'activate')) { @commands = ast_module_subsystem::get_deactivated_modules(\%modules); } # 'atlas-config deactivate' command will complete activated modules elsif ((defined $argv[0] && $argv[0] eq 'deactivate') && (defined $rargv_m1 && $rargv_m1 eq 'deactivate')) { @commands = ast_module_subsystem::get_activated_modules(\%modules); } # 'atlas-config uninstall' command will complete all modules as many times as desired elsif ((defined $argv[0] && $argv[0] eq 'uninstall')) { @commands = keys %modules; } # 'atlas-config repo' command will complete repo subcommands elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'repo')) { @commands = qw(add list remove edit install add-gradle-skip add-gradle-exclude); } # 'atlas-config repo remove' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'remove')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config repo edit' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'edit')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config repo install' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'install')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config repo list' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'list')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config repo add-gradle-skip' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'add-gradle-skip')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config repo add-gradle-exclude' command will complete repos elsif ((defined $argv[0] && $argv[0] eq 'repo') && (defined $rargv_m2 && $rargv_m2 eq 'repo') && (defined $rargv_m1 && $rargv_m1 eq 'add-gradle-exclude')) { @commands = ast_repo_subsystem::get_all_repos($ast_path); } # 'atlas-config preset' command will complete 'preset' subcommands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'preset')) { @commands = qw(save save-global edit edit-global remove remove-global list list-global namespace copy copy-global); } # 'atlas-config preset save' command will complete atlas shell tools commands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'save')) { @commands = keys %subcommand_classes; } # 'atlas-config preset edit' command will complete atlas shell tools commands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'edit')) { @commands = keys %subcommand_classes; } # 'atlas-config preset edit ' command will complete any presets for elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'edit') && (defined $rargv_m1)) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $rargv_m1); } # 'atlas-config preset edit-global' command will complete any global presets elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'edit-global')) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $ast_preset_subsystem::GLOBAL_FOLDER); } # 'atlas-config preset remove' command will complete atlas shell tools commands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'remove')) { @commands = keys %subcommand_classes; } # 'atlas-config preset remove ' command will complete any presets for elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'remove') && (defined $rargv_m1)) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $rargv_m1); } # 'atlas-config preset remove-global' command will complete any global presets elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'remove-global')) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $ast_preset_subsystem::GLOBAL_FOLDER); } # 'atlas-config preset list' command will complete atlas shell tools commands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'list')) { @commands = keys %subcommand_classes; } # 'atlas-config preset list ' command will complete any presets for elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'list') && (defined $rargv_m1)) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $rargv_m1); } # 'atlas-config preset list-global' will complete any global presets elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'list-global')) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $ast_preset_subsystem::GLOBAL_FOLDER); } # 'atlas-config preset namespace' command will complete 'namespace' subcommands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'namespace')) { @commands = qw(create remove list use); } # 'atlas-config preset namespace remove' command will complete namespaces elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'namespace') && (defined $rargv_m1 && $rargv_m1 eq 'remove')) { @commands = ast_preset_subsystem::get_namespaces_array($ast_path); } # 'atlas-config preset namespace list' command will complete namespaces elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'namespace') && (defined $rargv_m1 && $rargv_m1 eq 'list')) { @commands = ast_preset_subsystem::get_namespaces_array($ast_path); } # 'atlas-config preset namespace use' command will complete namespaces elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'namespace') && (defined $rargv_m1 && $rargv_m1 eq 'use')) { @commands = ast_preset_subsystem::get_namespaces_array($ast_path); } # 'atlas-config preset copy' command will complete atlas shell tools commands elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'copy')) { @commands = keys %subcommand_classes; } # 'atlas-config preset copy ' command will complete any presets for elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m3 && $rargv_m3 eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'copy') && (defined $rargv_m1)) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $rargv_m1); } # 'atlas-config preset copy-global' command will complete any global presets elsif ((defined $argv[0] && $argv[0] eq 'preset') && (defined $rargv_m2 && $rargv_m2 eq 'preset') && (defined $rargv_m1 && $rargv_m1 eq 'copy-global')) { @commands = ast_preset_subsystem::get_all_presets_for_command($ast_path, $ast_preset_subsystem::GLOBAL_FOLDER); } # 'atlas-config log' command will complete log subcommands elsif ((defined $argv[0] && $argv[0] eq 'log') && (defined $rargv_m1 && $rargv_m1 eq 'log')) { @commands = qw(reset set-level set-stream show); } # 'atlas-config log set-level' command will complete log levels elsif ((defined $argv[0] && $argv[0] eq 'log') && (defined $rargv_m2 && $rargv_m2 eq 'log') && (defined $rargv_m1 && $rargv_m1 eq 'set-level')) { @commands = qw(ALL TRACE DEBUG INFO WARN ERROR FATAL OFF); } # 'atlas-config log set-stream' command will complete log streams elsif ((defined $argv[0] && $argv[0] eq 'log') && (defined $rargv_m2 && $rargv_m2 eq 'log') && (defined $rargv_m1 && $rargv_m1 eq 'set-stream')) { @commands = qw(stdout stderr); } # 'atlas-config reset' command will complete reset subcommands elsif ((defined $argv[0] && $argv[0] eq 'reset') && (defined $rargv_m1 && $rargv_m1 eq 'reset')) { @commands = qw(all index log modules presets repos); } # Generate completion matches based on prefix of current word my @completion_matches = completion_match_prefix($rargv, \@commands); print join("\n", @completion_matches) . "\n"; return 1; } sub debug_dump_string { my $string = shift; my $file = shift; open my $handle, '>>', $file; print $handle $string; close $handle; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_log_subsystem.pm ================================================ package ast_log_subsystem; use warnings; use strict; use Exporter qw(import); use File::Spec; use ast_tty; use ast_utilities; # Export symbols: variables and subroutines our @EXPORT = qw( LOG4J_FOLDER LOG4J_FILE LOG4J_FILE_PATH DEFAULT_LOG4J_CONTENTS reset_log4j read_loglevel_from_file read_logstream_from_file replace_loglevel_in_file replace_logstream_in_file ); my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); our $LOG4J_FOLDER = 'log4j'; our $LOG4J_FILE = 'log4j.properties'; our $LOG4J_FILE_PATH = File::Spec->catfile($LOG4J_FOLDER, $LOG4J_FILE); # The default setting for the log4j file. my $DEFAULT_LOG4J_CONTENTS = "log4j.rootLogger=ERROR, stderr # DO NOT REMOVE/MODIFY THE ABOVE LINE OR ANY OF THIS FILE # Use 'atlas-config log' subcommand to manage the log configuration # If this file is corrupted, use 'atlas-config log --reset' to fix. # Direct log messages to stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.Target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n "; # Reset the log4j file to default. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: none sub reset_log4j { my $ast_path = shift; my $log4j_file = File::Spec->catfile($ast_path, $LOG4J_FOLDER, $LOG4J_FILE); open my $file_handle, '>', "$log4j_file"; print $file_handle $DEFAULT_LOG4J_CONTENTS; close $file_handle; } # Read the log level from a given logfile. The log file should be a log4j file # where the first line looks like: # log4j.rootLogger=LEVEL, STREAM #Params: # $logfile_path: the path to the log4j file # Return: the string level, eg. DEBUG, WARN, ERROR, etc. sub read_loglevel_from_file { my $logfile_path = shift; # Grab the first line and chomp the trailing newline. open my $logfile, '<', $logfile_path or die "could not read logfile from path $logfile_path"; my $firstline = <$logfile>; chomp $firstline; close $logfile; # firstline should now look like # log4j.rootLogger=LEVEL, STREAM my @split1 = split('=', $firstline); my @split2 = split(',', $split1[1]); my $level = $split2[0]; # remove leading and trailing whitespace $level =~ s/^\s+|\s+$//g; return $level; } # Read the log stream from a given logfile. The log file should be a log4j file # where the first line looks like: # log4j.rootLogger=LEVEL, STREAM #Params: # $logfile_path: the path to the log4j file # Return: the string stream, eg. stdout or stderr sub read_logstream_from_file { my $logfile_path = shift; # Grab the first line and chomp the trailing newline. open my $logfile, '<', $logfile_path or die "could not read logfile from path $logfile_path"; my $firstline = <$logfile>; chomp $firstline; close $logfile; # firstline should now look like # log4j.rootLogger=LEVEL, STREAM my @split1 = split('=', $firstline); my @split2 = split(',', $split1[1]); my $stream = $split2[1]; # remove leading and trailing whitespace $stream =~ s/^\s+|\s+$//g; return $stream; } # Replace the log level in a given logfile. The log file should be a log4j file # where the first line looks like: # log4j.rootLogger=LEVEL, stdout # Note that this function will not validate that the new level is actually a # valid log4j level. This is left up to the caller. #Params: # $logfile_path: the path to the log4j file # $new_level: the new level # Return: none sub replace_loglevel_in_file { my $logfile_path = shift; my $new_level = shift; # read the old level that we are replacing my $old_level = read_loglevel_from_file($logfile_path); # create a backup of the old log4j file rename $logfile_path, $logfile_path . '.bak'; open my $logfileIN, '<', $logfile_path . '.bak'; open my $logfileOUT, '>', $logfile_path; # replace the level my $firstline = <$logfileIN>; $firstline =~ s/${old_level}/${new_level}/; print $logfileOUT $firstline; # write the rest of the log4j file while (<$logfileIN>) { print $logfileOUT $_; } close $logfileIN; close $logfileOUT; } # Replace the log stream in a given logfile. The log file should be a log4j file # where the first line looks like: # log4j.rootLogger=LEVEL, STREAM # This function will die if the provided stream is not 'stdout' or 'stderr'. # This is necessary because we must compute stderr -> System.err and stdout -> # System.out to properly update the log4j file. #Params: # $logfile_path: the path to the log4j file # $new_stream: the new stream # Return: none sub replace_logstream_in_file { my $logfile_path = shift; my $new_stream = shift; # read the old stream that we are replacing my $old_stream = read_logstream_from_file($logfile_path); my $old_system; my $new_system; if ($new_stream eq 'stdout') { $old_system = 'System\.err'; $new_system = 'System.out'; } elsif ($new_stream eq 'stderr') { $old_system = 'System\.out'; $new_system = 'System.err'; } else { die "Invalid stream setting $new_stream, must be stdout or stderr"; } # create a backup of the old log4j file rename $logfile_path, $logfile_path . '.bak'; open my $logfileIN, '<', $logfile_path . '.bak'; open my $logfileOUT, '>', $logfile_path; while (<$logfileIN>) { $_ =~ s/${old_stream}/${new_stream}/g; $_ =~ s/${old_system}/${new_system}/g; print $logfileOUT $_; } close $logfileIN; close $logfileOUT; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_module_subsystem.pm ================================================ package ast_module_subsystem; use warnings; use strict; use Exporter qw(import); use File::Basename qw(basename); use File::Spec; use ast_utilities; use ast_tty; use ast_log_subsystem; # Export symbols: variables and subroutines our @EXPORT = qw( MODULE_SUFFIX DEACTIVATED_SUFFIX DEACTIVATED_MODULE_SUFFIX MODULES_FOLDER ACTIVE_INDEX_FILE ACTIVE_INDEX_PATH ACTIVATED DEACTIVATED GOOD_SYMLINK BROKEN_SYMLINK REAL_FILE SOURCE_KEY URI_KEY DATE_TIME_KEY REPO_NAME_KEY REPO_REF_KEY REPO_COMMIT_KEY METADATA_SEPARATOR get_subcommand_to_class_hash get_subcommand_to_description_hash get_module_to_status_hash get_module_to_symlink_hash get_module_to_target_hash get_module_to_metadata_hash get_activated_modules get_deactivated_modules perform_uninstall perform_activate perform_deactivate generate_active_module_index remove_active_module_index ); our $METADATA_SUFFIX = '.metadata'; our $MODULE_SUFFIX = '.jar'; our $DEACTIVATED_SUFFIX = '.deactivated'; our $DEACTIVATED_MODULE_SUFFIX = $MODULE_SUFFIX . $DEACTIVATED_SUFFIX; our $MODULES_FOLDER = 'modules'; our $ACTIVE_INDEX_FILE = '.active_module_index'; our $ACTIVE_INDEX_PATH = File::Spec->catfile($MODULES_FOLDER, $ACTIVE_INDEX_FILE); our $ACTIVATED = 1; our $DEACTIVATED = 0; our $GOOD_SYMLINK = 1; our $BROKEN_SYMLINK = -1; our $REAL_FILE = 0; # Module metadata keys/data our $SOURCE_KEY = "source-type"; our $URI_KEY = "URI"; our $DATE_TIME_KEY = "date-time"; our $REPO_NAME_KEY = "repo-name"; our $REPO_REF_KEY = "repo-ref"; our $REPO_COMMIT_KEY = "repo-commit"; our $METADATA_SEPARATOR = ":"; # Use ASCII record separator as delimiter. # This is also defined in Atlas class ActiveModuleIndexWriter. my $INDEX_DELIMITER = "\x1E"; # The Java class that creates the active_module_index my $INDEX_WRITER_CLASS = "org.openstreetmap.atlas.utilities.command.ActiveModuleIndexWriter"; my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); # Get a hash that maps subcommand names to their respective classes. The hash is # computed from the current active module index. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all subcommands to their classes. sub get_subcommand_to_class_hash { my $ast_path = shift; my $index_path = File::Spec->catfile($ast_path, $ACTIVE_INDEX_PATH); open my $index_fileIN, '<', $index_path or return(); my %subcommand_to_class = (); while (<$index_fileIN>) { my $line = $_; chomp $line; my @line_elements = split($INDEX_DELIMITER, $line); $subcommand_to_class{$line_elements[0]} = $line_elements[1]; } return %subcommand_to_class; } # Get a hash that maps subcommand names to their respective descriptions. The # hash is computed from the current active module index. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all subcommands to their descriptions. sub get_subcommand_to_description_hash { my $ast_path = shift; my $index_path = File::Spec->catfile($ast_path, $ACTIVE_INDEX_PATH); open my $index_fileIN, '<', $index_path or return(); my %subcommand_to_description = (); while (<$index_fileIN>) { my $line = $_; chomp $line; my @line_elements = split($INDEX_DELIMITER, $line); $subcommand_to_description{$line_elements[0]} = $line_elements[2]; } return %subcommand_to_description; } # Get a hash that maps all present module names to their activation status. # Activated modules are mapped to ACTIVATED, while deactivated modules are mapped # to DEACTIVATED. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all modules to their activation status. sub get_module_to_status_hash { my $ast_path = shift; my @find_command = ( "find", "${ast_path}/${MODULES_FOLDER}", "-maxdepth", "1", "(", "-name", "*$MODULE_SUFFIX", "-o", "-name", "*$DEACTIVATED_SUFFIX", ")", "-print0" ); my %modules = (); open FIND, "-|", @find_command; # TODO 'local' modifier makes sense here? confirm this, 'my' may make more sense # see https://www.perlmonks.org/?node_id=94007 local $/ = "\0"; while () { # FIND command is printing full paths, we just want the basename. # Also, we must chomp to remove the terminating null byte left over from # the '-print0' flag given to 'find'. my $module = $_; chomp $module; $module = basename($module); my $module_activated; # TODO: figure out how to interpolate $DEACTIVATED_SUFFIX into this regex if ($module =~ /.*\.deactivated$/) { $module_activated = $DEACTIVATED; } else { $module_activated = $ACTIVATED; } $module =~ s{$MODULE_SUFFIX}{}; $module =~ s{$DEACTIVATED_SUFFIX}{}; $modules{$module} = $module_activated; } close FIND; return %modules; } # Get a hash that maps all present module names to their symlink state. # Symlinked modules are mapped to GOOD_SYMLINK, regular modules are mapped to # REAL_FILE. Broken symlink modules are mapped to BROKEN_SYMLINK. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all modules to their symlink state. sub get_module_to_symlink_hash { my $ast_path = shift; my @find_command = ( "find", "${ast_path}/${MODULES_FOLDER}", "-maxdepth", "1", "(", "-name", "*$MODULE_SUFFIX", "-o", "-name", "*$DEACTIVATED_SUFFIX", ")", "-print0" ); my %modules = (); open FIND, "-|", @find_command; local $/ = "\0"; while () { # FIND command is printing full paths, we just want the basename. # Also, we must chomp to remove the terminating null byte left over from # the '-print0' flag given to 'find'. my $module = $_; chomp $module; $module = basename($module); my $module_islink; if (-l $_) { if (lstat $_ and not stat $_) { $module_islink = $BROKEN_SYMLINK; } else { $module_islink = $GOOD_SYMLINK; } } else { $module_islink = $REAL_FILE; } $module =~ s{$MODULE_SUFFIX}{}; $module =~ s{$DEACTIVATED_SUFFIX}{}; $modules{$module} = $module_islink; } close FIND; return %modules; } # Get a hash that maps all present module names to their symlink target. # If a module is not a symlink, it maps to an empty string. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all modules to their symlink target. sub get_module_to_target_hash { my $ast_path = shift; my @find_command = ( "find", "${ast_path}/${MODULES_FOLDER}", "-maxdepth", "1", "(", "-name", "*$MODULE_SUFFIX", "-o", "-name", "*$DEACTIVATED_SUFFIX", ")", "-print0" ); my %modules = (); open FIND, "-|", @find_command; local $/ = "\0"; while () { # FIND command is printing full paths, we just want the basename. # Also, we must chomp to remove the terminating null byte left over from # the '-print0' flag given to 'find'. my $module = $_; chomp $module; $module = basename($module); my $module_target; if (-l $_) { $module_target = readlink $_; } else { $module_target = ''; } $module =~ s{$MODULE_SUFFIX}{}; $module =~ s{$DEACTIVATED_SUFFIX}{}; $modules{$module} = $module_target; } close FIND; return %modules; } # Get a hash that maps all present module names to their metadata hashes, if present. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: a hash of all modules to their metadata hashes. sub get_module_to_metadata_hash { my $ast_path = shift; my @find_command = ( "find", "${ast_path}/${MODULES_FOLDER}", "-maxdepth", "1", "-name", "*$METADATA_SUFFIX", "-print0" ); my %modules = (); open FIND, "-|", @find_command; local $/ = "\0"; while () { # FIND command is printing full paths, we just want the basename. # Also, we must chomp to remove the terminating null byte left over from # the '-print0' flag given to 'find'. my $module = $_; chomp $module; my $module_basename = basename($module); $module_basename =~ s{$METADATA_SUFFIX}{}; my %metadata; open my $file_handle, '<', $module or die "Could not open module metadata file $module $!"; # we have to change the separator to newline in order to parse the file properly $/ = "\n"; while (my $line = <$file_handle>) { chomp $line; # trim excess whitespace from left and right $line =~ s/^\s+|\s+$//g; if ($line eq '' || substr($line, 0, 1) eq '#') { next; } my @line_split = split $METADATA_SEPARATOR, $line, 2; unless (defined $line_split[0]) { next; } # trim excess whitespace from left and right $line_split[0] =~ s/^\s+|\s+$//g; if (defined $line_split[1] && $line_split[1] !~ /^\s*$/) { # trim excess whitespace from left and right $line_split[1] =~ s/^\s+|\s+$//g; $metadata{$line_split[0]} = $line_split[1]; } } close $file_handle; $modules{$module_basename} = \%metadata; # change the separator back to \0 for FIND $/ = "\0"; } close FIND; return %modules; } # Get an array of all active modules, computed from the modules hash returned by # the 'get_module_to_status_hash' subroutine. # Params: # $modules_ref: a reference to the module hash returned by 'get_module_to_status_hash' # Return: an array of all active modules sub get_activated_modules { my $modules_ref = shift; my %modules = %{$modules_ref}; my @activated_modules = (); foreach my $module (keys %modules) { if ($modules{$module} == $ACTIVATED) { push @activated_modules, $module; } } return @activated_modules; } # Get an array of all deactived modules, computed from the modules hash returned by # the 'get_module_to_status_hash' subroutine. # Params: # $modules_ref: a reference to the module hash returned by 'get_module_to_status_hash' # Return: an array of all deactived modules sub get_deactivated_modules { my $modules_ref = shift; my %modules = %{$modules_ref}; my @deactivated_modules = (); foreach my $module (keys %modules) { if ($modules{$module} == $DEACTIVATED) { push @deactivated_modules, $module; } } return @deactivated_modules; } # Install a module using some given parameters. # Params: # $module_to_install: the path to the module to install # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the running program # $alternate_name: an alternate name for the module, empty to use path basename # $syminstall: install using a symlink instead of a copy # $skip_install: skip installation if module exists # $force_install: force overwrite if module exists # $install_deactivated: install module but do not activate # $metadata_ref: a reference to a metadata hash # $quiet: suppress non-essential output # # Return: 1 on success, 0 on failure sub perform_install { my $module_to_install = shift; my $ast_path = shift; my $program_name = shift; my $alternate_name = shift; my $syminstall = shift; my $skip_install = shift; my $force_install = shift; my $install_deactivated = shift; my $metadata_ref = shift; my $quiet = shift; unless (-f $module_to_install || -l $module_to_install) { ast_utilities::error_output($program_name, "no such file ${bold_stderr}${module_to_install}${reset_stderr}"); return 0; } my $modules_folder = File::Spec->catfile($ast_path, $MODULES_FOLDER); # Create the module name, respecting the alternate name if provided. my $module_basename; if (!$alternate_name eq "") { $module_basename = $alternate_name . $MODULE_SUFFIX; } else { $module_basename = basename($module_to_install); } # TODO: figure out how to interpolate $MODULE_SUFFIX into this regex unless ($module_basename =~ /.*\.jar$/) { ast_utilities::error_output($program_name, "module must end with '.jar' extension"); return 0; } $module_basename =~ s{$MODULE_SUFFIX}{}; # Handle the case where the module is already installed my %modules = get_module_to_status_hash($ast_path); if (defined $modules{$module_basename}) { ast_utilities::warn_output($program_name, "module ${bold_stderr}${module_basename}${reset_stderr} is already installed"); if ($skip_install) { print STDERR "Skipping installation.\n"; return 0; } unless ($force_install) { my $overwrite = ast_utilities::prompt_yn("Overwrite?"); unless ($overwrite) { print STDERR "Skipping installation.\n"; return 0; } } ast_utilities::warn_output($program_name, "overwriting ${bold_stderr}${module_basename}${reset_stderr}"); } # Construct the new module paths. # We create a path for both an activated and deactivated version. my $module_new_path = File::Spec->catfile($modules_folder, $module_basename . $MODULE_SUFFIX); my $module_new_path_deactivated = File::Spec->catfile($modules_folder, $module_basename . $DEACTIVATED_MODULE_SUFFIX); # If we made it here we are go to overwrite, so clean up any matching existing modules. unlink $module_new_path; unlink $module_new_path_deactivated; my $exitcode; if ($syminstall) { my $module_to_install_abs = Cwd::realpath($module_to_install); my $module_to_install_rel = File::Spec->abs2rel($module_to_install_abs, $modules_folder); if ($install_deactivated) { symlink($module_to_install_rel, $module_new_path_deactivated); } else { symlink($module_to_install_rel, $module_new_path); } $exitcode = $? >> 8; } else { my @command = (); push @command, "cp"; push @command, "$module_to_install"; if ($install_deactivated) { push @command, "$module_new_path_deactivated"; system {$command[0]} @command; } else { push @command, "$module_new_path"; system {$command[0]} @command; } $exitcode = $? >> 8; } if ($exitcode) { print STDERR "${red_stderr}${bold_stderr}Installation of module ${module_basename} failed.${reset_stderr} Operation exited with $exitcode.\n"; return 0; } else { unless ($install_deactivated) { my %modules = get_module_to_status_hash($ast_path); foreach my $module (keys %modules) { if ($modules{$module} == $ACTIVATED && $module ne $module_basename) { my $success = perform_deactivate($module, $ast_path, $program_name, $quiet); unless ($success) { ast_utilities::warn_output($program_name, "installation may not function properly"); } } } remove_active_module_index($ast_path, $program_name, $quiet); my $success = generate_active_module_index($ast_path, $program_name, $quiet, 0); unless ($success) { ast_utilities::warn_output($program_name, "partial installation may not function properly"); } } my $success = write_module_metadata($modules_folder, $module_basename, $metadata_ref, $program_name); unless ($success) { ast_utilities::warn_output($program_name, "metadata not written"); } unless ($quiet) { print "Module ${green_stdout}${bold_stdout}${module_basename}${reset_stdout} installed.\n"; } } return 1; } # Uninstall a module with a given name. # Params: # $module_to_uninstall: the name of the module to uninstall # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the running program # $quiet: suppress non-essential output # $force: force uninstall on activated module # Return: 1 on success, 0 on failure sub perform_uninstall { my $module_to_uninstall = shift; my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $force = shift; my %modules = get_module_to_status_hash($ast_path); unless (exists $modules{$module_to_uninstall}) { error_output($program_name, "no such module ${bold_stderr}${module_to_uninstall}${reset_stderr}"); return 0; } if ($modules{$module_to_uninstall} eq $ACTIVATED && $force != 1) { warn_output($program_name, "skipping activated module ${bold_stderr}${module_to_uninstall}${reset_stderr}"); return 0; } # try to remove the module my $modules_folder = File::Spec->catfile($ast_path, $MODULES_FOLDER); my $module_remove_path = File::Spec->catfile($modules_folder, $module_to_uninstall . $MODULE_SUFFIX); my $module_remove_path_deactivated = File::Spec->catfile($modules_folder, $module_to_uninstall . $DEACTIVATED_MODULE_SUFFIX); my $module_remove_path_metadata = File::Spec->catfile($modules_folder, $module_to_uninstall . $METADATA_SUFFIX); unlink $module_remove_path; unlink $module_remove_path_deactivated; unlink $module_remove_path_metadata; unless ($quiet) { print "Module ${bold_stdout}${module_to_uninstall}${reset_stdout} uninstalled.\n"; } return 1; } # Activate a module with a given name. # Params: # $module_to_activate: the name of the module to activate # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the running program # $quiet: suppress non-essential messages # Return: 1 on success, 0 on failure sub perform_activate { my $module_to_activate = shift; my $ast_path = shift; my $program_name = shift; my $quiet = shift; my %modules = get_module_to_status_hash($ast_path); unless (exists $modules{$module_to_activate}) { ast_utilities::error_output($program_name, "no such module ${bold_stderr}${module_to_activate}${reset_stderr}"); return 0; } my $status = $modules{$module_to_activate}; if ($status == $ACTIVATED) { ast_utilities::warn_output($program_name, "module ${bold_stderr}${module_to_activate}${reset_stderr} already activated"); return 1; } # We made it here, so we are good to activate the module! # Effectively, this just means removing the DEACTIVATED_MODULE_SUFFIX my $module_path_nosuffix = File::Spec->catfile($ast_path, $MODULES_FOLDER, $module_to_activate); my $old_module_path = $module_path_nosuffix . $DEACTIVATED_MODULE_SUFFIX; my $new_module_path = $module_path_nosuffix . $MODULE_SUFFIX; rename $old_module_path, $new_module_path; my $exitcode = $? >> 8; if ($exitcode) { print STDERR "${red_stderr}${bold_stderr}Activation of module ${module_to_activate} failed.${reset_stderr} Rename exited with $exitcode.\n"; return 0; } else { remove_active_module_index($ast_path, $program_name, $quiet); generate_active_module_index($ast_path, $program_name, $quiet, 0); unless ($quiet) { print "Module ${green_stdout}${bold_stdout}${module_to_activate}${reset_stdout} activated.\n"; } } return 1; } # Deactivate a module with a given name. # Params: # $module_to_deactivate: the name of the module to deactivate # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the running program # $quiet: suppress non-essential messages # Return: 1 on success, 0 on failure sub perform_deactivate { my $module_to_deactivate = shift; my $ast_path = shift; my $program_name = shift; my $quiet = shift; my %modules = get_module_to_status_hash($ast_path); unless (exists $modules{$module_to_deactivate}) { ast_utilities::error_output($program_name, "no such module ${bold_stderr}${module_to_deactivate}${reset_stderr}"); return 0; } my $status = $modules{$module_to_deactivate}; if ($status == $DEACTIVATED) { ast_utilities::warn_output($program_name, "module ${bold_stderr}${module_to_deactivate}${reset_stderr} already deactivated"); return 0; } # We made it here, so we are good to deactivate the module! # Effectively, this just means adding the DEACTIVATED_MODULE_SUFFIX my $module_path_nosuffix = File::Spec->catfile($ast_path, $MODULES_FOLDER, $module_to_deactivate); my $old_module_path = $module_path_nosuffix . $MODULE_SUFFIX; my $new_module_path = $module_path_nosuffix . $DEACTIVATED_MODULE_SUFFIX; rename $old_module_path, $new_module_path; my $exitcode = $? >> 8; if ($exitcode) { print STDERR "${red_stderr}${bold_stderr}Deactivation of module ${module_to_deactivate} failed.${reset_stderr} Rename exited with $exitcode.\n"; return 0; } else { unless ($quiet) { print "Module ${bold_stdout}${module_to_deactivate}${reset_stdout} deactivated.\n"; } } return 1; } # Create the active module index for the currently activated module. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $verbose_java: make the Java table printer use verbose output # Return: 1 on success, 0 on failure sub generate_active_module_index { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $verbose_java = shift; my %modules = get_module_to_status_hash($ast_path); my @activated_modules = get_activated_modules(\%modules); my $module = $activated_modules[0]; my $full_path_to_modules_folder = File::Spec->catfile($ast_path, $MODULES_FOLDER, "$module" . $MODULE_SUFFIX); my $full_path_to_log4j = File::Spec->catfile($ast_path, $ast_log_subsystem::LOG4J_FILE_PATH); my $full_index_path = File::Spec->catfile($ast_path, $ACTIVE_INDEX_PATH); my @java_command = (); push @java_command, "java"; push @java_command, "-Xms2G"; push @java_command, "-Xmx2G"; push @java_command, "-cp"; push @java_command, "${full_path_to_modules_folder}"; push @java_command, "-Dlog4j.configuration=file:${full_path_to_log4j}"; push @java_command, "${INDEX_WRITER_CLASS}"; push @java_command, "${full_index_path}"; if ($verbose_java && !$quiet) { push @java_command, "--verbose"; } if (scalar @activated_modules == 0) { ast_utilities::error_output($program_name, 'could not generate index'); print STDERR "No active modules found\n"; print STDERR "Try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} list${reset_stderr}' to see all installed modules.\n"; print STDERR "Then try '${bold_stderr}${ast_utilities::CONFIG_PROGRAM} activate ${reset_stderr}' to activate.\n"; return 0; } unless ($quiet) { print "Generating new index...\n"; } system {$java_command[0]} @java_command; my $exitcode = $? >> 8; unless ($exitcode == 0) { ast_utilities::error_output($program_name, 'could not generate index'); return 0; } unless ($quiet) { print "New index successfully generated.\n"; } return 1; } # Delete the active module index. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # Return: 1 on success, 0 on failure sub remove_active_module_index { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $full_index_path = File::Spec->catfile($ast_path, $ACTIVE_INDEX_PATH); unlink $full_index_path; unless ($quiet) { print "Cleared index.\n"; } } # Write a metadata hash for a module. # Params: # $modules_folder: the path to the module folder # $module_basename: the module basename # $metadata_ref: a reference to the metadata hash # $program_name: the name of the calling program # Return: 1 on success, 0 on failure sub write_module_metadata { my $modules_folder = shift; my $module_basename = shift; my $metadata_ref = shift; my $program_name = shift; my %metadata = %{$metadata_ref}; my $module_new_path_metadata = File::Spec->catfile($modules_folder, $module_basename . $METADATA_SUFFIX); open my $metadata_handle, '>', "$module_new_path_metadata"; if (defined $metadata{$SOURCE_KEY}) { print $metadata_handle $SOURCE_KEY . $METADATA_SEPARATOR . $metadata{$SOURCE_KEY} . "\n"; } else { ast_utilities::warn_output($program_name, "bad metadata: missing SOURCE_KEY"); close $metadata_handle; unlink $module_new_path_metadata; return 0; } if (defined $metadata{$URI_KEY}) { print $metadata_handle $URI_KEY . $METADATA_SEPARATOR . $metadata{$URI_KEY} . "\n"; } else { ast_utilities::warn_output($program_name, "bad metadata: missing URI_KEY"); close $metadata_handle; unlink $module_new_path_metadata; return 0; } if (defined $metadata{$DATE_TIME_KEY}) { print $metadata_handle $DATE_TIME_KEY . $METADATA_SEPARATOR . $metadata{$DATE_TIME_KEY} . "\n"; } else { ast_utilities::warn_output($program_name, "bad metadata: missing DATE_TIME_KEY"); close $metadata_handle; unlink $module_new_path_metadata; return 0; } if (defined $metadata{$REPO_NAME_KEY}) { print $metadata_handle $REPO_NAME_KEY . $METADATA_SEPARATOR . $metadata{$REPO_NAME_KEY} . "\n"; } if (defined $metadata{$REPO_REF_KEY}) { print $metadata_handle $REPO_REF_KEY . $METADATA_SEPARATOR . $metadata{$REPO_REF_KEY} . "\n"; } if (defined $metadata{$REPO_COMMIT_KEY}) { print $metadata_handle $REPO_COMMIT_KEY . $METADATA_SEPARATOR . $metadata{$REPO_COMMIT_KEY} . "\n"; } close $metadata_handle; return 1; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_preset_subsystem.pm ================================================ package ast_preset_subsystem; use warnings; use strict; use Exporter qw(import); use File::Spec; use File::Path qw(make_path rmtree); use File::Temp qw(tempdir tempfile); use ast_utilities; use ast_tty; # Export symbols: variables and subroutines our @EXPORT = qw( PRESETS_FOLDER CURRENT_NAMESPACE_FILE NAMESPACE_PATH DEFAULT_NAMESPACE reset_namespace get_namespace save_preset remove_preset remove_all_presets_for_command all_presets_for_command all_presets show_preset edit_preset copy_preset apply_preset_or_exit read_preset get_namespaces_array all_namespaces create_namespace use_namespace remove_namespace get_all_presets_in_current_namespace get_all_presets_for_command get_all_global_presets preset_regex_ok preset_regex ); our $PRESETS_FOLDER = 'presets'; our $GLOBAL_FOLDER = '.global'; our $CURRENT_NAMESPACE_FILE = '.current_namespace'; our $NAMESPACE_PATH = File::Spec->catfile($PRESETS_FOLDER, $CURRENT_NAMESPACE_FILE); our $DEFAULT_NAMESPACE = 'default'; my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); # A header for the preset edit screen. my $PRESET_EDIT_HEADER = "# Each line in this file will become a discrete ARGV element. For this # reason, please use '--option=optionArgument' syntax (note the '=') # for option arguments. Otherwise, your preset will not function properly. # Also note that all non-option arguments will be dropped, and any line # beginning with a '#' will be ignored. # # If you're stuck, hit then type :q! to abort the edit. # To save your changes, hit then type :wq"; # Reset the preset namespace to default. If an extra argument is supplied, # then reset the namespace to the supplied argument. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $new_namespace: an optional namespace to swap to # Return: none sub reset_namespace { my $ast_path = shift; my $new_namespace = shift; my $namespace_file = File::Spec->catfile($ast_path, $ast_preset_subsystem::NAMESPACE_PATH); open my $file_handle, '>', "$namespace_file"; if (defined $new_namespace) { print $file_handle "${new_namespace}\n"; } else { print $file_handle "${DEFAULT_NAMESPACE}\n"; } close $file_handle; } # Read and return the current namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: the current namespace sub get_namespace { my $ast_path = shift; my $namespace_file = File::Spec->catfile($ast_path, $NAMESPACE_PATH); open my $file_handle, '<', "$namespace_file"; my $firstline = <$file_handle>; chomp $firstline; close $file_handle; return $firstline; } # Save a preset for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # $preset: the name of the preset # $command: the name of the command # $namespace: the namespace to which to save # $argv_ref: a reference to an array containing all the options and args # Return: 1 on success, 0 on failure sub save_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $preset = shift; my $command = shift; my $namespace = shift; my $argv_ref = shift; my @argv = @{$argv_ref}; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); make_path("$preset_subfolder", { verbose => 0, mode => 0755 }); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); if (-f $preset_file) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "global preset ${bold_stderr}${preset}${reset_stderr} already exists"); } else { ast_utilities::error_output($program_name, "preset ${bold_stderr}${preset}${reset_stderr} already exists for ${bold_stderr}${command}${reset_stderr}"); } return 0; } my @detected_options = (); foreach my $arg (@argv) { # treat '-' as a regular argument if ($arg eq '-') { ast_utilities::warn_output($program_name, "preset discarding non-option arg \'${bold_stderr}$arg${reset_stderr}\'"); next; } # stop processing once we see '--' if ($arg eq '--') { last; } # detect an option or skip non-option arguments if (ast_utilities::string_starts_with($arg, '--') || ast_utilities::string_starts_with($arg, '-')) { push @detected_options, $arg; } else { ast_utilities::warn_output($program_name, "preset discarding non-option arg \'${bold_stderr}$arg${reset_stderr}\'"); } } if (scalar @detected_options == 0) { if (ast_utilities::is_dir_empty($preset_subfolder)) { rmdir($preset_subfolder); } ast_utilities::error_output($program_name, "cannot save an empty preset"); return 0; } if ($command eq $GLOBAL_FOLDER) { print "Global preset ${bold_stdout}${preset}${reset_stdout}:\n"; } else { print "Preset ${bold_stdout}${preset}${reset_stdout} for command ${bold_stdout}${command}${reset_stdout}:\n"; } print "\n${bunl_stdout}Preset ARGV${eunl_stdout}${reset_stdout}\n"; open my $file_handle, '>', "$preset_file"; foreach my $option (@detected_options) { print "${bold_stdout}${option}${reset_stdout}\n"; print $file_handle "${option}\n"; } close $file_handle; unless ($quiet) { if ($command eq $GLOBAL_FOLDER) { print "\nGlobal preset ${bold_stdout}${preset}${reset_stdout} saved.\n"; } else { print "\nPreset ${bold_stdout}${preset}${reset_stdout} saved for command ${bold_stdout}${command}${reset_stdout}.\n"; } } return 1; } # Remove a given preset for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $preset: the name of the preset # $command: the name of the command # $namespace: the namespace from which to remove # Return: 1 on success, 0 on failure sub remove_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $preset = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); unless (-f $preset_file) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "no such global preset ${bold_stderr}${preset}${reset_stderr}"); } else { ast_utilities::error_output($program_name, "no such preset ${bold_stderr}${preset}${reset_stderr} for command ${bold_stderr}${command}${reset_stderr}"); } return 0; } unlink $preset_file; unless ($quiet) { if ($command eq $GLOBAL_FOLDER) { print "Removed global preset ${bold_stdout}${preset}${reset_stdout}.\n"; } else { print "Removed preset ${bold_stdout}${preset}${reset_stdout} for ${bold_stdout}${command}${reset_stdout}.\n"; } } if (ast_utilities::is_dir_empty($preset_subfolder)) { rmdir($preset_subfolder); } return 1; } # Remove all presets for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $command: the name of the command # $namespace: the namespace from which to remove # Return: 1 on success, 0 on failure sub remove_all_presets_for_command { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); unless (-d $preset_subfolder) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "no global presets found"); } else { ast_utilities::error_output($program_name, "no presets found for command ${bold_stderr}${command}${reset_stderr}"); } return 0; } rmtree($preset_subfolder); unless ($quiet) { if ($command eq $GLOBAL_FOLDER) { print "Removed all global presets.\n"; } else { print "Removed all presets for ${bold_stdout}${command}${reset_stdout}.\n"; } } return 1; } # List all presets for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $command: the name of the command # $namespace: the namespace to save to # Return: 1 on success, 0 on failure sub all_presets_for_command { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); unless (-d $preset_subfolder) { if ($command eq $GLOBAL_FOLDER) { die "no global presets folder found in namespace ${namespace}"; } ast_utilities::error_output($program_name, "no presets found for ${bold_stderr}${command}${reset_stderr}"); return 0; } opendir my $presets_dir_handle, $preset_subfolder or die "Something went wrong opening dir: $!"; my @presets = readdir $presets_dir_handle; closedir $presets_dir_handle; # we need to filter '.' and '..' my @filtered_presets = (); for my $found_preset (@presets) { unless ($found_preset eq '.' || $found_preset eq '..') { push @filtered_presets, $found_preset; } } if (scalar @filtered_presets == 0) { if ($command eq $GLOBAL_FOLDER) { return 0; } return 0; } if ($command eq $GLOBAL_FOLDER) { print "Global presets:\n\n"; } else { print "Command ${bold_stdout}${command}${reset_stdout} presets:\n\n"; } foreach my $found_preset (sort {lc $a cmp lc $b} @filtered_presets) { print " ${bold_stdout}${found_preset}${reset_stdout}\n"; } print "\n"; return 1; } # List all presets in a given namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $namespace: the namespace to display # Return: 1 on success, 0 on failure sub all_presets { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $namespace = shift; my $namespace_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace); unless (-d $namespace_folder) { ast_utilities::error_output($program_name, "no such namespace ${bold_stderr}${namespace}${reset_stderr}"); return 0; } if (ast_utilities::is_dir_empty($namespace_folder)) { ast_utilities::error_output($program_name, "no presets found"); return 0; } opendir my $namespace_dir_handle, $namespace_folder or die "Something went wrong opening dir: $!"; my @command_folders = readdir $namespace_dir_handle; closedir $namespace_dir_handle; my $found_at_least_one = 0; foreach my $found_command (@command_folders) { my $command_folder = File::Spec->catfile($namespace_folder, $found_command); # we need to filter '.', '..', and the '.global' folder unless ($found_command eq '.' || $found_command eq '..' || $found_command eq $GLOBAL_FOLDER) { $found_at_least_one |= all_presets_for_command($ast_path, $program_name, $quiet, $found_command, $namespace); } } my $global_folder = File::Spec->catfile($namespace_folder, $GLOBAL_FOLDER); $found_at_least_one |= all_presets_for_command($ast_path, $program_name, $quiet, $GLOBAL_FOLDER, $namespace); unless ($found_at_least_one) { ast_utilities::error_output($program_name, "no presets found"); return 0; } return 1; } # Show a given preset for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $preset: the name of the preset # $command: the name of the command # $namespace: the namespace to save to # Return: 1 on success, 0 on failure sub show_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $preset = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); unless (-f $preset_file) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "no such global preset ${bold_stderr}${preset}${reset_stderr}"); } else { ast_utilities::error_output($program_name, "no such preset ${bold_stderr}${preset}${reset_stderr} for command ${bold_stderr}${command}${reset_stderr}"); } return 0; } my @presets_from_file = read_preset($ast_path, $program_name, $quiet, $preset, $command, $namespace); if ($command eq $GLOBAL_FOLDER) { print "Global preset ${bold_stdout}${preset}${reset_stdout}:\n"; } else { print "Preset ${bold_stdout}${preset}${reset_stdout} for command ${bold_stdout}${command}${reset_stdout}:\n"; } print "\n${bunl_stdout}Preset ARGV${eunl_stdout}${reset_stdout}\n"; for my $preset_from_file (@presets_from_file) { print "${bold_stdout}${preset_from_file}${reset_stderr}\n"; } print "\n"; return 1; } # Edit a given preset for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $preset: the name of the preset # $command: the name of the command # $namespace: the namespace # Return: 1 on success, 0 on failure sub edit_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $preset = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); make_path("$preset_subfolder", { verbose => 0, mode => 0755 }); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); my $creating_from_scratch = 0; unless (-f $preset_file) { $creating_from_scratch = 1; } my @presets_from_file = read_preset($ast_path, $program_name, $quiet, $preset, $command, $namespace); my $handle; my $stage_handle; my $tmpfile; my $preset_stage_file; my $tmpdir = tempdir(CLEANUP => 1); ($handle, $tmpfile) = tempfile(DIR => $tmpdir); ($stage_handle, $preset_stage_file) = tempfile(DIR => $tmpdir); close $handle; close $stage_handle; open $handle, '>', "$tmpfile"; if ($creating_from_scratch) { print $handle "# CREATING PRESET\n"; } else { print $handle "# EDITING PRESET\n"; } print $handle "# Preset: ${preset}\n"; print $handle "# Command: ${command}\n#\n"; print $handle "${PRESET_EDIT_HEADER}\n"; foreach my $cur_preset (@presets_from_file) { print $handle "${cur_preset}\n"; } print $handle "\n"; close $handle; my @editor = ast_utilities::get_editor(); push @editor, "$tmpfile"; system {$editor[0]} @editor; open $handle, '<', "$tmpfile"; open $stage_handle, '>', "$preset_stage_file"; while (my $line = <$handle>) { chomp $line; # skip empty lines if ($line eq '') { next; } # trim excess whitespace from left and right $line =~ s/^\s+|\s+$//g; # skip line if it starts with '#' if ($line =~ /^#.*/) { next; } # treat '-' as a regular argument if ($line eq '-') { ast_utilities::warn_output($program_name, "preset discarding non-option arg \'${bold_stderr}${line}${reset_stderr}\'"); next; } # stop processing once we see '--' if ($line eq '--') { last; } # detect an option or skip non-option arguments if (ast_utilities::string_starts_with($line, '--') || ast_utilities::string_starts_with($line, '-')) { print $stage_handle "$line\n"; } else { ast_utilities::warn_output($program_name, "preset discarding non-option arg \'${bold_stderr}${line}${reset_stderr}\'"); } } close $handle; close $stage_handle; # Verify that the staged preset file is not empty if (-z $preset_stage_file) { ast_utilities::error_output($program_name, "preset cannot be empty"); if (ast_utilities::is_dir_empty($preset_subfolder)) { rmdir($preset_subfolder); } return 0; } my @command = (); push @command, "cp"; push @command, "$preset_stage_file"; push @command, "$preset_file"; system {$command[0]} @command; close $tmpdir; show_preset($ast_path, $program_name, $quiet, $preset, $command, $namespace); return 1; } # For a given command, copy a source preset to a destination preset. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $src_preset: the source preset # $dest_preset: the destination preset # $command: the name of the command # $namespace: the namespace # Return: 1 on success, 0 on failure sub copy_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $src_preset = shift; my $dest_preset = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); my $source_file = File::Spec->catfile($preset_subfolder, $src_preset); my $dest_file = File::Spec->catfile($preset_subfolder, $dest_preset); unless (-e $source_file) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "no such global preset ${bold_stderr}${src_preset}${reset_stderr}"); } else { ast_utilities::error_output($program_name, "no such preset ${bold_stderr}${src_preset}${reset_stderr} for command ${bold_stderr}${command}${reset_stderr}"); } return 0; } if (-e $dest_file) { if ($command eq $GLOBAL_FOLDER) { ast_utilities::error_output($program_name, "global preset ${bold_stderr}${dest_preset}${reset_stderr} already exists"); } else { ast_utilities::error_output($program_name, "preset ${bold_stderr}${dest_preset}${reset_stderr} already exists for ${bold_stderr}${command}${reset_stderr}"); } return 0; } my @command = (); push @command, "cp"; push @command, "$source_file"; push @command, "$dest_file"; system {$command[0]} @command; unless ($quiet) { print "Copied contents of preset ${bold_stdout}${src_preset}${reset_stdout} into new preset ${bold_stdout}${dest_preset}${reset_stdout}.\n"; } return 1; } # Apply preset(s) for a given command. Returns an updated argv array with the # preset(s) applied. If the preset(s) do not exist, it will error and exit. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppres non-essential output # $presets: the name of the preset(s) # $command: the name of the command # $namespace: the namespace # $argv_ref: a reference to an array containing all the options and args # Return: the updated argv array sub apply_preset_or_exit { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $presets = shift; my $command = shift; my $namespace = shift; my $argv_ref = shift; my @argv = @{$argv_ref}; my @presets_comma = split ',', $presets; my @presets_colon = split ':', $presets; my @presets_array; # User cannot use both ',' and ':', pick one if (scalar @presets_colon > 1 && scalar @presets_comma > 1) { ast_utilities::error_output($program_name, "cannot use both \',\' and \':\' to split presets, choose one"); exit 1; } # In this case, either user split on ':' to separate or there was only 1 preset if (scalar @presets_colon >= scalar @presets_comma) { @presets_array = @presets_colon; } # Otherwise, the user split on ',' so use that else { @presets_array = @presets_comma; } my @final_argv = (); my $use_global = 0; foreach my $preset (@presets_array) { my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); unless (-f $preset_file) { # Fall back and try global presets my $global_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $GLOBAL_FOLDER); my $global_file = File::Spec->catfile($global_subfolder, $preset); unless (-f $global_file) { ast_utilities::error_output($program_name, "preset ${bold_stderr}${preset}${reset_stderr} not found for command ${bold_stderr}${command}${reset_stderr}"); ast_utilities::error_output($program_name, "preset ${bold_stderr}${preset}${reset_stderr} not found in globals"); all_presets_for_command($ast_path, $program_name, $quiet, $command, $namespace); all_presets_for_command($ast_path, $program_name, $quiet, $GLOBAL_FOLDER, $namespace); exit 1; } $use_global = 1; } my @argv_from_presets = (); if ($use_global) { @argv_from_presets = read_preset($ast_path, $program_name, $quiet, $preset, $GLOBAL_FOLDER, $namespace); } else { @argv_from_presets = read_preset($ast_path, $program_name, $quiet, $preset, $command, $namespace); } foreach my $preset_argv_elem (@argv_from_presets) { push @final_argv, $preset_argv_elem; } } foreach my $argv_elem (@argv) { push @final_argv, $argv_elem; } return @final_argv; } # Read a preset for a given command. Returns the preset in an array # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $preset: the name of the preset # $command: the name of the command # $namespace: the current namespace # Return: the preset array, or an empty array on error sub read_preset { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $preset = shift; my $command = shift; my $namespace = shift; my $preset_subfolder = File::Spec->catfile($ast_path, $PRESETS_FOLDER, $namespace, $command); my $preset_file = File::Spec->catfile($preset_subfolder, $preset); unless (-f $preset_file) { return(); } my @options = (); open my $file_handle, '<', $preset_file; # NOTE this fails if a preset option argument contained a newline while (my $line = <$file_handle>) { chomp $line; push @options, $line; } close($file_handle); return @options; } # Get an array containing each namespace as an element. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: the namespace array sub get_namespaces_array { my $ast_path = shift; my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); unless (-d $preset_folder) { die "The folder $PRESETS_FOLDER did not exist at $ast_path"; } opendir my $presets_dir_handle, $preset_folder or die "Something went wrong opening dir: $!"; my @namespaces = readdir $presets_dir_handle; closedir $presets_dir_handle; # we need to filter '.', '..', and '.current_namespace' my @filtered_namespaces = (); for my $found_namespace (@namespaces) { unless ($found_namespace eq '.' || $found_namespace eq '..' || $found_namespace eq $CURRENT_NAMESPACE_FILE) { push @filtered_namespaces, $found_namespace; } } return @filtered_namespaces; } # Print all namespaces, and denote the current checked out namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # Return: 1 on success, 0 on failure sub all_namespaces { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $current_namespace = get_namespace($ast_path); my @namespaces = get_namespaces_array($ast_path); if (scalar @namespaces == 0) { error_output($program_name, "no namespaces found"); return 0; } print "${bold_stdout}Preset namespaces:${reset_stdout}\n\n"; for my $found_namespace (sort {lc $a cmp lc $b} @namespaces) { if ($found_namespace eq $current_namespace) { print " * ${bold_stdout}${green_stdout}${found_namespace}${reset_stdout}\n"; } else { print " ${found_namespace}\n"; } } print "\n"; return 1; } # Create a new namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $new_namespace: the namespace to create # Return: 1 on success, 0 on failure sub create_namespace { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $new_namespace = shift; my $current_namespace = get_namespace($ast_path); my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $new_namespace_folder = File::Spec->catfile($preset_folder, $new_namespace); my $global_folder = File::Spec->catfile($preset_folder, $new_namespace, $GLOBAL_FOLDER); unless (-d $preset_folder) { die "The folder $PRESETS_FOLDER did not exist at $ast_path"; } if (-d $new_namespace_folder) { ast_utilities::error_output($program_name, "namespace ${bold_stderr}${new_namespace}${reset_stderr} already exists"); return 0; } make_path("$new_namespace_folder", { verbose => 0, mode => 0755 }); make_path("$global_folder", { verbose => 0, mode => 0755 }); unless ($quiet) { print "Created namespace ${bold_stdout}${new_namespace}${reset_stdout}.\n"; } return 1; } # Check out a given namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $namespace: the namespace to use # Return: 1 on success, 0 on failure sub use_namespace { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $namespace = shift; my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $namespace_folder = File::Spec->catfile($preset_folder, $namespace); unless (-d $preset_folder) { die "The folder $PRESETS_FOLDER did not exist at $ast_path"; } unless (-d $namespace_folder) { ast_utilities::error_output($program_name, "no such namespace ${bold_stderr}${namespace}${reset_stderr}"); return 0; } reset_namespace($ast_path, $namespace); unless ($quiet) { print "Now using namespace ${bold_stdout}${namespace}${reset_stdout}.\n"; } return 1; } # Remove a namespace. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output # $namespace: the namespace to remove # Return: 1 on success, 0 on failure sub remove_namespace { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $namespace = shift; my $current_namespace = get_namespace($ast_path); my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $namespace_folder = File::Spec->catfile($preset_folder, $namespace); unless (-d $preset_folder) { die "The folder $PRESETS_FOLDER did not exist at $ast_path"; } unless (-d $namespace_folder) { ast_utilities::error_output($program_name, "no such namespace ${bold_stderr}${namespace}${reset_stderr}"); return 0; } if ($namespace eq $DEFAULT_NAMESPACE) { ast_utilities::error_output($program_name, "cannot remove the default namespace"); return 0; } if ($namespace eq $current_namespace) { ast_utilities::error_output($program_name, "cannot remove in-use namespace ${bold_stderr}${namespace}${reset_stderr}"); return 0; } rmtree($namespace_folder); unless ($quiet) { print "Removed namespace ${bold_stdout}${namespace}${reset_stdout}.\n"; } return 1; } # Get an array of all presets in the current namespace, including global presets. # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: the preset array sub get_all_presets_in_current_namespace { my $ast_path = shift; my $namespace = get_namespace($ast_path); my @all_presets = (); my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $namespace_folder = File::Spec->catfile($preset_folder, $namespace); my $global_folder = File::Spec->catfile($namespace_folder, $GLOBAL_FOLDER); opendir my $namespace_dir_handle, $namespace_folder or die "Something went wrong opening dir: $!"; my @command_folders = readdir $namespace_dir_handle; closedir $namespace_dir_handle; foreach my $found_command (@command_folders) { my $command_folder = File::Spec->catfile($namespace_folder, $found_command); # we need to filter '.', '..' unless ($found_command eq '.' || $found_command eq '..') { opendir my $command_dir_handle, $command_folder or die "Something went wrong opening dir: $!"; my @preset_files = readdir $command_dir_handle; closedir $command_dir_handle; foreach my $found_preset (@preset_files) { # we need to filter '.', '..' unless ($found_preset eq '.' || $found_preset eq '..') { push @all_presets, $found_preset; } } } } opendir my $global_dir_handle, $global_folder or die "Something went wrong opening dir: $!"; my @preset_files = readdir $global_dir_handle; closedir $global_dir_handle; foreach my $found_preset (@preset_files) { # we need to filter '.', '..' unless ($found_preset eq '.' || $found_preset eq '..') { push @all_presets, $found_preset; } } return @all_presets; } # Get an array of all presets in the current namespace for a given command. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $command: the command from which to check presets # Return: the preset array sub get_all_presets_for_command { my $ast_path = shift; my $command = shift; my $namespace = get_namespace($ast_path); my @all_presets = (); my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $namespace_folder = File::Spec->catfile($preset_folder, $namespace); my $command_folder = File::Spec->catfile($namespace_folder, $command); unless (-d $command_folder) { return @all_presets; } opendir my $command_dir_handle, $command_folder or die "Something went wrong opening dir: $!"; my @preset_files = readdir $command_dir_handle; closedir $command_dir_handle; foreach my $found_preset (@preset_files) { # we need to filter '.', '..' unless ($found_preset eq '.' || $found_preset eq '..') { push @all_presets, $found_preset; } } return @all_presets; } # Get an array of all global presets in the current namespace # Params: # $ast_path: the path to the atlas-shell-tools data folder # Return: the preset array sub get_all_global_presets { my $ast_path = shift; my $namespace = get_namespace($ast_path); my @all_presets = (); my $preset_folder = File::Spec->catfile($ast_path, $PRESETS_FOLDER); my $namespace_folder = File::Spec->catfile($preset_folder, $namespace); my $global_folder = File::Spec->catfile($namespace_folder, $GLOBAL_FOLDER); unless (-d $global_folder) { return @all_presets; } opendir my $global_dir_handle, $global_folder or die "Something went wrong opening dir: $!"; my @preset_files = readdir $global_dir_handle; closedir $global_dir_handle; foreach my $found_preset (@preset_files) { # we need to filter '.', '..' unless ($found_preset eq '.' || $found_preset eq '..') { push @all_presets, $found_preset; } } return @all_presets; } # Check that a preset name matches the approved name regex. # Params: # $preset: the preset to check # Return: if the preset name matched the regex sub preset_regex_ok { my $preset = shift; my $regex = preset_regex(); if ($preset =~ m/$regex/) { return 1; } return 0; } # Get the valid preset name regex. # Params: none # Return: the valid preset name regex sub preset_regex { return "^[_a-zA-Z0-9][_a-zA-Z0-9-]*\$"; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_repo_subsystem.pm ================================================ package ast_repo_subsystem; use warnings; use strict; use Exporter qw(import); use File::Basename qw(basename); use File::Path qw(make_path rmtree); use File::Temp qw(tempdir tempfile); use POSIX qw(strftime); use ast_tty; use ast_module_subsystem; use ast_utilities; # Export symbols: variables and subroutines our @EXPORT = qw( REPOS_FOLDER create_repo edit_repo list_repos remove_repo install_repo get_all_repos add_skip_variable add_exclude_variable get_repo_settings print_repo_settings ); our $REPOS_FOLDER = 'repos'; our $REPO_CONFIG = 'repo_config'; my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); my $REPO_EDIT_HEADER = "# Lines beginning with \"#\" are ignored # # Add exclude packages for gradle using \"exclude\". # To exclude multiple packages, simply repeat this config variable for each package. E.g. # exclude = com.example.package # exclude = com.example.anotherpackage # # Skip gradle tasks using \"skip\". # To skip multiple tasks, simply repeat this config variable for each task. E.g. # skip = javadoc # skip = integrationTest # # If you're stuck, hit then type :q! to abort the edit. # To save your changes, hit then type :wq"; # # TODO fix all the hardcoded stuff. # e.g. 'url', 'ref', 'exclude', etc. # # Create a new repo. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # $repo: the name of the repo # $url: the repo URL # $ref: the ref (a branch, tag, even a commit) # Return: 1 on success, 0 on failure sub create_repo { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $url = shift; my $ref = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); if (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} already exists"); return 0; } make_path("$repo_subfolder", { verbose => 0, mode => 0755 }); my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); open my $file_handle, '>', "$repo_config_file"; print $file_handle "url = ${url}\n"; print $file_handle "ref = ${ref}\n"; close $file_handle; unless ($quiet) { print "New repo: ${bold_stdout}${repo}${reset_stdout}\nURL: ${bold_stdout}${url}${reset_stdout}\nRef: ${bold_stdout}${ref}${reset_stdout}\n"; } return 1; } # Edit a repo. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # $repo: the name of the repo # Return: 1 on success, 0 on failure sub edit_repo { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "no such repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); print STDERR "To resolve, please remove the repo and re-add.\n"; return 0; } # Create the staging file my $handle; my $staging_file; my $tmpdir = tempdir(CLEANUP => 1); ($handle, $staging_file) = tempfile(DIR => $tmpdir); close $handle; # copy the current config file into the staging file open my $stage_handle, '>', "$staging_file"; print $stage_handle "# Config for repo ${repo}\n"; print $stage_handle "${REPO_EDIT_HEADER}\n"; my $url = read_single_config_variable_from_arbitrary_file($repo_config_file, 'url'); my $ref = read_single_config_variable_from_arbitrary_file($repo_config_file, 'ref'); my @skips = read_multiple_config_variables_from_arbitrary_file($repo_config_file, 'skip'); my @excludes = read_multiple_config_variables_from_arbitrary_file($repo_config_file, 'exclude'); print $stage_handle "url = ${url}\n"; print $stage_handle "ref = ${ref}\n"; foreach my $skip (@skips) { print $stage_handle "skip = ${skip}\n"; } foreach my $exclude (@excludes) { print $stage_handle "exclude = ${exclude}\n"; } close $stage_handle; # open the staging file in the user's editor my @editor = ast_utilities::get_editor(); push @editor, "$staging_file"; system {$editor[0]} @editor; # confirm that the staging file is not malformed, i.e. it must have a valid URL and ref $url = read_single_config_variable_from_arbitrary_file($staging_file, 'url'); $ref = read_single_config_variable_from_arbitrary_file($staging_file, 'ref'); if ($url eq '' || $ref eq '') { ast_utilities::error_output($program_name, "failed to parse \'url\' and \'ref\' config variables"); print STDERR "Aborting edit without saving...\n"; return 0; } # copy the staging file back into the actual config file open $handle, '>', "$repo_config_file"; $url = read_single_config_variable_from_arbitrary_file($staging_file, 'url'); $ref = read_single_config_variable_from_arbitrary_file($staging_file, 'ref'); @skips = read_multiple_config_variables_from_arbitrary_file($staging_file, 'skip'); @excludes = read_multiple_config_variables_from_arbitrary_file($staging_file, 'exclude'); print $handle "url = ${url}\n"; print $handle "ref = ${ref}\n"; foreach my $skip (@skips) { print $handle "skip = ${skip}\n"; } foreach my $exclude (@excludes) { print $handle "exclude = ${exclude}\n"; } close $handle; return 1; } # List the repos. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # Return: 1 on success, 0 on failure sub list_repos { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my @filtered_repos = get_all_repos($ast_path); if (scalar @filtered_repos == 0) { ast_utilities::error_output($program_name, "found no repos"); return 0; } print "${bold_stdout}Registered repos:${reset_stdout}\n\n"; for my $found_repo (sort {lc $a cmp lc $b} @filtered_repos) { my $url = read_single_config_variable($ast_path, $program_name, $quiet, $found_repo, 'url'); my $ref = read_single_config_variable($ast_path, $program_name, $quiet, $found_repo, 'ref'); unless (defined $url && defined $ref) { ast_utilities::error_output($program_name, "repo list operation failed"); return 0; } print " ${bold_stdout}${found_repo}${reset_stdout} : ${url} (${ref})\n"; } print "\n"; return 1; } # Remove an existing repo. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # $repo: the name of the repo # Return: 1 on success, 0 on failure sub remove_repo { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "no such repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } rmtree($repo_subfolder); unless ($quiet) { print "Removed repo ${bold_stdout}${repo}${reset_stdout}.\n"; } return 1; } # Install module using an existing repo. # Params: # $ast_path: the path to the atlas-shell-tools data folder # $program_name: the name of the calling program # $quiet: suppress non-essential output output # $repo: the name of the repo # $ref_override: an optional override ref # Return: 1 on success, 0 on failure sub install_repo { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $ref_override = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "no such repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } my %module_metadata = ast_module_subsystem::get_module_to_metadata_hash($ast_path); my $url = read_single_config_variable($ast_path, $program_name, $quiet, $repo, 'url'); my $ref = read_single_config_variable($ast_path, $program_name, $quiet, $repo, 'ref'); my @excludes = read_multiple_config_variables($ast_path, $program_name, $quiet, $repo, 'exclude'); my @skips = read_multiple_config_variables($ast_path, $program_name, $quiet, $repo, 'skip'); unless (defined $url && defined $ref) { ast_utilities::error_output($program_name, "repo install operation failed"); return 0; } my $ref_to_use; if ($ref_override eq '') { $ref_to_use = $ref; } else { $ref_to_use = $ref_override; } my $tmpdir = tempdir(CLEANUP => 1); # First, we can check to see if the provided ref is already represented in one of the installed # modules. Since this uses git ls-remote, this only works when the ref is a tag or a branch. If # the ref was a commit hash, then we will need to clone the repo first to determine if an install # is necessary. my @command = (); push @command, "git"; push @command, "ls-remote"; push @command, "${url}"; push @command, "${ref_to_use}"; my $lsremote_result = ast_utilities::read_command_output(\@command); my @remote_ref = split /\s+/, $lsremote_result; my $remote_commit_hash; if (scalar @remote_ref > 0) { $remote_commit_hash = $remote_ref[0]; } if (defined $remote_commit_hash) { foreach my $module_key (keys %module_metadata) { my %metadata = %{$module_metadata{$module_key}}; my $module_commit_hash = $metadata{$ast_module_subsystem::REPO_COMMIT_KEY}; if (defined $module_commit_hash && $module_commit_hash eq $remote_commit_hash) { ast_utilities::warn_output($program_name, "nothing to do"); print STDERR "Ref ${bold_stderr}${ref_to_use}${reset_stderr} in repo ${bold_stderr}${repo}${reset_stderr} refers to commit ${bold_stderr}${remote_commit_hash}${reset_stderr}.\n"; print STDERR "Installed module ${bold_stderr}${module_key}${reset_stderr} was built from this commit.\n"; print STDERR "Try \'${bold_stderr}${ast_utilities::CONFIG_PROGRAM} activate ${module_key}${reset_stderr}\' to use this version.\n"; return 1; } } } @command = (); push @command, "git"; push @command, "clone"; push @command, "${url}"; push @command, "${tmpdir}"; my $success = system {$command[0]} @command; unless ($success == 0) { ast_utilities::error_output($program_name, "repo install operation failed"); return 0; } chdir $tmpdir or die "$!"; @command = (); push @command, "git"; push @command, "checkout"; push @command, "${ref_to_use}"; $success = system {$command[0]} @command; unless ($success == 0) { ast_utilities::error_output($program_name, "repo install operation failed"); return 0; } my $installed_commit_hash = `git rev-parse HEAD`; my $installed_commit_hash_short = `git rev-parse --short HEAD`; chomp $installed_commit_hash; chomp $installed_commit_hash_short; my $tentative_module_name = "${repo}-${installed_commit_hash_short}"; my %modules = ast_module_subsystem::get_module_to_status_hash($ast_path); my @module_names = keys %modules; foreach my $module_name (@module_names) { if ($tentative_module_name eq $module_name) { print STDERR "\n"; ast_utilities::warn_output($program_name, "nothing to do"); print STDERR "Ref ${bold_stderr}${ref_to_use}${reset_stderr} in repo ${bold_stderr}${repo}${reset_stderr} refers to commit ${bold_stderr}${installed_commit_hash}${reset_stderr}.\n"; print STDERR "Installed module ${bold_stderr}${module_name}${reset_stderr} was built from this commit.\n"; print STDERR "Try \'${bold_stderr}${ast_utilities::CONFIG_PROGRAM} activate ${module_name}${reset_stderr}\' to use this version.\n"; return 1; } } my $gradle_injection = " task atlasshelltools(type: Jar) { baseName = project.name classifier = '-AST' duplicatesStrategy = 'exclude' from { configurations.atlasshelltools.collect { it.isDirectory() ? it : zipTree(it).matching { exclude { it.path.contains('META-INF') && (it.path.endsWith('.SF') || it.path.endsWith('.DSA') || it.path.endsWith('.RSA')) } } } } with jar zip64 = true } configurations { atlasshelltools { %s } } dependencies { atlasshelltools project.configurations.getByName('implementation') if (packages.slf4j != null) { atlasshelltools packages.slf4j.api } if (packages.log4j != null) { atlasshelltools packages.log4j.api atlasshelltools packages.log4j.slf4j } } "; my @excludes_mapped = (); foreach my $exclude_element (@excludes) { push @excludes_mapped, "exclude group: \'${exclude_element}\';"; } $gradle_injection = sprintf($gradle_injection, join("\n", @excludes_mapped)); open my $file_handle, '>>', "$tmpdir/build.gradle" or die "Could not open build.gradle $!"; print $file_handle "${gradle_injection}\n"; close $file_handle; @command = (); push @command, "./gradlew"; push @command, "clean"; push @command, "atlasshelltools"; foreach my $skip_element (@skips) { push @command, "-x"; push @command, "$skip_element"; } $success = system {$command[0]} @command; unless ($success == 0) { ast_utilities::error_output($program_name, "repo install operation failed"); return 0; } my @find_command = ( "find", ".", "-type", "f", "-name", "*-AST.jar", "-print0" ); open FIND, "-|", @find_command; # TODO 'local' modifier makes sense here? confirm this, 'my' may make more sense # see https://www.perlmonks.org/?node_id=94007 local $/ = "\0"; while () { # FIND command is printing full paths, we just want the basename. # Also, we must chomp to remove the terminating null byte left over from # the '-print0' flag given to 'find'. my $module = $_; chomp $module; my $module_basename = basename($module); my %local_metadata; $local_metadata{$ast_module_subsystem::SOURCE_KEY} = "repo"; $local_metadata{$ast_module_subsystem::URI_KEY} = "${url}"; $local_metadata{$ast_module_subsystem::REPO_NAME_KEY} = "${repo}"; $local_metadata{$ast_module_subsystem::REPO_REF_KEY} = "${ref_to_use}"; $local_metadata{$ast_module_subsystem::REPO_COMMIT_KEY} = "${installed_commit_hash}"; $local_metadata{$ast_module_subsystem::DATE_TIME_KEY} = strftime("%Y-%m-%d %H:%M:%S UTC", gmtime(time)); # install the module! my $success = ast_module_subsystem::perform_install($module, $ast_path, $program_name, "${repo}-${installed_commit_hash_short}", 0, 0, 1, 0, \%local_metadata, 0); unless ($success) { ast_utilities::error_output($program_name, "repo install operation failed"); return 0; } } return 1; } # Get an array containing all repo names. Useful for autocomplete and listing code. sub get_all_repos { my $ast_path = shift; my $repo_folder = File::Spec->catfile($ast_path, $REPOS_FOLDER); opendir my $repo_dir_handle, $repo_folder or die "Something went wrong opening dir: $!"; my @repos = readdir $repo_dir_handle; closedir $repo_dir_handle; # we need to filter '.' and '..' my @filtered_repos = (); for my $found_repo (@repos) { unless ($found_repo eq '.' || $found_repo eq '..') { push @filtered_repos, $found_repo; } } return @filtered_repos; } # TODO refactor DRY # Wrapper for append_config_variable_to_file. Used by UI code. sub add_skip_variable { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $value = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} does not exist"); return 0; } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } append_config_variable_to_file($repo_config_file, 'skip', $value); return 1; } # TODO refactor DRY # Wrapper for append_config_variable_to_file. Used by UI code. sub add_exclude_variable { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $value = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} does not exist"); return 0; } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } append_config_variable_to_file($repo_config_file, 'exclude', $value); return 1; } # Get an array containing string-ified repo settings. The array is useful for # output purposes. sub get_repo_settings { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} does not exist"); return(); } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); return(); } my @settings = (); open my $file_handle, '<', $repo_config_file or die "Could not open file $repo_config_file $!"; while (my $line = <$file_handle>) { chomp $line; if ($line eq '' || substr($line, 0, 1) eq '#') { next; } # trim excess whitespace from left and right $line =~ s/^\s+|\s+$//g; if ($line eq '' || substr($line, 0, 1) eq '#') { next; } my @line_split = split '=', $line, 2; unless (defined $line_split[0]) { next; } # trim excess whitespace from left and right $line_split[0] =~ s/^\s+|\s+$//g; if (defined $line_split[1] && $line_split[1] !~ /^\s*$/) { # trim excess whitespace from left and right $line_split[1] =~ s/^\s+|\s+$//g; push @settings, "$line_split[0] = $line_split[1]"; } } close $file_handle; return @settings; } # Print repo settings using get_repo_settings. sub print_repo_settings { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; if ($quiet) { return; } my @settings = get_repo_settings($ast_path, $program_name, $quiet, $repo); if (scalar @settings != 0) { print "${bold_stdout}${repo} settings:${reset_stdout}\n"; } foreach my $setting (@settings) { print "${setting}\n"; } } # TODO refactor DRY # Given an arbitrary file path, opens it and attempts to read the first config # variable that matches the given variable. If the value cannot be read, returns # an empty string. sub read_single_config_variable_from_arbitrary_file { my $file = shift; my $variable = shift; my $value = ''; open my $file_handle, '<', $file or die "Could not open file $file $!"; while (my $line = <$file_handle>) { chomp $line; # trim excess whitespace from left and right $line =~ s/^\s+|\s+$//g; if ($line eq '' || substr($line, 0, 1) eq '#') { next; } my @line_split = split '=', $line, 2; unless (defined $line_split[0]) { next; } # trim excess whitespace from left and right $line_split[0] =~ s/^\s+|\s+$//g; if ($line_split[0] eq $variable) { if (defined $line_split[1] && $line_split[1] !~ /^\s*$/) { # trim excess whitespace from left and right $line_split[1] =~ s/^\s+|\s+$//g; $value = $line_split[1]; } } } close $file_handle; return $value; } # TODO refactor DRY # Given an arbitrary file path, opens it and attempts to read the all config # variables that match the given variable. The values will be returned in an array. # If no values could be read, the array will be empty. sub read_multiple_config_variables_from_arbitrary_file { my $file = shift; my $variable = shift; my @values = (); open my $file_handle, '<', $file or die "Could not open file $file $!"; while (my $line = <$file_handle>) { chomp $line; # trim excess whitespace from left and right $line =~ s/^\s+|\s+$//g; if ($line eq '' || substr($line, 0, 1) eq '#') { next; } my @line_split = split '=', $line, 2; unless (defined $line_split[0]) { next; } # trim excess whitespace from left and right $line_split[0] =~ s/^\s+|\s+$//g; if ($line_split[0] eq $variable) { if (defined $line_split[1] && $line_split[1] !~ /^\s*$/) { # trim excess whitespace from left and right $line_split[1] =~ s/^\s+|\s+$//g; push @values, $line_split[1]; } } } close $file_handle; return @values; } # Given a file, a variable, and a value, append the variable setting to the file. sub append_config_variable_to_file { my $file = shift; my $variable = shift; my $value = shift; open my $file_handle, '>>', $file or die "Could not open file $file $!"; print $file_handle "${variable} = ${value}\n"; close $file_handle; } # Wrapper for read_single_config_variable_from_arbitrary_file that uses a given # ast_path and repo name. This makes it easier for some of the repo subroutines to # do proper error handling. sub read_single_config_variable { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $variable = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} does not exist"); return 0; } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } my $value = read_single_config_variable_from_arbitrary_file($repo_config_file, $variable); if ($value eq '') { ast_utilities::error_output($program_name, "failed to parse config setting \'${variable}\' for repo ${bold_stderr}${repo}${reset_stderr}"); return undef; } return $value; } # Wrapper for read_multiple_config_variables_from_arbitrary_file that uses a given # ast_path and repo name. This makes it easier for some of the repo subroutines to # do proper error handling. sub read_multiple_config_variables { my $ast_path = shift; my $program_name = shift; my $quiet = shift; my $repo = shift; my $variable = shift; my $repo_subfolder = File::Spec->catfile($ast_path, $REPOS_FOLDER, $repo); unless (-d $repo_subfolder) { ast_utilities::error_output($program_name, "repo ${bold_stderr}${repo}${reset_stderr} does not exist"); return 0; } my $repo_config_file = File::Spec->catfile($repo_subfolder, $REPO_CONFIG); unless (-f $repo_config_file) { ast_utilities::error_output($program_name, "could not find config file for repo ${bold_stderr}${repo}${reset_stderr}"); return 0; } my @values = read_multiple_config_variables_from_arbitrary_file($repo_config_file, $variable); return @values; } # Check that a repo name matches the approved name regex. # Params: # $repo: the repo to check # Return: if the repo name matched the regex sub repo_regex_ok { my $repo = shift; if ($repo =~ m/^[_a-zA-Z0-9][_a-zA-Z0-9-]*$/) { return 1; } return 0; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_tty.pm ================================================ package ast_tty; use warnings; use strict; use Exporter qw(import); # Export symbols: variables and subroutines our @EXPORT = qw( is_no_colors is_no_colors_stdout is_no_colors_stderr ansi_red ansi_green ansi_magenta ansi_bold ansi_reset ansi_begin_underln ansi_end_underln terminal_width ); my $no_colors_stdout = is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ansi_reset(); my $no_colors_stderr = is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ansi_reset(); # Determine if we should disable text/color formatting for output. Various # conditions are checked, and if none of them trigger then we can use colors! # We also make one check for explicit use of colors, to allow a case where a # user has set NO_COLOR, but would like to make an exception for atlas-shell-tools. # Params: none # Return: 1 if no colors, 0 otherwise sub is_no_colors { # check for dumb # TODO need to check for xterm too? if ($ENV{'TERM'} eq "dumb") { return 1; } # explicitly use colors for atlas-shell-tools if (exists $ENV{'ATLAS_SHELL_TOOLS_USE_COLOR'}) { return 0; } # respect the NO_COLOR env var if (exists $ENV{'NO_COLOR'}) { return 1; } if (exists $ENV{'ATLAS_SHELL_TOOLS_NO_COLOR'}) { return 1; } return 0; } # Same as the is_no_colors check, but also looks to see if stdout is a TTY. # Params: none # Return: 1 if no colors, 0 otherwise sub is_no_colors_stdout { my $no_colors = is_no_colors(); my $is_stdout_tty = -t STDOUT ? 1 : 0; return 1 if $no_colors || !$is_stdout_tty; return 0; } # Same as the is_no_colors check, but also looks to see if stderr is a TTY. # Params: none # Return: 1 if no colors, 0 otherwise sub is_no_colors_stderr { my $no_colors = is_no_colors(); my $is_stderr_tty = -t STDERR ? 1 : 0; return 1 if $no_colors || !$is_stderr_tty; return 0; } sub ansi_red { return `tput setaf 1`; } sub ansi_green { return `tput setaf 2`; } sub ansi_magenta { return `tput setaf 5`; } sub ansi_bold { return `tput bold`; } sub ansi_blink { return `tput blink`; } sub ansi_reset { return `tput sgr0`; } sub ansi_begin_underln { return `tput smul`; } sub ansi_end_underln { return `tput rmul`; } sub terminal_width { # 'tput' returns a string with a trailing newline my $cols = `tput cols`; # Explicitly convert to an integer here, removing the newline # This allows allows for calling code to do math with the value return int($cols); } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: atlas-shell-tools/scripts/common/ast_utilities.pm ================================================ package ast_utilities; use warnings; use strict; use Exporter qw(import); use File::Path qw(make_path); use List::Util qw(min); use ast_tty; use ast_log_subsystem; use ast_preset_subsystem; use ast_repo_subsystem; # Export symbols: variables and subroutines our @EXPORT = qw( ATLAS_SHELL_TOOLS_VERSION COMMAND_PROGRAM CONFIG_PROGRAM JAVA_NO_COLOR_SENTINEL JAVA_COLOR_STDOUT JAVA_NO_COLOR_STDOUT JAVA_COLOR_STDERR JAVA_NO_COLOR_STDERR JAVA_USE_PAGER JAVA_NO_USE_PAGER JAVA_MARKER_SENTINEL verify_environment_or_exit create_data_directory display_and_exit getopt_failure_and_exit error_output warn_output prompt prompt_yn get_pager get_editor get_man string_starts_with is_dir_empty levenshtein read_command_output ); our $ATLAS_SHELL_TOOLS_VERSION = "atlas-shell-tools version 1.0.0"; our $COMMAND_PROGRAM = 'atlas'; our $CONFIG_PROGRAM = 'atlas-config'; our $JAVA_COLOR_STDOUT = "___atlas-shell-tools_color_stdout_SPECIALARGUMENT___"; our $JAVA_NO_COLOR_STDOUT = "___atlas-shell-tools_nocolor_stdout_SPECIALARGUMENT___"; our $JAVA_COLOR_STDERR = "___atlas-shell-tools_color_stderr_SPECIALARGUMENT___"; our $JAVA_NO_COLOR_STDERR = "___atlas-shell-tools_nocolor_stderr_SPECIALARGUMENT___"; our $JAVA_USE_PAGER = "___atlas-shell-tools_use_pager_SPECIALARGUMENT___"; our $JAVA_NO_USE_PAGER = "___atlas-shell-tools_no_use_pager_SPECIALARGUMENT___"; our $JAVA_MARKER_SENTINEL = "___atlas-shell-tools_LAST_ARG_MARKER_SENTINEL___"; my $integrity_file = ".atlas-shell-tools-integrity-file"; my $no_colors_stdout = ast_tty::is_no_colors_stdout(); my $red_stdout = $no_colors_stdout ? "" : ast_tty::ansi_red(); my $green_stdout = $no_colors_stdout ? "" : ast_tty::ansi_green(); my $magenta_stdout = $no_colors_stdout ? "" : ast_tty::ansi_magenta(); my $bold_stdout = $no_colors_stdout ? "" : ast_tty::ansi_bold(); my $bunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_begin_underln(); my $eunl_stdout = $no_colors_stdout ? "" : ast_tty::ansi_end_underln(); my $reset_stdout = $no_colors_stdout ? "" : ast_tty::ansi_reset(); my $no_colors_stderr = ast_tty::is_no_colors_stderr(); my $red_stderr = $no_colors_stderr ? "" : ast_tty::ansi_red(); my $green_stderr = $no_colors_stderr ? "" : ast_tty::ansi_green(); my $magenta_stderr = $no_colors_stderr ? "" : ast_tty::ansi_magenta(); my $bold_stderr = $no_colors_stderr ? "" : ast_tty::ansi_bold(); my $reset_stderr = $no_colors_stderr ? "" : ast_tty::ansi_reset(); # Ensure that the necessary environment variables are configured. If not, # exit with an error. # Params: none # Return: none sub verify_environment_or_exit { unless (defined $ENV{HOME}) { print STDERR "Error: HOME environment variable is not set\n"; exit 1; } unless (-d $ENV{HOME}) { print STDERR "Error: the directory referenced by HOME does not exist\n"; exit 1; } unless (-w $ENV{HOME}) { print STDERR "Error: the directory referenced by HOME is not writable\n"; exit 1; } unless (defined $ENV{ATLAS_SHELL_TOOLS_HOME}) { print STDERR "Error: ATLAS_SHELL_TOOLS_HOME environment variable is not set\n"; exit 1; } unless (-f "$ENV{ATLAS_SHELL_TOOLS_HOME}/${integrity_file}") { print STDERR "Error: ATLAS_SHELL_TOOLS_HOME environment variable is not a valid installation\n"; exit 1 } } # Create the XDG data directory. Defaults to "$HOME/.local/share" but respects # the XDG_DATA_HOME env variable if set. # Params: none # Return: the newly set data directory sub create_data_directory { # The directory for data storage. Client code must access this variable thru # create_data_directory(), which optionally modifies this variable based on the # XDG_DATA_HOME environment variable. my $data_directory = "$ENV{HOME}/.local/share"; # Respect XDG_DATA_HOME per the XDG Base Directory specification # https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html if (defined $ENV{XDG_DATA_HOME}) { $data_directory = $ENV{XDG_DATA_HOME}; } # Create data subdirectories if necessary $data_directory = File::Spec->catfile($data_directory, 'atlas-shell-tools'); my $full_log4j_path = File::Spec->catfile($data_directory, $ast_log_subsystem::LOG4J_FOLDER); my $full_module_path = File::Spec->catfile($data_directory, $ast_module_subsystem::MODULES_FOLDER); my $full_presets_path = File::Spec->catfile($data_directory, $ast_preset_subsystem::PRESETS_FOLDER); my $default_namespace_path = File::Spec->catfile($data_directory, $ast_preset_subsystem::PRESETS_FOLDER, $ast_preset_subsystem::DEFAULT_NAMESPACE); my $full_repos_path = File::Spec->catfile($data_directory, $ast_repo_subsystem::REPOS_FOLDER); make_path("$data_directory", "$full_module_path", "$full_log4j_path", "$full_presets_path", "$default_namespace_path", "$full_repos_path", { verbose => 0, mode => 0755 }); # reset the log4j file if it is missing my $log4j_file = File::Spec->catfile($data_directory, $ast_log_subsystem::LOG4J_FOLDER, $ast_log_subsystem::LOG4J_FILE); unless (-f $log4j_file) { ast_log_subsystem::reset_log4j($data_directory); } # reset the current namespace file if it is missing my $current_namespace_file = File::Spec->catfile($data_directory, $ast_preset_subsystem::NAMESPACE_PATH); unless (-f $current_namespace_file) { ast_preset_subsystem::reset_namespace($data_directory); } # add a .global folder to all namespaces if it is missing my @namespaces = ast_preset_subsystem::get_namespaces_array($data_directory); foreach my $namespace (@namespaces) { my $global_subfolder = File::Spec->catfile($data_directory, $ast_preset_subsystem::PRESETS_FOLDER, $namespace, $ast_preset_subsystem::GLOBAL_FOLDER); make_path("$global_subfolder", { verbose => 0, mode => 0755 }); } return $data_directory; } # Display the given message and exit. Default behaviour is to use pagination, # but this can be disabled with the "skip_paging" parameter. # Params: # $message: the message text # $skip_paging: a boolean value that determines if the pager should be skipped # Return: none sub display_and_exit { my $message = shift; my $skip_paging = shift; unless (defined $skip_paging) { $skip_paging = 0; } my @pager_command = get_pager(); if ($skip_paging) { print "$message"; } else { # NOTE: there is no easy way to prevent shell interference should the pager # command array contain only one element. open PAGER, "|-", @pager_command or die $!; print PAGER "$message"; close PAGER; } exit 0; } # Print a failure message for getopt failures. # Params: # $program_name: the name of the failing program # $subcommand_name: the optional name of the subcommand # Returns: none sub getopt_failure_and_exit { my $program_name = shift; my $subcommand_name = shift; if (defined $subcommand_name) { print STDERR "Try '${bold_stderr}${program_name} ${subcommand_name} --help${reset_stderr}' for more information.\n"; } else { print STDERR "Try '${bold_stderr}${program_name} --help${reset_stderr}' for more information.\n"; } exit 1; } # Print a command error message. The format is: # "$command: error: $message" # This routine will use colors/formatting if allowed by environment settings. # This routine will place output on stderr. # Params: # $command: the name of the command # $message: the message # Return: none sub error_output { my $command = shift; my $message = shift; my $no_colors = ast_tty::is_no_colors_stderr(); my $red = $no_colors ? "" : ast_tty::ansi_red(); my $bold = $no_colors ? "" : ast_tty::ansi_bold(); my $reset = $no_colors ? "" : ast_tty::ansi_reset(); print STDERR "$command: ${red}${bold}error:${reset} $message\n" } # Print a command warn message. The format is: # "$command: warn: $message" # This routine will use colors/formatting if allowed by environment settings. # This routine will place output on stderr. # Params: # $command: the name of the command # $message: the message # Return: none sub warn_output { my $command = shift; my $message = shift; my $no_colors = ast_tty::is_no_colors_stderr(); my $magenta = $no_colors ? "" : ast_tty::ansi_magenta(); my $bold = $no_colors ? "" : ast_tty::ansi_bold(); my $reset = $no_colors ? "" : ast_tty::ansi_reset(); print STDERR "$command: ${magenta}${bold}warn:${reset} $message\n" } # Prompt the user for input. # Params: # $query: the prompt string # Return: the input sub prompt { my $query = shift; # take a prompt string as argument local $| = 1; # activate autoflush to immediately show the prompt print $query; my $answer = ; if (defined $answer) { chomp($answer); } return $answer; } # Prompt the user for y/n confirmation. # Params: # $query: the prompt string # Return: 1 if user gave y, 0 if user gave n sub prompt_yn { my $query = shift; my $answer = prompt("$query\n[y/N]: "); unless (defined $answer) { return 0; } return 1 if lc($answer) eq 'y'; return 0; } # Get a pager command capable of displaying formatting codes. Checks the value # of the ATLAS_SHELL_TOOLS_PAGER env variable, and uses that instead if it is set. # If that is unset, fall back to PAGER. If that is unset, try a default. # Params: none # Return: the pager command array sub get_pager { my @pager_command = (); if (defined $ENV{ATLAS_SHELL_TOOLS_PAGER} && $ENV{ATLAS_SHELL_TOOLS_PAGER} ne '') { @pager_command = split /\s+/, $ENV{ATLAS_SHELL_TOOLS_PAGER}; } elsif (defined $ENV{PAGER} && $ENV{PAGER} ne '') { @pager_command = split /\s+/, $ENV{PAGER}; } else { push @pager_command, 'less'; # Options (see less(1) for more info): # -c -> clear the screen before displaying # -S -> chop long lines instead of wrapping them # -R -> actually display ANSI "color" control sequences as colors/formatting # -M -> use verbose prompt # -i -> searches ignore case # -s -> squeeze consecutive blank lines # TODO consider -F, -X options here? # https://unix.stackexchange.com/questions/107315/less-quit-if-one-screen-without-no-init push @pager_command, '-cSRMis'; } return @pager_command; } # Get an editor command capable of displaying and editing a text file. Checks # the value of the ATLAS_SHELL_TOOLS_EDITOR env variable, and uses that instead if it points to an # editor. If that is unset, fall back to EDITOR. If that is unset, try a default. # Params: none # Return: the editor command sub get_editor { my @editor_command = (); if (defined $ENV{ATLAS_SHELL_TOOLS_EDITOR} && $ENV{ATLAS_SHELL_TOOLS_EDITOR} ne '') { @editor_command = split /\s+/, $ENV{ATLAS_SHELL_TOOLS_EDITOR}; } elsif (defined $ENV{EDITOR} && $ENV{EDITOR} ne '') { @editor_command = split /\s+/, $ENV{EDITOR}; } else { push @editor_command, 'vim'; # The '+' option tells vim to start with the cursor at the end of the file. # This is generally convenient for most atlas-shell-tools use-cases. push @editor_command, '+'; } return @editor_command; } # Get a man command capable of displaying some desired manpage. # Params: # $skip_paging: optionally disable paging for man # Return: the man command array sub get_man { my @man_command = (); my $skip_paging = shift; push @man_command, 'man'; if ($skip_paging) { push @man_command, '-P'; push @man_command, 'cat'; } return @man_command; } # Check if a given string starts with a given prefix. # Params: # $string: the string to search # $prefix: the prefix to check # Return: 1 if $string starts with $prefix, 0 otherwise sub string_starts_with { my $string = shift; my $prefix = shift; return substr($string, 0, length($prefix)) eq $prefix; } # Check if a given directory is empty. # Params: # $dir: the directory to check # Return: 1 if the directory is empty, 0 otherwise sub is_dir_empty { my ($dir) = @_; opendir my $h, $dir or die "Cannot open directory: '$dir': $!"; while (defined(my $entry = readdir $h)) { return unless $entry =~ /^[.][.]?\z/; } closedir $h; return 1; } # Compute the Levenshtein distance between two strings. # Params: # $string1: the first string # $string2: the second string # Return: the Levenshtein distnace sub levenshtein { my $string1 = shift; my $string2 = shift; # split the strings at each character my @letters1 = split //, $string1; my @letters2 = split //, $string2; # memoization table my @distance; $distance[$_][0] = $_ foreach (0 .. @letters1); $distance[0][$_] = $_ foreach (0 .. @letters2); foreach my $i (1 .. @letters1) { foreach my $j (1 .. @letters2) { my $cost = $letters1[$i - 1] eq $letters2[$j - 1] ? 0 : 1; $distance[$i][$j] = min($distance[$i - 1][$j] + 1, $distance[$i][$j - 1] + 1, $distance[$i - 1][$j - 1] + $cost); } } return $distance[@letters1][@letters2]; } # Read the output of a given command array. # Params: # $command_ref: a reference to an array containing the command args # Return: the output of a given command array sub read_command_output { my $command_ref = shift; my @command = @{$command_ref}; open COMMAND, "-|", @command or die $!; my $output = ''; while () { # Not the most efficient way to do things. # Perhaps some kind of slurp is needed. File::Slurp could work but does # have an outstanding Unicode bug. Need to investigate more. $output = $output . $_; } close COMMAND; return $output; } # Perl modules must return a value. Returning a value perl considers "truthy" # signals that the module loaded successfully. 1; ================================================ FILE: build.gradle ================================================ plugins { id 'checkstyle' id 'groovy' id 'idea' id 'jacoco' id 'java' id 'java-library' id 'maven-publish' id 'signing' id 'com.diffplug.spotless' version '6.25.0' id 'com.google.protobuf' version '0.8.18' id 'org.sonarqube' version '3.3' } apply from: 'dependencies.gradle' apply from: 'gradle/quality.gradle' apply from: 'gradle/protobuf.gradle' apply from: 'gradle/pyatlas.gradle' apply from: 'gradle/deployment.gradle' description = "Atlas Library" sourceCompatibility=11 targetCompatibility=11 repositories { // For geotools maven { url "https://repo.osgeo.org/repository/release/" content { // osgeo removed the jar and added a -norce version excludeVersion("log4j", "log4j", "1.2.17") } } mavenCentral() } configurations { shaded.extendsFrom(implementation) } dependencies { implementation packages.artifact implementation packages.checkstyle api packages.classgraph implementation packages.commons.cli implementation packages.commons.compress implementation packages.commons.csv implementation packages.commons.io implementation packages.commons.lang implementation packages.commons.math implementation packages.commons.text implementation packages.diff_utils api packages.geotools api packages.groovy implementation packages.groovy_json implementation packages.gson api packages.guava api packages.http api packages.jackson.core api packages.jackson.databind api packages.jackson.dataformat api packages.jim_fs api packages.jsonassert api packages.jts // Support JUnit 3/4 tests api packages.junit.junit4 // only API level due to CoreTestRule implementation packages.opencsv api packages.osmosis.core implementation packages.osmosis.hstore implementation packages.osmosis.osmbinary implementation packages.osmosis.pbf implementation packages.osmosis.xml implementation packages.protobuf_java implementation packages.protoc implementation packages.slf4j.api implementation packages.spatial4j testImplementation packages.checkstyle_tests // Google Truth is needed for checkstyle tests (as of checkstyle 9.2.1) testImplementation packages.google_truth testImplementation packages.junit.api testImplementation packages.junit.engine testImplementation packages.junit.junit4 testImplementation packages.junit.params testImplementation packages.junit.vintage checkstyle files("build/libs/${project.name}-${project.version}.jar") checkstyle packages.checkstyle shaded packages.log4j.api shaded packages.log4j.slf4j } task shaded(type: Jar) { archiveBaseName = project.name classifier = 'shaded' from { configurations.shaded.collect { it.isDirectory() ? it : zipTree(it).matching{ exclude { it.path.contains('META-INF') && (it.path.endsWith('.SF') || it.path.endsWith('.DSA') || it.path.endsWith('.RSA')) } } } } with jar zip64 = true } /** * Artifact related items */ task javadocJar(type: Jar) { classifier = 'javadoc' from javadoc } task sourcesJar(type: Jar) { classifier = 'sources' from sourceSets.main.allSource } artifacts { archives javadocJar, sourcesJar } /* * This is to skip the tasks for which there is a skip=true * environment variable */ def skippedTaskNames = System.getenv().findAll { key, value -> key.startsWith("skip") && value.equalsIgnoreCase("true") }.keySet().collect { it.substring(4) } gradle.startParameter.excludedTaskNames += skippedTaskNames idea { project { languageLevel = '1.8' } } /* * Workaround Gradle not liking duplicate files * I suspect that it is due to fake duplicates. * For more information, see https://github.com/gradle/gradle/issues/17236 * (AKA, remove this when that issue is fixed) * * Note: It may be useful to disable this from time to time on Gradle update * to see if either it is fixed or if there is better debugging * information available. */ tasks.each { task -> if (task.hasProperty("duplicatesStrategy")) { task.setProperty("duplicatesStrategy", "EXCLUDE") } } ================================================ FILE: config/checkstyle/arrangement.txt ================================================ interface,public,non_static interface,protected,non_static interface,package_private,non_static interface,private,non_static enum,public,non_static enum,protected,non_static enum,package_private,non_static enum,private,non_static class,public,non_static class,protected,non_static class,package_private,non_static class,private,non_static # Here do not force order on static variables # as they might have dependencies on each other # field,public,static # field,protected,static # field,package_private,static # field,private,static static_initializer_block,,non_static field,public,non_static field,protected,non_static field,package_private,non_static field,private,non_static method,public,static method,protected,static method,package_private,static method,private,static initializer_block,,non_static constructor,public,non_static constructor,protected,non_static constructor,package_private,non_static constructor,private,non_static method,public,non_static method,protected,non_static method,package_private,non_static method,private,non_static ================================================ FILE: config/checkstyle/checkstyle.xml ================================================ ================================================ FILE: config/checkstyle/suppressions.xml ================================================ ================================================ FILE: config/format/code_format.xml ================================================ ================================================ FILE: config/log4j/log4j.properties ================================================ log4j.rootLogger=INFO, stdout # Direct log messages to stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n # Remove unused logging log4j.logger.org.reflections=WARN # Show our logs log4j.logger.org.openstreetmap.atlas=TRACE log4j.logger.org.openstreetmap.atlas.utilities=INFO log4j.logger.org.openstreetmap.atlas.streaming.resource=INFO log4j.logger.org.openstreetmap.atlas.geography.atlas.items.complex.bignode=DEBUG log4j.logger.org.openstreetmap.atlas.geography.atlas.items.TurnRestriction=DEBUG log4j.logger.org.openstreetmap.atlas.geography.atlas.pbf=INFO log4j.logger.org.openstreetmap.atlas.geography.atlas.packed=INFO log4j.logger.org.openstreetmap.atlas.geography.atlas.delta=INFO log4j.logger.org.openstreetmap.atlas.tags=INFO ================================================ FILE: dependencies.gradle ================================================ project.ext.versions = [ checkstyle: '9.3', junit4: '4.13.2', junit5: '5.8.2', jacoco: '0.8.7', slf4j: '1.7.32', log4j: '2.17.1', opencsv: '2.3', gson: '2.9.0', http: '4.5.13', jts: '1.18.2', spatial4j: '0.8', geotools: '26.2', osmosis: '0.48.3', commons_cli: '1.5.0', commons_compress: '1.21', commons_csv: '1.9.0', commons_io: '2.11.0', commons_lang: '3.12.0', commons_math: '3.6.1', commons_text: '1.9', classgraph: '4.8.139', guava: '31.0.1-jre', google_truth: '1.1.3', jsonassert:'1.5.0', jackson_core:'2.10.5', jackson_databind:'2.10.5', jackson_dataformat_yaml:'2.10.5', protobuf_java:'3.19.1', protoc:'3.19.1', artifact:'3.8.4', groovy: '3.0.9', atlas_checkstyle: '6.6.1', diff_utils: '4.0', groovy_json: '3.0.9', jim_fs: '1.2' ] project.ext.packages = [ junit: [ junit4: "junit:junit:${versions.junit4}", api: "org.junit.jupiter:junit-jupiter-api:${versions.junit5}", engine: "org.junit.jupiter:junit-jupiter-engine:${versions.junit5}", params: "org.junit.jupiter:junit-jupiter-params:${versions.junit5}", vintage: "org.junit.vintage:junit-vintage-engine:${versions.junit5}", ], slf4j: [ api: "org.slf4j:slf4j-api:${versions.slf4j}", ], log4j: [ api: "org.apache.logging.log4j:log4j:${versions.log4j}", slf4j: "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}", ], opencsv: "net.sf.opencsv:opencsv:${versions.opencsv}", google_truth: "com.google.truth:truth:${versions.google_truth}", gson: "com.google.code.gson:gson:${versions.gson}", http: "org.apache.httpcomponents:httpclient:${versions.http}", jts: "org.locationtech.jts:jts-core:${versions.jts}", spatial4j: "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}", geotools: "org.geotools:gt-shapefile:${versions.geotools}", osmosis: [ core: "org.openstreetmap.osmosis:osmosis-core:${versions.osmosis}", hstore:"org.openstreetmap.osmosis:osmosis-hstore-jdbc:${versions.osmosis}", osmbinary: "org.openstreetmap.osmosis:osmosis-osm-binary:${versions.osmosis}", pbf: "org.openstreetmap.osmosis:osmosis-pbf:${versions.osmosis}", pg:"org.openstreetmap.osmosis:osmosis-pgsnapshot:${versions.osmosis}", xml: "org.openstreetmap.osmosis:osmosis-xml:${versions.osmosis}", ], commons:[ cli: "commons-cli:commons-cli:${versions.commons_cli}", compress: "org.apache.commons:commons-compress:${versions.commons_compress}", csv : "org.apache.commons:commons-csv:${versions.commons_csv}", io: "commons-io:commons-io:${versions.commons_io}", lang: "org.apache.commons:commons-lang3:${versions.commons_lang}", math: "org.apache.commons:commons-math3:${versions.commons_math}", text: "org.apache.commons:commons-text:${versions.commons_text}" ], classgraph: "io.github.classgraph:classgraph:${versions.classgraph}", guava: "com.google.guava:guava:${versions.guava}", jsonassert: "org.skyscreamer:jsonassert:${versions.jsonassert}", jackson:[ core: "com.fasterxml.jackson.core:jackson-core:${versions.jackson_core}", databind: "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}", dataformat: "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson_dataformat_yaml}" ], protobuf_java: "com.google.protobuf:protobuf-java:${versions.protobuf_java}", protoc: "com.google.protobuf:protoc:${versions.protoc}", artifact: "org.apache.maven:maven-artifact:${versions.artifact}", groovy: "org.codehaus.groovy:groovy:${versions.groovy}", checkstyle: "com.puppycrawl.tools:checkstyle:${versions.checkstyle}", checkstyle_tests: "com.puppycrawl.tools:checkstyle:${versions.checkstyle}:tests", atlas_checkstyle: "org.openstreetmap.atlas:atlas:${versions.atlas_checkstyle}", diff_utils: "io.github.java-diff-utils:java-diff-utils:${versions.diff_utils}", groovy_json: "org.codehaus.groovy:groovy-json:${versions.groovy_json}", jim_fs: "com.google.jimfs:jimfs:${versions.jim_fs}" ] ================================================ FILE: gradle.properties ================================================ group=org.openstreetmap.atlas version=7.0.10-SNAPSHOT maven2_url=https://oss.sonatype.org/service/local/staging/deploy/maven2/ snapshot_url=https://oss.sonatype.org/content/repositories/snapshots/ project_name="OSM Atlas Library" project_description="Library to load OSM data into an Atlas format" project_url=https://github.com/osmlab/atlas project_license_url=https://github.com/osmlab/atlas/blob/main/LICENSE project_license_slug="BSD 3 Clause" project_developer=matthieun project_scm=scm:git:https://github.com/osmlab/atlas.git ================================================ FILE: pyatlas/README.md ================================================ # PyAtlas #### A simplified Atlas API for Python ---- ## Getting Started To get setup in a new project folder, run: $ mkdir newproj && cd newproj $ virtualenv venv --python=python3 $ source venv/bin/activate NOTE: `pyatlas` will automatically install the dependencies it needs, including the Protocol Buffers Python runtime - `protobuf-2.6.1`. Therefore, it is highly recommended that you develop `pyatlas` based projects in a Python virtual environment - you may need to install `virtualenv` if you have not already. (If you want to create a `pyatlas` distribution that does not automatically pull in dependencies, see the next section.) Now that you have your virtual environment set up, you can install `pyatlas` with: (venv) $ pip install pyatlas If pip is unable to find the `pyatlas` package, you may need to build it from source yourself. Check the next section for more info. To test that everything went smoothly, create a file `helloatlas.py` with the following code: ```python from pyatlas import pyatlas_globalfunc pyatlas_globalfunc.hello_atlas() ``` Now run: (venv) $ python helloatlas.py If you see: Hello Atlas! then you're good to go! ---- ## Building the `pyatlas` module To build the `pyatlas` module from source, run: $ git clone https://github.com/osmlab/atlas.git $ cd atlas $ ./gradlew cleanPyatlas buildPyatlas This will generate a wheel file at `pyatlas/dist`. You can now install this with `pip` like $ cd /path/to/project/that/uses/pyatlas $ virtualenv venv --python=python2.7 $ source venv/bin/activate $ pip install /path/to/atlas/pyatlas/dist/pyatlas-VERSION.whl Again, it is recommended that you do this in the desired virtual environment. If you want to build a `pyatlas` wheel file that does not automatically pull dependencies, open up `setup.py` and remove the lines that say install_requires=[ . . . ], Then re-run the `./gradlew cleanPyatlas buildPyatlas` command from above and reinstall using `pip`. Note that you will now need to manage the required dependencies manually. ### Note on the formatter `pyatlas` uses the `yapf` formatting library to check for code format issues when building. If you are running into issues after modifying `pyatlas`, try running ./gradlew applyFormatPyatlas Now `pyatlas` should make it past the `CHECK` format step! Note there is an issue that causes the formatter to goof if a source file does not end with a newline (\n) character. If the `CHECK` format step is consistently failing after repeated `APPLY` steps, and you are seeing a message like the following: atlas.py: found issue, reformatting... with no formatter diff being displayed, check to make sure that the file has an ending newline. ---- ## Documentation `pyatlas` documentation is automatically generated using the `pydoc` tool and stored in the `doc` folder. To build the documentation, run the gradle build command: $ ./gradlew cleanPyatlas buildPyatlas This will generate HTML files detailing the functions and classes available in each module. ---- ## Some sample use cases `pyatlas` is a highly capable subset of the API provided by the Java `Atlas`. Here are some examples to get you started. Note that all of these examples were ran using the `test.atlas` provided in the `resources` folder, and assume that you have an atlas variable defined like: ```python from pyatlas.atlas import Atlas atlas = Atlas('/path/to/atlas/pyatlas/resources/test.atlas') ``` #### Getting features and metadata You can get filtered iterables over an `Atlas`'s features using the methods provided in the `Atlas` class. ```python # print all Nodes for node in atlas.nodes(): print node # print all Edges that have 'key1' as a tag key for edge in atlas.edges(predicate=lambda e: 'key1' in e.get_tags().keys()): print edge ``` You can also get a feature with a specific identifier like: ```python # print the Relation with Atlas ID 2 print atlas.relation(2) ``` Metadata about the `Atlas` is also available. For a quick sample, try something like: ```python metadata = atlas.metadata() print metadata.number_of_points print metadata.country ``` Check out `doc/atlas.html` and `doc/atlas_metadata.html` for more information. #### Operating on features The `Atlas` features themselves support a set of operations defined in their respective classes. Here is a quick example: ```python # print the tag dict for Point with Atlas ID 3 print atlas.point(3).get_tags() # print all Relations of which the Node with Atlas ID 1 is a member for relation in atlas.node(1).relations(): print relation # print all the members of Relation with Atlas ID 1 for member in atlas.relation(1).get_members(): # print the RelationMember object print member # print the actual AtlasEntity contained in the RelationMember print member.get_entity() ``` `Node`s and `Edge`s, in particular, support traversal through their connectivity API. Here are just the basics of what you can do with the connectivity interface: ```python # print Edges connected to Node with ID 3 for edge in atlas.node(3).in_edges(): print edge for edge in atlas.node(3).out_edges(): print edge for edge in atlas.node(3).connected_edges(): print edge # print the start and end Nodes of Edge 1 print atlas.edge(1).start() print atlas.edge(1).end() ``` Many more methods are provided. See the classes in `doc/atlas_entities.html` for more information. #### Geometry `pyatlas` features some really simple geometry primitives for working with locations and shapes on the surface of the Earth. Here is a simple example that uses these primitives: ```python from pyatlas import geometry from pyatlas.geometry import Location, PolyLine, Polygon, Rectangle # Location constructor (lat/lon ordering) uses dm7 by default, see Location docs for info on dm7 loc1 = Location(385000000, -1160200000) # create the same Location but with degree values instead (lat/lon ordering) loc2 = geometry.location_with_degrees(38.5, -116.02) print loc1.get_latitude_deg() print loc2.get_latitude() # create a new PolyLine with two shape points polyline1 = PolyLine([Location(385000000, -1160200000), Location(395000000, -116300000)]) for loc in polyline1.locations(): print loc print polyline1.bounds() # create a new Polygon with specified vertices polygon1 = Polygon([geometry.location_with_degrees(0, 0), geometry.location_with_degrees(10, 0), geometry.location_with_degrees(5, 10)]) print polygon1 # print the vertices, will print the first again at the end to simulate closedness for loc in polygon1.closed_loop(): print loc print polygon1.bounds() # will print True, since the point lies inside the triangle print polygon1.fully_geometrically_encloses_location(geometry.location_with_degrees(5, 5)) # create a new Rectangle with given lower left and upper right corners rect = Rectangle(geometry.location_with_degrees(0, 0), geometry.location_with_degrees(20, 20)) print rect # this Rectangle intersects (overlaps at any point) polygon1 print rect.intersects(polygon1) ``` See the classes in `doc/geometry.html` for more information. #### Spatial queries `pyatlas` supports some simple spatial queries over its feature space. The queries use the geometry primitives provided by the `geometry` module, but convert to [Shapely](https://github.com/Toblerity/Shapely) primitives under the hood to make queries into a native [libgeos-backed](https://github.com/OSGeo/geos) R-tree. Below are examples for a few of the spatial queries the `Atlas` supports: ```python from pyatlas import geometry from pyatlas.geometry import Rectangle # print all Points intersecting a given Polygon that also have "key1" as a tag key lower_left = geometry.location_with_degrees(37, -118.02) upper_right = geometry.location_with_degrees(39, -118) for point in atlas.points_within(Rectangle(lower_left, upper_right), predicate=lambda e: 'key1' in e.get_tags().keys()): print point # print all Relations with at least one member intersecting a given Polygon lower_left = geometry.location_with_degrees(37.999, -118.001) upper_right = geometry.location_with_degrees(38.001, -117.999) for relation in atlas.relations_with_entities_intersecting(Rectangle(lower_left, upper_right)): print relation # print all Edges that intersect a given Polygon lower_left = geometry.location_with_degrees(38, -120) upper_right = geometry.location_with_degrees(40, -117) for edge in atlas.edges_intersecting(Rectangle(lower_left, upper_right)): print edge ``` See `doc/atlas.html` for more information on the available spatial queries. ================================================ FILE: pyatlas/clean.sh ================================================ #!/usr/bin/env bash # general case script abort if a command fails # this can be overridden with a custom error message using '|| err_shutdown' set -e set -o pipefail ### define utility functions ### ################################ err_shutdown() { echo "clean.sh: ERROR: $1" deactivate exit 1 } ### check to prevent users from running this script directly ### ################################################################ if [ "$1" != "ranFromGradle" ]; then err_shutdown "this script should be run using the atlas gradle task 'cleanPyatlas'" fi ### set up variables to store directory names ### ################################################# gradle_project_root_dir="$(pwd)" pyatlas_dir="pyatlas" pyatlas_srcdir="pyatlas" pyatlas_testdir="unit_tests" pyatlas_docdir="doc" pyatlas_root_dir="$gradle_project_root_dir/$pyatlas_dir" venv_path="$pyatlas_root_dir/__pyatlas_venv__" protoc_path="$pyatlas_root_dir/protoc" ### abort the script if the pyatlas source folder is not present ### #################################################################### if [ ! -d "$pyatlas_root_dir/$pyatlas_srcdir" ]; then err_shutdown "pyatlas source folder not found" fi ### clean up the build artifacts ### #################################### echo "Cleaning build artifacts if present..." rm -rf "$pyatlas_root_dir/build" rm -rf "$pyatlas_root_dir/dist" rm -rf "$pyatlas_root_dir/pyatlas.egg-info" rm -f "$pyatlas_root_dir/LICENSE" rm -rf "$venv_path" rm -f "$protoc_path" # use 'find' to handle case where filenames contain spaces find "$pyatlas_root_dir/$pyatlas_srcdir/autogen" -type f -name "*_pb2.py" -delete find "$pyatlas_root_dir/$pyatlas_srcdir" -type f -name "*.pyc" -delete find "$pyatlas_root_dir/$pyatlas_srcdir/autogen" -type f -name "*.pyc" -delete find "$pyatlas_root_dir/$pyatlas_testdir" -type f -name "*.pyc" -delete find "$pyatlas_root_dir/$pyatlas_docdir" -type f -name "*.html" -delete ================================================ FILE: pyatlas/doc/how_to_get_the_docs.txt ================================================ Use the 'buildPyatlas' gradle target to generate the docs! From the root atlas project directory run: $ ./gradlew buildPyatlas ================================================ FILE: pyatlas/format.sh ================================================ #!/usr/bin/env bash # general case script abort if a command fails # this can be overridden with a custom error message using '|| err_shutdown' set -e set -o pipefail ### define utility functions ### ################################ err_shutdown() { echo "format.sh: ERROR: $1" deactivate exit 1 } ### check to prevent users from running this script directly ### ################################################################ if [ "$1" != "ranFromGradle" ]; then err_shutdown "this script should be run using the atlas gradle task 'formatPyatlas'" fi ### get CHECK or APPLY mode ### ############################### format_mode=$2 ### set up variables to store directory names ### ################################################# gradle_project_root_dir="$(pwd)" pyatlas_dir="pyatlas" pyatlas_srcdir="pyatlas" pyatlas_testdir="unit_tests" pyatlas_root_dir="$gradle_project_root_dir/$pyatlas_dir" venv_path="$pyatlas_root_dir/__pyatlas_venv__" pyatlas_format_script="yapf_format.py" ### abort the script if the pyatlas source folder is not present ### #################################################################### if [ ! -d "$pyatlas_root_dir/$pyatlas_srcdir" ]; then err_shutdown "pyatlas source folder not found" fi ### format the module source code ### ##################################### # start the venv if [ ! -d "$venv_path" ]; then err_shutdown "missing $venv_path" fi # shellcheck source=/dev/null source "$venv_path/bin/activate" # enter the pyatlas project directory so the formatting script will work pushd "$pyatlas_root_dir" pip install yapf==0.22.0 if ! python "$pyatlas_format_script" "$pyatlas_srcdir" "$format_mode"; then err_shutdown "CHECK format step failed: run './gradlew applyFormatPyatlas'" fi if ! python "$pyatlas_format_script" "$pyatlas_testdir" "$format_mode"; then err_shutdown "CHECK format step failed: run './gradlew applyFormatPyatlas'" fi # get back to gradle project directory popd # shutdown the venv deactivate ================================================ FILE: pyatlas/package.sh ================================================ #!/usr/bin/env bash # THIS ENTIRE SCRIPT IS A MASSIVE HACK # THIS SHOULD REALLY BE DONE WITH GRADLE # general case script abort if a command fails # this can be overridden with a custom error message using '|| err_shutdown' set -e set -o pipefail ### define utility functions ### ################################ err_shutdown() { echo "package.sh: ERROR: $1" deactivate exit 1 } ### check to prevent users from running this script directly ### ################################################################ if [ "$1" != "ranFromGradle" ]; then err_shutdown "this script should be run using the atlas gradle task 'packagePyatlas'" fi ### set up variables to store directory names ### ################################################# gradle_project_root_dir="$(pwd)" pyatlas_dir="pyatlas" pyatlas_srcdir="pyatlas" doc_dir="doc" pyatlas_root_dir="$gradle_project_root_dir/$pyatlas_dir" venv_path="$pyatlas_root_dir/__pyatlas_venv__" protofiles_dir="$gradle_project_root_dir/src/main/proto" ### abort the script if the pyatlas source folder is not present ### #################################################################### if [ ! -d "$pyatlas_root_dir/$pyatlas_srcdir" ]; then err_shutdown "pyatlas source folder not found" fi ### determine if wget is installed ### ###################################### if command -v wget ; then wget_command="$(command -v wget)" else err_shutdown "'command -v wget' returned non-zero exit status: install wget to run this script" fi ### download protoc and compile the atlas proto files into python ### ##################################################################### echo "Preparing protoc..." # hack to grab the protoc version from dependencies.gradle #protoc_version=$(grep 'protoc' "$gradle_project_root_dir/dependencies.gradle" | awk -F':' '{print $2; exit}' | tr -d "'" | tr -d ',') # TODO HACK FIXME hardcoding a proto3 version until we can upgrade atlas, proto2 is not compat with python3 protoc_version='3.11.1' protoc_path="$pyatlas_root_dir/protoc" # detemine what platform we are on if [ ! -f "$protoc_path" ]; then if [ "$(uname)" == "Darwin" ]; then download_link="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${protoc_version}/protoc-${protoc_version}-osx-x86_64.exe" "$wget_command" "$download_link" -O "$protoc_path" || err_shutdown "wget of '$download_link' failed" elif [ "$(uname)" == "Linux" ]; then download_link="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${protoc_version}/protoc-${protoc_version}-linux-x86_64.exe" "$wget_command" "$download_link" -O "$protoc_path" || err_shutdown "wget of '$download_link' failed" else err_shutdown "unrecognized platform $(uname)" fi fi chmod 700 "$protoc_path" # complicated mess to handle case where a proto filename has a space # basically, 'find' outputs each file separated by a NUL terminator # read -r -d '' reads raw input delimited by NUL characters while IFS= read -r -d '' protofile do "$protoc_path" "$protofile" --proto_path="$protofiles_dir" --python_out="$pyatlas_root_dir/$pyatlas_srcdir/autogen" || err_shutdown "protoc invocation failed" done < <(find "$protofiles_dir" -type f -name "*.proto" -print0) # TODO HACK FIXME this is to fix badly generated import statements by proto3 library # See this solution: https://github.com/protocolbuffers/protobuf/issues/4546#issuecomment-384252863 # FIXME this will fail if source file has a space pushd "$pyatlas_root_dir/$pyatlas_srcdir/autogen" if [ "$(uname)" == "Darwin" ]; then sed -i '' 's/^\(import.*pb2\)/from . \1/g' ./*.py elif [ "$(uname)" == "Linux" ]; then sed --in-place="" "s/^\(import.*pb2\)/from . \1/g" ./*.py else err_shutdown "unrecognized platform $(uname)" fi popd ### build the module and documentation ### ########################################## # start the venv if [ ! -d "$venv_path" ]; then err_shutdown "missing $venv_path" fi # shellcheck source=/dev/null source "$venv_path/bin/activate" # copy the LICENSE to the pyatlas folder cp "$gradle_project_root_dir/LICENSE" "$pyatlas_root_dir" # grab the build version from gradle.properties and inject it into setup.py # remove the -SNAPSHOT text if present atlas_version=$(grep "version=" "$gradle_project_root_dir/gradle.properties" | cut -f2 -d "=" | sed 's/-SNAPSHOT//g') # GNU and BSD sed have different "in-place" flag syntax if [ "$(uname)" == "Darwin" ]; then sed -i "" "s/version=.*/version=\"$atlas_version\",/" "$pyatlas_root_dir/setup.py" elif [ "$(uname)" == "Linux" ]; then sed --in-place="" "s/version=.*/version=\"$atlas_version\",/" "$pyatlas_root_dir/setup.py" else err_shutdown "unrecognized platform $(uname)" fi # enter the pyatlas project directory so module metadata is generated correctly pushd "$pyatlas_root_dir" echo "Building and packaging pyatlas module..." python "setup.py" sdist -d "dist" bdist_wheel -d "dist" # self-install and create the docs pip install -e . # hack to make pydoc work export PYTHONPATH="$PYTHONPATH:$pyatlas_root_dir/$pyatlas_srcdir:$pyatlas_root_dir/$pyatlas_srcdir/autogen" # FIXME this will fail if source file has a space pydoc -w "$pyatlas_srcdir"/*.py mv ./*.html "$doc_dir" # this would be the correct way, but for some reason the 'find exec' fails on atlas.py #find "$pyatlas_srcdir"/*.py -exec pydoc -w {} \; #find "$pyatlas_root_dir/"*.html -exec mv {} "$doc_dir" \; # get back to gradle project directory popd # reset version field in setup.py # GNU and BSD sed have different "in-place" flag syntax if [ "$(uname)" == "Darwin" ]; then sed -i "" "s/version=.*/version=/" "$pyatlas_root_dir/setup.py" elif [ "$(uname)" == "Linux" ]; then sed --in-place="" "s/version=.*/version=/" "$pyatlas_root_dir/setup.py" else err_shutdown "unrecognized platform $(uname)" fi # shutdown the venv deactivate ================================================ FILE: pyatlas/pyatlas/__init__.py ================================================ ================================================ FILE: pyatlas/pyatlas/atlas.py ================================================ """ This module defines the Atlas and the various helper types it needs to operate. Atlas is a representation of an OpenStreetMap region in memory. It is a navigable collection of unidirectional Edges and Nodes. It is designed to be close to the OpenStreetMap model. It also contains a collection of non-navigable geolocated items that can be Points, Lines or Areas. All can be members of Relations. """ import zipfile import pyatlas.autogen.ProtoAtlasMetaData_pb2 import pyatlas.autogen.ProtoLongArray_pb2 import pyatlas.autogen.ProtoLongArrayOfArrays_pb2 import pyatlas.autogen.ProtoIntegerStringDictionary_pb2 import pyatlas.autogen.ProtoPackedTagStore_pb2 import pyatlas.autogen.ProtoLongToLongMap_pb2 import pyatlas.autogen.ProtoLongToLongMultiMap_pb2 import pyatlas.autogen.ProtoPolyLineArray_pb2 import pyatlas.autogen.ProtoPolygonArray_pb2 import pyatlas.autogen.ProtoByteArrayOfArrays_pb2 import pyatlas.autogen.ProtoIntegerArrayOfArrays_pb2 import pyatlas.atlas_entities import pyatlas.geometry import pyatlas.atlas_metadata import pyatlas.spatial_index class Atlas(object): """ The Atlas - current implementation is *not* threadsafe. """ def __init__(self, atlas_file_path, lazy_loading=True): """ Create a new Atlas backed by the atlas file at the provided path (string). Disabling lazy loading will force the full Atlas into memory. This will incur a significant perfomance penalty at creation time, but will make subsequent queries as fast as possible. """ self.serializer = _AtlasSerializer(atlas_file_path, self) self.lazy_loading = lazy_loading # --- PackedAtlas fields --- # The field names match up with the name of their corresponding ZIP entry. # These ZIP entry names come from from the PackedAtlas Java implementation. self.metaData = None self.dictionary = None self.pointIdentifiers = None self.pointIdentifierToPointArrayIndex = None self.pointLocations = None self.pointTags = None self.pointIndexToRelationIndices = None self.lineIdentifiers = None self.lineIdentifierToLineArrayIndex = None self.linePolyLines = None self.lineTags = None self.lineIndexToRelationIndices = None self.areaIdentifiers = None self.areaIdentifierToAreaArrayIndex = None self.areaPolygons = None self.areaTags = None self.areaIndexToRelationIndices = None self.nodeIdentifiers = None self.nodeIdentifierToNodeArrayIndex = None self.nodeLocations = None self.nodeTags = None self.nodeInEdgesIndices = None self.nodeOutEdgesIndices = None self.nodeIndexToRelationIndices = None self.edgeIdentifiers = None self.edgeIdentifierToEdgeArrayIndex = None self.edgeStartNodeIndex = None self.edgeEndNodeIndex = None self.edgePolyLines = None self.edgeTags = None self.edgeIndexToRelationIndices = None self.relationIdentifiers = None self.relationIdentifierToRelationArrayIndex = None self.relationMemberTypes = None self.relationMemberIndices = None self.relationMemberRoles = None self.relationTags = None self.relationIndexToRelationIndices = None # --- spatial indices --- self.point_spatial_index = None self.line_spatial_index = None self.area_spatial_index = None self.node_spatial_index = None self.edge_spatial_index = None self.relation_spatial_index = None if not self.lazy_loading: self.load_all_fields() def metadata(self): """ Get the metadata associated with this Atlas. See the AtlasMetaData class for more information on the metadata format. """ if self.metaData is None: self.serializer._load_field(self.serializer._FIELD_METADATA) return self.metaData def points(self, predicate=lambda p: True): """ Get a generator for Points in this Atlas. Can optionally also accept a predicate to filter the generated Points. """ for i, element in enumerate(self._get_pointIdentifiers().elements): point = pyatlas.atlas_entities.Point(self, i) if predicate(point): yield point def point(self, identifier): """ Get a Point with a given Atlas identifier. Returns None if there is no Point with the given identifier. """ identifier_to_index = self._get_pointIdentifierToPointArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Point(self, identifier_to_index[identifier]) return None def lines(self, predicate=lambda l: True): """ Get a generator for Lines in this Atlas. Can optionally also accept a predicate to filter the generated Lines. """ for i, element in enumerate(self._get_lineIdentifiers().elements): line = pyatlas.atlas_entities.Line(self, i) if predicate(line): yield line def line(self, identifier): """ Get a Line with a given Atlas identifier. Returns None if there is no Line with the given identifier. """ identifier_to_index = self._get_lineIdentifierToLineArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Line(self, identifier_to_index[identifier]) return None def areas(self, predicate=lambda a: True): """ Get a generator for Areas in this Atlas. Can optionally also accept a predicate to filter the generated Areas. """ for i, element in enumerate(self._get_areaIdentifiers().elements): area = pyatlas.atlas_entities.Area(self, i) if predicate(area): yield area def area(self, identifier): """ Get an Area with a given Atlas identifier. Returns None if there is no Area with the given identifier. """ identifier_to_index = self._get_areaIdentifierToAreaArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Area(self, identifier_to_index[identifier]) return None def nodes(self, predicate=lambda n: True): """ Get a generator for Nodes in this Atlas. Can optionally also accept a predicate to filter the generated Nodes. """ for i, element in enumerate(self._get_nodeIdentifiers().elements): node = pyatlas.atlas_entities.Node(self, i) if predicate(node): yield node def node(self, identifier): """ Get a Node with a given Atlas identifier. Returns None if there is no Node with the given identifier. """ identifier_to_index = self._get_nodeIdentifierToNodeArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Node(self, identifier_to_index[identifier]) return None def edges(self, predicate=lambda e: True): """ Get a generator for Edges in this Atlas. Can optionally also accept a predicate to filter the generated Edges. """ for i, element in enumerate(self._get_edgeIdentifiers().elements): edge = pyatlas.atlas_entities.Edge(self, i) if predicate(edge): yield edge def edge(self, identifier): """ Get an Edge with a given Atlas identifier. Returns None if there is no Edge with the given identifier. """ identifier_to_index = self._get_edgeIdentifierToEdgeArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Edge(self, identifier_to_index[identifier]) return None def relations(self, predicate=lambda r: True): """ Get a generator for Relations in this Atlas. Can optionally also accept a predicate to filter the generated Relations. """ for i, element in enumerate(self._get_relationIdentifiers().elements): relation = pyatlas.atlas_entities.Relation(self, i) if predicate(relation): yield relation def relation(self, identifier): """ Get a Relation with a given Atlas identifier. Returns None if there is no Relation with the given identifier. """ identifier_to_index = self._get_relationIdentifierToRelationArrayIndex() if identifier in identifier_to_index: return pyatlas.atlas_entities.Relation(self, identifier_to_index[identifier]) return None def entities(self, predicate=lambda e: True): """ Get a generator for all AtlasEntities in this Atlas. Can optionally also accept a predicate to filter the generated entities. """ for point in self.points(): if predicate(point): yield point for line in self.lines(): if predicate(line): yield line for area in self.areas(): if predicate(area): yield area for node in self.nodes(): if predicate(node): yield node for edge in self.edges(): if predicate(edge): yield edge for relation in self.relations(): if predicate(relation): yield relation def entity(self, identifier, entity_type): """ Get an AtlasEntity with a given Atlas identifier and EntityType. Returns None if there is no entity with the given identifier and type. """ if entity_type == pyatlas.atlas_entities.EntityType.POINT: return self.point(identifier) elif entity_type == pyatlas.atlas_entities.EntityType.LINE: return self.line(identifier) elif entity_type == pyatlas.atlas_entities.EntityType.AREA: return self.area(identifier) elif entity_type == pyatlas.atlas_entities.EntityType.NODE: return self.node(identifier) elif entity_type == pyatlas.atlas_entities.EntityType.EDGE: return self.edge(identifier) elif entity_type == pyatlas.atlas_entities.EntityType.RELATION: return self.relation(identifier) else: raise ValueError('invalid EntityType value ' + str(entity_type)) def points_at(self, location, predicate=lambda p: True): """ Get a frozenset of all Points at some Location. Can optionally accept a predicate to further filter the Points. """ points_list = self._get_point_spatial_index().get(location.bounds(), predicate=predicate) return points_list def points_within(self, polygon, predicate=lambda p: True): """ Get a frozenset of all Points within some polygon. Can optionally accept a predicate to further filter the Points. """ points = self._get_point_spatial_index().get(polygon.bounds(), predicate=predicate) points_set = set() for point in points: if polygon.fully_geometrically_encloses_location(point.as_location()): points_set.add(point) return frozenset(points_set) def lines_containing(self, location, predicate=lambda l: True): """ Get a frozenset of all Lines containing some Location. Can optionally accept a predicate to further filter the Lines. """ lines = self._get_line_spatial_index().get(location.bounds(), predicate=predicate) lines_set = set() for line in lines: polyline = line.as_polyline() if location.bounds().overlaps_polyline(polyline): lines_set.add(line) return frozenset(lines_set) def lines_intersecting_polyline(self, polyline, predicate=lambda l: True): """ Get a frozenset of all Lines within or intersecting some PolyLine. Can optionally accept a predicate to further filter the Lines. """ lines = self._get_line_spatial_index().get(polyline.bounds(), predicate=predicate) lines_set = set() for line in lines: polyline = line.as_polyline() if polyline.intersects_polyline(polyline): lines_set.add(line) return frozenset(lines_set) def lines_intersecting(self, polygon, predicate=lambda l: True): """ Get a frozenset of all Lines within or intersecting some Polygon. Can optionally accept a predicate to further filter the Lines. """ lines = self._get_line_spatial_index().get(polygon.bounds(), predicate=predicate) lines_set = set() for line in lines: polyline = line.as_polyline() if polygon.overlaps_polyline(polyline): lines_set.add(line) return frozenset(lines_set) def areas_covering(self, location, predicate=lambda a: True): """ Get a frozenset of all Areas covering some Location. Can optionally accept a predicate to further filter the Areas. """ areas = self._get_area_spatial_index().get(location.bounds(), predicate=predicate) areas_set = set() for area in areas: if area.as_polygon().fully_geometrically_encloses_location(location): areas_set.add(area) return frozenset(areas_set) def areas_intersecting(self, polygon, predicate=lambda a: True): """ Get a frozenset of all Areas within or intersecting some Polygon. Can optionally accept a predicate to further filter the Areas. """ areas = self._get_area_spatial_index().get(polygon.bounds(), predicate=predicate) areas_set = set() for area in areas: if polygon.intersects(area.as_polygon()): areas_set.add(area) return frozenset(areas_set) def nodes_at(self, location, predicate=lambda n: True): """ Get a frozenset of all Nodes at some Location. Can optionally accept a predicate to further filter the Nodes. """ nodes_list = self._get_node_spatial_index().get(location.bounds(), predicate=predicate) return nodes_list def nodes_within(self, polygon, predicate=lambda n: True): """ Get a frozenset of all Nodes within some Polygon. Can optionally accept a predicate to further filter the Nodes. """ nodes = self._get_node_spatial_index().get(polygon.bounds(), predicate=predicate) nodes_set = set() for node in nodes: if polygon.fully_geometrically_encloses_location(node.as_location()): nodes_set.add(node) return frozenset(nodes_set) def edges_containing(self, location, predicate=lambda e: True): """ Get a frozenset of all Edges containing some Location. Can optionally accept a predicate to further filter the Edges. """ edges = self._get_edge_spatial_index().get(location.bounds(), predicate=predicate) edges_set = set() for edge in edges: polyline = edge.as_polyline() if location.bounds().overlaps_polyline(polyline): edges_set.add(edge) return frozenset(edges_set) def edges_intersecting_polyline(self, polyline, predicate=lambda e: True): """ Get a frozenset of all Edges within or intersecting some PolyLine. Can optionally accept a predicate to further filter the Edges. """ edges = self._get_edge_spatial_index().get(polyline.bounds(), predicate=predicate) edges_set = set() for edge in edges: polyline = edge.as_polyline() if polyline.intersects_polyline(polyline): edges_set.add(edge) return frozenset(edges_set) def edges_intersecting(self, polygon, predicate=lambda e: True): """ Get a frozenset of all Edges within or intersecting some Polygon. Can optionally accept a predicate to further filter the Edges. """ edges = self._get_edge_spatial_index().get(polygon.bounds(), predicate=predicate) edges_set = set() for edge in edges: polyline = edge.as_polyline() if polygon.overlaps_polyline(polyline): edges_set.add(edge) return frozenset(edges_set) def relations_with_entities_intersecting(self, polygon, predicate=lambda r: True): """ Return a frozenset of Relations which have at least one feature intersecting some Polygon. Can optionally accept a predicate to further filter the Relations. """ relations = self._get_relation_spatial_index().get(polygon.bounds(), predicate=predicate) relations_set = set() for relation in relations: if relation.intersects(polygon): relations_set.add(relation) return frozenset(relations_set) def load_all_fields(self): """ Force this Atlas to load all its fields from its backing store. """ self.serializer._load_all_fields() self._get_point_spatial_index() self._get_line_spatial_index() self._get_area_spatial_index() self._get_node_spatial_index() self._get_edge_spatial_index() self._get_relation_spatial_index() def number_of_points(self): """ Get the number of Points in this Atlas """ return len(self._get_pointIdentifiers().elements) def number_of_lines(self): """ Get the number of Lines in this Atlas """ return len(self._get_lineIdentifiers().elements) def number_of_areas(self): """ Get the number of Areas in this Atlas """ return len(self._get_areaIdentifiers().elements) def number_of_nodes(self): """ Get the number of Nodes in this Atlas """ return len(self._get_nodeIdentifiers().elements) def number_of_edges(self): """ Get the number of Edges in this Atlas """ return len(self._get_edgeIdentifiers().elements) def number_of_relations(self): """ Get the number of Relations in this Atlas """ return len(self._get_relationIdentifiers().elements) # --- PackedAtlas field loading functions --- def _get_dictionary(self): if self.dictionary is None: self.serializer._load_field(self.serializer._FIELD_DICTIONARY) return self.dictionary def _get_pointIdentifiers(self): if self.pointIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_POINT_IDENTIFIERS) return self.pointIdentifiers def _get_pointIdentifierToPointArrayIndex(self): if self.pointIdentifierToPointArrayIndex is None: self.serializer._load_field( self.serializer._FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX) return self.pointIdentifierToPointArrayIndex def _get_pointLocations(self): if self.pointLocations is None: self.serializer._load_field(self.serializer._FIELD_POINT_LOCATIONS) return self.pointLocations def _get_pointTags(self): if self.pointTags is None: self.serializer._load_field(self.serializer._FIELD_POINT_TAGS) self.pointTags.set_dictionary(self._get_dictionary()) return self.pointTags def _get_pointIndexToRelationIndices(self): if self.pointIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_POINT_INDEX_TO_RELATION_INDICES) return self.pointIndexToRelationIndices def _get_lineIdentifiers(self): if self.lineIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_LINE_IDENTIFIERS) return self.lineIdentifiers def _get_lineIdentifierToLineArrayIndex(self): if self.lineIdentifierToLineArrayIndex is None: self.serializer._load_field(self.serializer._FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX) return self.lineIdentifierToLineArrayIndex def _get_linePolyLines(self): if self.linePolyLines is None: self.serializer._load_field(self.serializer._FIELD_LINE_POLYLINES) return self.linePolyLines def _get_lineTags(self): if self.lineTags is None: self.serializer._load_field(self.serializer._FIELD_LINE_TAGS) self.lineTags.set_dictionary(self._get_dictionary()) return self.lineTags def _get_lineIndexToRelationIndices(self): if self.lineIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_LINE_INDEX_TO_RELATION_INDICES) return self.lineIndexToRelationIndices def _get_areaIdentifiers(self): if self.areaIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_AREA_IDENTIFIERS) return self.areaIdentifiers def _get_areaIdentifierToAreaArrayIndex(self): if self.areaIdentifierToAreaArrayIndex is None: self.serializer._load_field(self.serializer._FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX) return self.areaIdentifierToAreaArrayIndex def _get_areaPolygons(self): if self.areaPolygons is None: self.serializer._load_field(self.serializer._FIELD_AREA_POLYGONS) return self.areaPolygons def _get_areaTags(self): if self.areaTags is None: self.serializer._load_field(self.serializer._FIELD_AREA_TAGS) self.areaTags.set_dictionary(self._get_dictionary()) return self.areaTags def _get_areaIndexToRelationIndices(self): if self.areaIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_AREA_INDEX_TO_RELATION_INDICES) return self.areaIndexToRelationIndices def _get_nodeIdentifiers(self): if self.nodeIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_NODE_IDENTIFIERS) return self.nodeIdentifiers def _get_nodeIdentifierToNodeArrayIndex(self): if self.nodeIdentifierToNodeArrayIndex is None: self.serializer._load_field(self.serializer._FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX) return self.nodeIdentifierToNodeArrayIndex def _get_nodeLocations(self): if self.nodeLocations is None: self.serializer._load_field(self.serializer._FIELD_NODE_LOCATIONS) return self.nodeLocations def _get_nodeTags(self): if self.nodeTags is None: self.serializer._load_field(self.serializer._FIELD_NODE_TAGS) self.nodeTags.set_dictionary(self._get_dictionary()) return self.nodeTags def _get_nodeInEdgesIndices(self): if self.nodeInEdgesIndices is None: self.serializer._load_field(self.serializer._FIELD_NODE_IN_EDGES_INDICES) return self.nodeInEdgesIndices def _get_nodeOutEdgesIndices(self): if self.nodeOutEdgesIndices is None: self.serializer._load_field(self.serializer._FIELD_NODE_OUT_EDGES_INDICES) return self.nodeOutEdgesIndices def _get_nodeIndexToRelationIndices(self): if self.nodeIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_NODE_INDEX_TO_RELATION_INDICES) return self.nodeIndexToRelationIndices def _get_edgeIdentifiers(self): if self.edgeIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_EDGE_IDENTIFIERS) return self.edgeIdentifiers def _get_edgeIdentifierToEdgeArrayIndex(self): if self.edgeIdentifierToEdgeArrayIndex is None: self.serializer._load_field(self.serializer._FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX) return self.edgeIdentifierToEdgeArrayIndex def _get_edgeStartNodeIndex(self): if self.edgeStartNodeIndex is None: self.serializer._load_field(self.serializer._FIELD_EDGE_START_NODE_INDEX) return self.edgeStartNodeIndex def _get_edgeEndNodeIndex(self): if self.edgeEndNodeIndex is None: self.serializer._load_field(self.serializer._FIELD_EDGE_END_NODE_INDEX) return self.edgeEndNodeIndex def _get_edgePolyLines(self): if self.edgePolyLines is None: self.serializer._load_field(self.serializer._FIELD_EDGE_POLYLINES) return self.edgePolyLines def _get_edgeTags(self): if self.edgeTags is None: self.serializer._load_field(self.serializer._FIELD_EDGE_TAGS) self.edgeTags.set_dictionary(self._get_dictionary()) return self.edgeTags def _get_edgeIndexToRelationIndices(self): if self.edgeIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_EDGE_INDEX_TO_RELATION_INDICES) return self.edgeIndexToRelationIndices def _get_relationIdentifiers(self): if self.relationIdentifiers is None: self.serializer._load_field(self.serializer._FIELD_RELATION_IDENTIFIERS) return self.relationIdentifiers def _get_relationIdentifierToRelationArrayIndex(self): if self.relationIdentifierToRelationArrayIndex is None: self.serializer._load_field( self.serializer._FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX) return self.relationIdentifierToRelationArrayIndex def _get_relationMemberTypes(self): if self.relationMemberTypes is None: self.serializer._load_field(self.serializer._FIELD_RELATION_MEMBER_TYPES) return self.relationMemberTypes def _get_relationMemberIndices(self): if self.relationMemberIndices is None: self.serializer._load_field(self.serializer._FIELD_RELATION_MEMBER_INDICES) return self.relationMemberIndices def _get_relationMemberRoles(self): if self.relationMemberRoles is None: self.serializer._load_field(self.serializer._FIELD_RELATION_MEMBER_ROLES) return self.relationMemberRoles def _get_relationTags(self): if self.relationTags is None: self.serializer._load_field(self.serializer._FIELD_RELATION_TAGS) self.relationTags.set_dictionary(self._get_dictionary()) return self.relationTags def _get_relationIndexToRelationIndices(self): if self.relationIndexToRelationIndices is None: self.serializer._load_field(self.serializer._FIELD_RELATION_INDEX_TO_RELATION_INDICES) return self.relationIndexToRelationIndices # --- spatial index loading functions --- def _get_point_spatial_index(self): if self.point_spatial_index is None: self.point_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.POINT, self.points()) self.point_spatial_index.initialize_rtree() return self.point_spatial_index def _get_line_spatial_index(self): if self.line_spatial_index is None: self.line_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.LINE, self.lines()) self.line_spatial_index.initialize_rtree() return self.line_spatial_index def _get_area_spatial_index(self): if self.area_spatial_index is None: self.area_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.AREA, self.areas()) self.area_spatial_index.initialize_rtree() return self.area_spatial_index def _get_node_spatial_index(self): if self.node_spatial_index is None: self.node_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.NODE, self.nodes()) self.node_spatial_index.initialize_rtree() return self.node_spatial_index def _get_edge_spatial_index(self): if self.edge_spatial_index is None: self.edge_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.EDGE, self.edges()) self.edge_spatial_index.initialize_rtree() return self.edge_spatial_index def _get_relation_spatial_index(self): if self.relation_spatial_index is None: self.relation_spatial_index = pyatlas.spatial_index.SpatialIndex( self, pyatlas.atlas_entities.EntityType.RELATION, self.relations()) self.relation_spatial_index.initialize_rtree() return self.relation_spatial_index class _AtlasSerializer(object): """ The Atlas serializer. Used by Atlas to read ZIP entries from the backing store. This class should not be used directly. """ _FIELD_METADATA = 'metaData' _FIELD_DICTIONARY = 'dictionary' _FIELD_POINT_IDENTIFIERS = 'pointIdentifiers' _FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX = 'pointIdentifierToPointArrayIndex' _FIELD_POINT_LOCATIONS = 'pointLocations' _FIELD_POINT_TAGS = 'pointTags' _FIELD_POINT_INDEX_TO_RELATION_INDICES = 'pointIndexToRelationIndices' _FIELD_LINE_IDENTIFIERS = 'lineIdentifiers' _FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX = 'lineIdentifierToLineArrayIndex' _FIELD_LINE_POLYLINES = 'linePolyLines' _FIELD_LINE_TAGS = 'lineTags' _FIELD_LINE_INDEX_TO_RELATION_INDICES = 'lineIndexToRelationIndices' _FIELD_AREA_IDENTIFIERS = 'areaIdentifiers' _FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX = 'areaIdentifierToAreaArrayIndex' _FIELD_AREA_POLYGONS = 'areaPolygons' _FIELD_AREA_TAGS = 'areaTags' _FIELD_AREA_INDEX_TO_RELATION_INDICES = 'areaIndexToRelationIndices' _FIELD_NODE_IDENTIFIERS = 'nodeIdentifiers' _FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX = 'nodeIdentifierToNodeArrayIndex' _FIELD_NODE_LOCATIONS = 'nodeLocations' _FIELD_NODE_TAGS = 'nodeTags' _FIELD_NODE_IN_EDGES_INDICES = 'nodeInEdgesIndices' _FIELD_NODE_OUT_EDGES_INDICES = 'nodeOutEdgesIndices' _FIELD_NODE_INDEX_TO_RELATION_INDICES = 'nodeIndexToRelationIndices' _FIELD_EDGE_IDENTIFIERS = 'edgeIdentifiers' _FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX = 'edgeIdentifierToEdgeArrayIndex' _FIELD_EDGE_START_NODE_INDEX = 'edgeStartNodeIndex' _FIELD_EDGE_END_NODE_INDEX = 'edgeEndNodeIndex' _FIELD_EDGE_POLYLINES = 'edgePolyLines' _FIELD_EDGE_TAGS = 'edgeTags' _FIELD_EDGE_INDEX_TO_RELATION_INDICES = 'edgeIndexToRelationIndices' _FIELD_RELATION_IDENTIFIERS = 'relationIdentifiers' _FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX = 'relationIdentifierToRelationArrayIndex' _FIELD_RELATION_MEMBER_TYPES = 'relationMemberTypes' _FIELD_RELATION_MEMBER_INDICES = 'relationMemberIndices' _FIELD_RELATION_MEMBER_ROLES = 'relationMemberRoles' _FIELD_RELATION_TAGS = 'relationTags' _FIELD_RELATION_INDEX_TO_RELATION_INDICES = 'relationIndexToRelationIndices' # yapf: disable _FIELD_NAMES_TO_LOAD_METHODS = { _FIELD_METADATA: '_load_metadata', _FIELD_DICTIONARY: '_load_dictionary', _FIELD_POINT_IDENTIFIERS: '_load_pointIdentifiers', _FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX: '_load_pointIdentifierToPointArrayIndex', _FIELD_POINT_LOCATIONS: '_load_pointLocations', _FIELD_POINT_TAGS: '_load_pointTags', _FIELD_POINT_INDEX_TO_RELATION_INDICES: '_load_pointIndexToRelationIndices', _FIELD_LINE_IDENTIFIERS: '_load_lineIdentifiers', _FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX: '_load_lineIdentifierToLineArrayIndex', _FIELD_LINE_POLYLINES: '_load_linePolylines', _FIELD_LINE_TAGS: '_load_lineTags', _FIELD_LINE_INDEX_TO_RELATION_INDICES: '_load_lineIndexToRelationIndices', _FIELD_AREA_IDENTIFIERS: '_load_areaIdentifiers', _FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX: '_load_areaIdentifierToAreaArrayIndex', _FIELD_AREA_POLYGONS: '_load_areaPolygons', _FIELD_AREA_TAGS: '_load_areaTags', _FIELD_AREA_INDEX_TO_RELATION_INDICES: '_load_areaIndexToRelationIndices', _FIELD_NODE_IDENTIFIERS: '_load_nodeIdentifiers', _FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX: '_load_nodeIdentifierToNodeArrayIndex', _FIELD_NODE_LOCATIONS: '_load_nodeLocations', _FIELD_NODE_TAGS: '_load_nodeTags', _FIELD_NODE_IN_EDGES_INDICES: '_load_nodeInEdgesIndices', _FIELD_NODE_OUT_EDGES_INDICES: '_load_nodeOutEdgesIndices', _FIELD_NODE_INDEX_TO_RELATION_INDICES: '_load_nodeIndexToRelationIndices', _FIELD_EDGE_IDENTIFIERS: '_load_edgeIdentifiers', _FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX: '_load_edgeIdentifierToEdgeArrayIndex', _FIELD_EDGE_START_NODE_INDEX: '_load_edgeStartNodeIndex', _FIELD_EDGE_END_NODE_INDEX: '_load_edgeEndNodeIndex', _FIELD_EDGE_POLYLINES: '_load_edgePolylines', _FIELD_EDGE_TAGS: '_load_edgeTags', _FIELD_EDGE_INDEX_TO_RELATION_INDICES: '_load_edgeIndexToRelationIndices', _FIELD_RELATION_IDENTIFIERS: '_load_relationIdentifiers', _FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX: '_load_relationIdentifierToRelationArrayIndex', _FIELD_RELATION_MEMBER_TYPES: '_load_relationMemberTypes', _FIELD_RELATION_MEMBER_INDICES: '_load_relationMemberIndices', _FIELD_RELATION_MEMBER_ROLES: '_load_relationMemberRoles', _FIELD_RELATION_TAGS: '_load_relationTags', _FIELD_RELATION_INDEX_TO_RELATION_INDICES: '_load_relationIndexToRelationIndices' } # yapf: enable def __init__(self, atlas_file, atlas): self.atlas_file = atlas_file self.atlas = atlas def _load_all_fields(self): for field in self._FIELD_NAMES_TO_LOAD_METHODS: self._load_field(field) def _load_metadata(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_METADATA) proto_metadata = pyatlas.autogen.ProtoAtlasMetaData_pb2.ProtoAtlasMetaData() proto_metadata.ParseFromString(zip_entry_data) self.atlas.metaData = pyatlas.atlas_metadata._get_atlas_metadata_from_proto(proto_metadata) def _load_dictionary(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_DICTIONARY) proto_dictionary = pyatlas.autogen.ProtoIntegerStringDictionary_pb2.ProtoIntegerStringDictionary( ) proto_dictionary.ParseFromString(zip_entry_data) self.atlas.dictionary = _get_integer_dictionary_from_proto(proto_dictionary) def _load_pointIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_POINT_IDENTIFIERS) self.atlas.pointIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.pointIdentifiers.ParseFromString(zip_entry_data) def _load_pointIdentifierToPointArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.pointIdentifierToPointArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_pointLocations(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_POINT_LOCATIONS) self.atlas.pointLocations = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.pointLocations.ParseFromString(zip_entry_data) def _load_pointTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_POINT_TAGS) proto_point_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_point_tags.ParseFromString(zip_entry_data) self.atlas.pointTags = _get_packed_tag_store_from_proto(proto_point_tags) def _load_pointIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_POINT_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.pointIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_lineIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_LINE_IDENTIFIERS) self.atlas.lineIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.lineIdentifiers.ParseFromString(zip_entry_data) def _load_lineIdentifierToLineArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.lineIdentifierToLineArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_linePolylines(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_LINE_POLYLINES) proto_array = pyatlas.autogen.ProtoPolyLineArray_pb2.ProtoPolyLineArray() proto_array.ParseFromString(zip_entry_data) result = [] for encoding in proto_array.encodings: poly_line = pyatlas.geometry.decompress_polyline(encoding) result.append(poly_line) self.atlas.linePolyLines = result def _load_lineTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_LINE_TAGS) proto_line_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_line_tags.ParseFromString(zip_entry_data) self.atlas.lineTags = _get_packed_tag_store_from_proto(proto_line_tags) def _load_lineIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_LINE_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.lineIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_areaIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_AREA_IDENTIFIERS) self.atlas.areaIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.areaIdentifiers.ParseFromString(zip_entry_data) def _load_areaIdentifierToAreaArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.areaIdentifierToAreaArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_areaPolygons(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_AREA_POLYGONS) proto_array = pyatlas.autogen.ProtoPolygonArray_pb2.ProtoPolygonArray() proto_array.ParseFromString(zip_entry_data) result = [] for encoding in proto_array.encodings: polygon0 = pyatlas.geometry.decompress_polygon(encoding) result.append(polygon0) self.atlas.areaPolygons = result def _load_areaTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_AREA_TAGS) proto_area_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_area_tags.ParseFromString(zip_entry_data) self.atlas.areaTags = _get_packed_tag_store_from_proto(proto_area_tags) def _load_areaIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_AREA_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.areaIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_nodeIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_IDENTIFIERS) self.atlas.nodeIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.nodeIdentifiers.ParseFromString(zip_entry_data) def _load_nodeIdentifierToNodeArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.nodeIdentifierToNodeArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_nodeLocations(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_LOCATIONS) self.atlas.nodeLocations = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.nodeLocations.ParseFromString(zip_entry_data) def _load_nodeTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_TAGS) proto_node_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_node_tags.ParseFromString(zip_entry_data) self.atlas.nodeTags = _get_packed_tag_store_from_proto(proto_node_tags) def _load_nodeInEdgesIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_IN_EDGES_INDICES) self.atlas.nodeInEdgesIndices = pyatlas.autogen.ProtoLongArrayOfArrays_pb2.ProtoLongArrayOfArrays( ) self.atlas.nodeInEdgesIndices.ParseFromString(zip_entry_data) def _load_nodeOutEdgesIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_OUT_EDGES_INDICES) self.atlas.nodeOutEdgesIndices = pyatlas.autogen.ProtoLongArrayOfArrays_pb2.ProtoLongArrayOfArrays( ) self.atlas.nodeOutEdgesIndices.ParseFromString(zip_entry_data) def _load_nodeIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_NODE_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.nodeIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_edgeIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_IDENTIFIERS) self.atlas.edgeIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.edgeIdentifiers.ParseFromString(zip_entry_data) def _load_edgeIdentifierToEdgeArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.edgeIdentifierToEdgeArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_edgeStartNodeIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_START_NODE_INDEX) self.atlas.edgeStartNodeIndex = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.edgeStartNodeIndex.ParseFromString(zip_entry_data) def _load_edgeEndNodeIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_END_NODE_INDEX) self.atlas.edgeEndNodeIndex = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.edgeEndNodeIndex.ParseFromString(zip_entry_data) def _load_edgePolylines(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_POLYLINES) proto_array = pyatlas.autogen.ProtoPolyLineArray_pb2.ProtoPolyLineArray() proto_array.ParseFromString(zip_entry_data) result = [] for encoding in proto_array.encodings: poly_line = pyatlas.geometry.decompress_polyline(encoding) result.append(poly_line) self.atlas.edgePolyLines = result def _load_edgeTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_TAGS) proto_edge_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_edge_tags.ParseFromString(zip_entry_data) self.atlas.edgeTags = _get_packed_tag_store_from_proto(proto_edge_tags) def _load_edgeIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_EDGE_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.edgeIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_relationIdentifiers(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_IDENTIFIERS) self.atlas.relationIdentifiers = pyatlas.autogen.ProtoLongArray_pb2.ProtoLongArray() self.atlas.relationIdentifiers.ParseFromString(zip_entry_data) def _load_relationIdentifierToRelationArrayIndex(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX) proto_map = pyatlas.autogen.ProtoLongToLongMap_pb2.ProtoLongToLongMap() proto_map.ParseFromString(zip_entry_data) self.atlas.relationIdentifierToRelationArrayIndex = _get_dict_from_longtolongmap(proto_map) def _load_relationMemberTypes(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_MEMBER_TYPES) self.atlas.relationMemberTypes = pyatlas.autogen.ProtoByteArrayOfArrays_pb2.ProtoByteArrayOfArrays( ) self.atlas.relationMemberTypes.ParseFromString(zip_entry_data) def _load_relationMemberIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_MEMBER_INDICES) self.atlas.relationMemberIndices = pyatlas.autogen.ProtoLongArrayOfArrays_pb2.ProtoLongArrayOfArrays( ) self.atlas.relationMemberIndices.ParseFromString(zip_entry_data) def _load_relationMemberRoles(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_MEMBER_ROLES) self.atlas.relationMemberRoles = pyatlas.autogen.ProtoIntegerArrayOfArrays_pb2.ProtoIntegerArrayOfArrays( ) self.atlas.relationMemberRoles.ParseFromString(zip_entry_data) def _load_relationTags(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_TAGS) proto_relation_tags = pyatlas.autogen.ProtoPackedTagStore_pb2.ProtoPackedTagStore() proto_relation_tags.ParseFromString(zip_entry_data) self.atlas.relationTags = _get_packed_tag_store_from_proto(proto_relation_tags) def _load_relationIndexToRelationIndices(self): zip_entry_data = _read_zipentry(self.atlas_file, self._FIELD_RELATION_INDEX_TO_RELATION_INDICES) proto_multimap = pyatlas.autogen.ProtoLongToLongMultiMap_pb2.ProtoLongToLongMultiMap() proto_multimap.ParseFromString(zip_entry_data) self.atlas.relationIndexToRelationIndices = _get_dict_from_protolongtolongmultimap( proto_multimap) def _load_field(self, field_name): if field_name not in self._FIELD_NAMES_TO_LOAD_METHODS: raise KeyError('unrecognized field {}'.format(field_name)) # reflection code to get the appropriate load method for the field we are loading method_name = self._FIELD_NAMES_TO_LOAD_METHODS[field_name] load_method_to_call = getattr(self, method_name) load_method_to_call() class _PackedTagStore(object): """ Stores indexes into the global tag dictionary. """ def __init__(self, dictionary): self.keys = [] self.values = [] self.dictionary = dictionary def set_dictionary(self, dictionary): self.dictionary = dictionary def __str__(self): string = "KEYS:\n" string += str(self.keys) string += "\nVALUES:\n" string += str(self.values) return string def to_key_value_dict(self, index): if self.dictionary is None: raise ValueError('dictionary is not set') result = {} if len(self.keys) == 0: return result key_array = self.keys[index] value_array = self.values[index] if len(key_array) != len(value_array): raise IndexError('array length mismatch') for key, value in zip(key_array, value_array): result[self.dictionary.word(key)] = self.dictionary.word(value) return result class _IntegerDictionary(object): """ Integer string two-way dictionary. """ def __init__(self): self.word_to_index = {} self.index_to_word = {} def add(self, index, word): if word in self.word_to_index: return self.word_to_index[word] self.word_to_index[word] = index self.index_to_word[index] = word def size(self): return len(self.word_to_index) def word(self, index): # TODO this could throw a KeyError, how to handle? return self.index_to_word[index] # --- static utility functions --- def _read_zipentry(zip_file, entry): """ Read a zip entry from a given zip file. """ with zipfile.ZipFile(zip_file, 'r') as myzip: return myzip.read(entry) def _get_packed_tag_store_from_proto(proto_store): """ Take a decoded ProtoPackedTagStore object and turn it into a more user friendly PackedTagStore object. """ new_store = _PackedTagStore(None) for key_array in proto_store.keys.arrays: new_sub_array = [] for element in key_array.elements: new_sub_array.append(element) new_store.keys.append(new_sub_array) for value_array in proto_store.values.arrays: new_sub_array = [] for element in value_array.elements: new_sub_array.append(element) new_store.values.append(new_sub_array) return new_store def _get_integer_dictionary_from_proto(proto_integer_dictionary): """ Takes a decoded ProtoIntegerStringDictionary object and turns it into a more user friendly IntegerDictionary object. """ new_dict = _IntegerDictionary() size_indexes = len(proto_integer_dictionary.indexes) size_words = len(proto_integer_dictionary.words) if size_words != size_indexes: raise IndexError('proto array sizes do not match') for index, word in zip(proto_integer_dictionary.indexes, proto_integer_dictionary.words): new_dict.add(index, word) return new_dict def _get_dict_from_longtolongmap(proto_map): """ Convert the ProtoLongToLongMap_pb2 type to a simple dict. """ if len(proto_map.keys.elements) != len(proto_map.values.elements): raise IndexError('array length mismatch') new_dict = {} for key, value in zip(proto_map.keys.elements, proto_map.values.elements): new_dict[key] = value return new_dict def _get_dict_from_protolongtolongmultimap(proto_map): """ Convert the ProtoLongToLongMultiMap_pb2 type to a simple dict. """ if len(proto_map.keys.elements) != len(proto_map.values): raise IndexError('array length mismatch') new_dict = {} for key, array_value in zip(proto_map.keys.elements, proto_map.values): value_list = [] for value in array_value.elements: value_list.append(value) new_dict[key] = value_list return new_dict ================================================ FILE: pyatlas/pyatlas/atlas_entities.py ================================================ """ This module defines the Atlas entity types. Entities are features that can be queried from the Atlas, so things like Nodes, Lines, Relations, etc. The Atlas entities themselves are flyweight classes, and simply store references into the Atlas feature arrays. In general, the entity classes in this module should not be instantiated directly. Instead, entity objects should be obtained through the appropriate Atlas API method. """ import pyatlas.identifier_converters import pyatlas.geometry class EntityType(object): """ An enum for AtlasEntity types. Valid settings are NODE, EDGE, AREA, LINE, POINT, and RELATION. """ def __init__(self): raise NotImplementedError # these MUST match the Java implementation for serialization compatibility NODE = 0 EDGE = 1 AREA = 2 LINE = 3 POINT = 4 RELATION = 5 _strs = { NODE: "NODE", EDGE: "EDGE", AREA: "AREA", LINE: "LINE", POINT: "POINT", RELATION: "RELATION", } class AtlasEntity(pyatlas.geometry.Boundable): """ A tagged, located entity in an Atlas. Can be a member of a relation. An AtlasEntity should not be instantiated directly. Use one of the appropriate sub-classes. """ def __init__(self, parent_atlas): """ AtlasEntity should not be instantiated directly. """ self.parent_atlas = parent_atlas def __eq__(self, other): """ Determine if this AtlasEntity is equal to another. Two entities are considered equal if they have the same identifier and the same type. """ return self.get_identifier() == other.get_identifier() and self.get_type( ) == other.get_type() def __ne__(self, other): """ Determine if this AtlasEntity is NOT equal to another. Inverse of the comparison made by the __eq__() method. """ return not (self.get_identifier() == other.get_identifier() and self.get_type() == other.get_type()) def __hash__(self): """ Compute a hashcode for this AtlasEntity. """ return self.get_identifier() * 31 + self.get_type() def get_identifier(self): """ Get the Atlas identifier of this AtlasEntity. """ raise NotImplementedError('subclass must implement') def get_tags(self): """ Get a dictionary of this AtlasEntity's tags. """ raise NotImplementedError('subclass must implement') def bounds(self): """ Compute the bounding Rectangle of this AtlasEntity. """ raise NotImplementedError('subclass must implement') def intersects(self, polygon): """ Check if this AtlasEntity intersects some Polygon. """ raise NotImplementedError('subclass must implement') def relations(self): """ Get the set of Relations of which this AtlasEntity is a member. """ raise NotImplementedError('subclass must implement') def get_type(self): """ Get the EntityType of this AtlasEntity """ raise NotImplementedError('subclass must implement') def get_osm_identifier(self): """ Get the OSM identifier of this AtlasEntity. """ atlas_id = self.get_identifier() return pyatlas.identifier_converters.get_osm_identifier(atlas_id) def get_parent_atlas(self): """ Get the Atlas that contains this AtlasEntity. """ return self.parent_atlas def _get_relations_helper(self, relation_map, index): """ Subclasses of AtlasEntity can use this helper function to avoid code duplication in their relations() implementations. """ relation_set = set() if index not in relation_map: return relation_set for relation_index in relation_map[index]: relation = Relation(self.get_parent_atlas(), relation_index) relation_set.add(relation) return frozenset(relation_set) class Point(AtlasEntity): """ An Atlas Point. Points are non-navigable. """ def __init__(self, parent_atlas, index): """ Constuct a new Point. This should not be called directly. """ super(Point, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Point. """ result = '[ ' result += 'Point: id=' + str(self.get_identifier()) result += ', geom=' + str(self.as_location()) result += ', tags=' + str(self.get_tags()) string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ' ]' return result def get_identifier(self): """ Get the Atlas identifier of this Point. """ return self.get_parent_atlas()._get_pointIdentifiers().elements[self.index] def as_location(self): """ Get the Location of this Point. """ long_location = self.get_parent_atlas()._get_pointLocations().elements[self.index] return pyatlas.geometry.location_from_packed_int(long_location) def get_tags(self): """ Get a dictionary of this Point's tags. """ point_tag_store = self.get_parent_atlas()._get_pointTags() return point_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Point. """ return self.as_location().bounds() def intersects(self, polygon): """ Check if this Point intersects some Polygon. """ return self.as_location().intersects(polygon) def relations(self): """ Get the frozenset of Relations of which this Point is a member. Returns an empty set if this Point is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_pointIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.POINT. """ return EntityType.POINT class Line(AtlasEntity): """ An Atlas Line. Lines are non-navigable. """ def __init__(self, parent_atlas, index): """ Constuct a new Line. This should not be called directly. """ super(Line, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Line. """ result = '[ ' result += 'Line: id=' + str(self.get_identifier()) result += ', geom=' + str(self.as_polyline()) result += ', tags=' + str(self.get_tags()) string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ' ]' return result def as_polyline(self): """ Get the PolyLine geometry of this Line. """ return self.get_parent_atlas()._get_linePolyLines()[self.index] def get_identifier(self): """ Get the Atlas identifier of this Line. """ return self.get_parent_atlas()._get_lineIdentifiers().elements[self.index] def get_tags(self): """ Get a dictionary of this Line's tags. """ line_tag_store = self.get_parent_atlas()._get_lineTags() return line_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Line. """ return self.as_polyline().bounds() def intersects(self, polygon): """ Check if this Line intersects some Polygon. """ return self.as_polyline().intersects(polygon) def relations(self): """ Get the frozenset of Relations of which this Line is a member. Returns an empty set if this Line is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_lineIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.LINE. """ return EntityType.LINE class Area(AtlasEntity): """ An Atlas Area. """ def __init__(self, parent_atlas, index): """ Constuct a new Area. This should not be called directly. """ super(Area, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Area. """ result = '[ ' result += 'Area: id=' + str(self.get_identifier()) result += ', geom=' + str(self.as_polygon()) result += ', tags=' + str(self.get_tags()) string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ' ]' return result def get_identifier(self): """ Get the Atlas identifier of this Area. """ return self.get_parent_atlas()._get_areaIdentifiers().elements[self.index] def as_polygon(self): """ Get the Polygon geometry of this Area. """ return self.get_parent_atlas()._get_areaPolygons()[self.index] def get_tags(self): """ Get a dictionary of this Area's tags. """ area_tag_store = self.get_parent_atlas()._get_areaTags() return area_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Area. """ return self.as_polygon().bounds() def intersects(self, polygon): """ Check if this Area intersects some Polygon. """ return self.as_polygon().intersects(polygon) def relations(self): """ Get the frozenset of Relations of which this Area is a member. Returns an empty set if this Area is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_areaIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.AREA. """ return EntityType.AREA class Node(AtlasEntity): """ An Atlas Node. A Node is like a Point, except it is part of a navigable structure that can be traversed using the Node and Edge API methods. """ def __init__(self, parent_atlas, index): """ Constuct a new Node. This should not be called directly. """ super(Node, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Node. """ result = '[ ' result += 'Node: id=' + str(self.get_identifier()) result += ', geom=' + str(self.as_location()) result += ', tags=' + str(self.get_tags()) string = "" for edge in self.in_edges(): string += str(edge.get_identifier()) + ',' result += ', inEdges=[' + string + ']' string = "" for edge in self.out_edges(): string += str(edge.get_identifier()) + ',' result += ', outEdges=[' + string + ']' string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ' ]' return result def get_identifier(self): """ Get the Atlas identifier of this node. """ return self.get_parent_atlas()._get_nodeIdentifiers().elements[self.index] def as_location(self): """ Get the Location of this Node. """ long_location = self.get_parent_atlas()._get_nodeLocations().elements[self.index] return pyatlas.geometry.location_from_packed_int(long_location) def get_tags(self): """ Get a dictionary of this Node's tags. """ node_tag_store = self.get_parent_atlas()._get_nodeTags() return node_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Point. """ return self.as_location().bounds() def intersects(self, polygon): """ Check if this Node intersects some Polygon. """ return self.as_location().intersects(polygon) def in_edges(self): """ Get a list of incoming Edges to this Node. The list is sorted by the Edges' Atlas IDs. """ result = [] node_in_edges_indices = self.get_parent_atlas()._get_nodeInEdgesIndices() for index in node_in_edges_indices.arrays[self.index].elements: result.append(Edge(self.get_parent_atlas(), index)) return sorted(result) def out_edges(self): """ Get a list of outgoing Edges from this Node. The list is sorted by the Edges' Atlas IDs. """ result = [] node_out_edges_indices = self.get_parent_atlas()._get_nodeOutEdgesIndices() for index in node_out_edges_indices.arrays[self.index].elements: result.append(Edge(self.get_parent_atlas(), index)) return sorted(result) def connected_edges(self): """ Get a list of all Edges connected to this Node. The list is sorted by the Edges' Atlas IDs. """ result = [] for edge in self.in_edges(): result.append(edge) for edge in self.out_edges(): result.append(edge) return sorted(result) def get_absolute_valence(self): """ Get the number of Edges connected to this node. Considers all Edges, not just master Edges. """ return len(self.connected_edges()) def get_valence(self): """ Get the number of Edges connected to this node. Only considers the master Edges. """ connected_edges = self.connected_edges() valence = 0 for edge in connected_edges: if edge.is_master_edge(): valence += 1 return valence def relations(self): """ Get the frozenset of Relations of which this Node is a member. Returns an empty set if this Node is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_nodeIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.NODE. """ return EntityType.NODE class Edge(AtlasEntity): """ A unidirectional Atlas Edge. An Edge is like a Line, except it is part of a navigable structure that can be traversed using the Node and Edge API methods. Bidirectional OSM ways are represented with two opposing Edges, where one of them is the master Edge. The master Edge will have a positive identifier and the same traffic direction as OSM. """ def __init__(self, parent_atlas, index): """ Constuct a new Edge. This should not be called directly. """ super(Edge, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Edge. """ result = '[ ' result += 'Edge: id=' + str(self.get_identifier()) string = "" string += str(self.start().get_identifier()) + ',' result += ', start=[' + string + ']' string = "" string += str(self.end().get_identifier()) + ',' result += ', end=[' + string + ']' result += ', geom=' + str(self.as_polyline()) result += ', tags=' + str(self.get_tags()) string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ' ]' return result def __lt__(self, other): """ Custom implementation of less-than so that collections of Edges can be easily sorted. """ return self.get_identifier() < other.get_identifier() def as_polyline(self): """ Get the PolyLine geometry of this Edge. """ return self.get_parent_atlas()._get_edgePolyLines()[self.index] def get_identifier(self): """ Get the Atlas identifier of this Edge. :return: the Atlas id """ return self.get_parent_atlas()._get_edgeIdentifiers().elements[self.index] def get_tags(self): """ Get a dictionary of this Edge's tags. """ edge_tag_store = self.get_parent_atlas()._get_edgeTags() return edge_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Edge. """ return self.as_polyline().bounds() def intersects(self, polygon): """ Check if this Edge intersects some Polygon. """ return self.as_polyline().intersects(polygon) def relations(self): """ Get the frozenset of Relations of which this Edge is a member. Returns an empty set if this Edge is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_edgeIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def connected_nodes(self): """ Get a frozenset of the Nodes connected to this Edge. """ result = set() result.add(self.start()) result.add(self.end()) return frozenset(result) def connected_edges(self): """ Get a frozenset of the Edges connected at the ends of the Nodes of this Edge. The set will not contain the Edge this method is called on, but will contain the reversed Edge if this Edge is part of a two-way road. """ result = set() for edge in self.end().connected_edges(): if self != edge: result.add(edge) for edge in self.start().connected_edges(): if self != edge: result.add(edge) return result def start(self): """ Get the starting Node of this Edge. """ edge_start_node_index = self.get_parent_atlas()._get_edgeStartNodeIndex() index = edge_start_node_index.elements[self.index] return Node(self.get_parent_atlas(), index) def end(self): """ Get the ending Node of this Edge. """ edge_end_node_index = self.get_parent_atlas()._get_edgeEndNodeIndex() index = edge_end_node_index.elements[self.index] return Node(self.get_parent_atlas(), index) def get_master_edge(self): """ Get the master for this Edge. Returns itself if this is the master Edge. """ if self.is_master_edge(): return self else: return self.get_parent_atlas().edge(-1 * self.get_identifier()) def is_master_edge(self): """ Checks if this Edge is a master edge. """ return self.get_identifier() > 0 def has_reversed_edge(self): """ Checks if this Edge is a member of a bidirectional Edge pairing. """ return self.get_parent_atlas().edge(-1 * self.get_identifier()) is not None def is_reversed_edge(self, candidate): """ Check if the candidate Edge is the bidirectional reverse of this Edge. """ if candidate.get_type() != EntityType.EDGE: return False return self.get_identifier() == -1 * candidate.get_identifier() def get_reversed_edge(self): """ Get the bidirectional pair Edge to this Edge, if it exists. Returns None if it does not. """ if not self.has_reversed_edge(): return None return self.get_parent_atlas().edge(-1 * self.get_identifier()) def get_highway_tag_value(self): """ Get the value of the "highway" tag of this Edge, if present. Returns None if there is no "highway" tag. """ tags = self.get_tags() if 'highway' in tags: return tags['highway'] else: return None def is_connected_at_end_to(self, candidates): """ Given a set of AtlasEntity candidates, test if this edge is directly connected at its end to at least one of the candidates. """ for entity in candidates: if entity.get_type() == EntityType.NODE: if self.end() == entity: return True if entity.get_type() == EntityType.EDGE: if self.end() == entity.start(): return True return False def is_connected_at_start_to(self, candidates): """ Given a set of AtlasEntity candidates, test if this edge is directly connected at its start to at least one of the candidates. """ for entity in candidates: if entity.get_type() == EntityType.NODE: if self.start() == entity: return True if entity.get_type() == EntityType.EDGE: if self.start() == entity.end(): return True return False def is_way_sectioned(self): """ Determine if this Edge is a way-sectioned road. """ return pyatlas.identifier_converters.get_way_section_index(self.get_identifier()) != 0 def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.EDGE. """ return EntityType.EDGE class Relation(AtlasEntity): """ An Atlas Relation. Aggregates AtlasEntities in a logical relationship. Can contain other Relations as members. """ def __init__(self, parent_atlas, index): """ Constuct a new Relation. This should not be called directly. """ super(Relation, self).__init__(parent_atlas) self.index = index def __str__(self): """ Get a string representation of this Relation. """ result = '[ ' result += 'Relation: id=' + str(self.get_identifier()) string = '' for member in self.get_members(): string += str(member) + ',' result += ', members=[' + string + ']' string = '' for relation in self.relations(): string += str(relation.get_identifier()) + ',' result += ', relations=[' + string + ']' result += ', tags=' + str(self.get_tags()) result += ' ]' return result def get_identifier(self): """ Get the Atlas identifier of this Relation. """ return self.get_parent_atlas()._get_relationIdentifiers().elements[self.index] def get_members(self): """ Get a sorted list of this Relation's members. The members are in RelationMember form. """ result = [] relation_identifiers = self.get_parent_atlas()._get_relationIdentifiers() relation_member_types = self.get_parent_atlas()._get_relationMemberTypes() relation_member_indices = self.get_parent_atlas()._get_relationMemberIndices() relation_member_roles = self.get_parent_atlas()._get_relationMemberRoles() dictionary = self.get_parent_atlas()._get_dictionary() array_index = 0 # the relationMemberTypes field is a byte array, so the Python treats # it as a string. We need to convert it to a true byte array. for type_value in bytearray(relation_member_types.arrays[self.index].elements): member_index = relation_member_indices.arrays[self.index].elements[array_index] role = dictionary.word(relation_member_roles.arrays[self.index].elements[array_index]) if type_value == EntityType.NODE: entity = Node(self.get_parent_atlas(), member_index) elif type_value == EntityType.EDGE: entity = Edge(self.get_parent_atlas(), member_index) elif type_value == EntityType.AREA: entity = Area(self.get_parent_atlas(), member_index) elif type_value == EntityType.LINE: entity = Line(self.get_parent_atlas(), member_index) elif type_value == EntityType.POINT: entity = Point(self.get_parent_atlas(), member_index) elif type_value == EntityType.RELATION: entity = Relation(self.get_parent_atlas(), member_index) else: raise ValueError('invalid EntityType value ' + str(type_value)) result.append(RelationMember(role, entity, relation_identifiers.elements[self.index])) array_index += 1 return sorted(result) def get_tags(self): """ Get a dictionary of this Relation's tags. """ relation_tag_store = self.get_parent_atlas()._get_relationTags() return relation_tag_store.to_key_value_dict(self.index) def bounds(self): """ Compute the bounding Rectangle of this Relation's members. """ # FIXME this fails if Relations have self-referencing members # this will never happen in a PackedAtlas so it should be OK for now # if pyatlas ever supports MultiAtlas then this will be a concern members = self.get_members() if len(members) == 0: return pyatlas.geometry.Rectangle(0, 0) entities_to_consider = [] for member in self.get_members(): entity = member.get_entity() if entity is None: raise ValueError('entity was None, how did this happen?') entities_to_consider.append(entity) return pyatlas.geometry.bounds_atlasentities(entities_to_consider) def intersects(self, polygon): """ Check if any member of this Relation intersects some Polygon. """ # FIXME this fails if Relations have self-referencing members # this will never happen in a PackedAtlas so it should be OK for now # if pyatlas ever supports MultiAtlas then this will be a concern for member in self.get_members(): entity = member.get_entity() if entity.intersects(polygon): return True return False def relations(self): """ Get the frozenset of Relations of which this Relation is a member. Returns an empty set if this Relation is not a member of any Relations. """ relation_map = self.get_parent_atlas()._get_relationIndexToRelationIndices() return self._get_relations_helper(relation_map, self.index) def get_type(self): """ Overrides superclass get_type(). Always returns EntityType.RELATION. """ return EntityType.RELATION class RelationMember(object): """ A container type for Relation members. A RelationMember has a role as well as a reference to its AtlasEntity. """ def __init__(self, role, entity, identifier): """ Create a new RelationMember. """ self.role = role self.entity = entity self.identifier = identifier def __lt__(self, other): """ Define an ordering for RelationMembers. Compare EntityTypes, then identifiers, then roles. """ if self.entity.get_type() < other.entity.get_type(): return True elif self.entity.get_type() > other.entity.get_type(): return False else: if self.identifier < other.identifier: return True elif self.identifier > other.identifier: return False else: if self.role < other.role: return True else: return False def __str__(self): """ Get a string representation of this RelationMember. """ result = '[ ' result += 'id=' + str(self.get_entity().get_identifier()) result += ', type=' + entity_type_to_str(self.entity.get_type()) result += ', role=' + str(self.get_role()) result += ' ]' return result def get_entity(self): """ Get this RelationMember's AtlasEntity. """ return self.entity def get_relation_identifier(self): """ Get the identifier of the Relation of which this RelationMember is a member. """ return self.identifier def get_role(self): """ Get the role of this RelationMember. """ return self.role def entity_type_to_str(value): """ Convert an EntityType enum to a string representation. """ return EntityType._strs[value] ================================================ FILE: pyatlas/pyatlas/atlas_metadata.py ================================================ """ This module defines the AtlasMetaData container type. """ class AtlasMetaData(object): """ Container class for an Atlas's metadata. Readable metadata fields: number_of_points (long) number_of_lines (long) number_of_areas (long) number_of_nodes (long) number_of_edges (long) number_of_relations (long) original (bool) code_version (string) data_version (string) country (string) shard_name (string) tags (dict) """ def __init__(self): self.number_of_edges = 0 self.number_of_nodes = 0 self.number_of_areas = 0 self.number_of_lines = 0 self.number_of_points = 0 self.number_of_relations = 0 self.original = False self.code_version = "" self.data_version = "" self.country = "" self.shard_name = "" self.tags = {} def _get_atlas_metadata_from_proto(proto_atlas_metadata): """ Take a decoded ProtoAtlasMetaData object and turn it into a more user friendly AtlasMetaData object. """ new_atlas_metadata = AtlasMetaData() new_atlas_metadata.number_of_edges = proto_atlas_metadata.edgeNumber new_atlas_metadata.number_of_nodes = proto_atlas_metadata.nodeNumber new_atlas_metadata.number_of_areas = proto_atlas_metadata.areaNumber new_atlas_metadata.number_of_lines = proto_atlas_metadata.lineNumber new_atlas_metadata.number_of_points = proto_atlas_metadata.pointNumber new_atlas_metadata.number_of_relations = proto_atlas_metadata.relationNumber new_atlas_metadata.original = proto_atlas_metadata.original new_atlas_metadata.code_version = proto_atlas_metadata.codeVersion new_atlas_metadata.data_version = proto_atlas_metadata.dataVersion new_atlas_metadata.country = proto_atlas_metadata.country new_atlas_metadata.shard_name = proto_atlas_metadata.shardName # convert prototags and fill the tag dict for proto_tag in proto_atlas_metadata.tags: new_atlas_metadata.tags[proto_tag.key] = proto_tag.value return new_atlas_metadata ================================================ FILE: pyatlas/pyatlas/autogen/__init__.py ================================================ ================================================ FILE: pyatlas/pyatlas/geometry.py ================================================ """ This module defines the pyatlas geometry primitives as well as various helping functions for manipulating the geometry. These primitives are built using lat-long locations on the Earth. """ import math import shapely.geometry # --- Location definition constants --- _LATITUDE_MIN_DM7 = -900000000 _LATITUDE_MAX_DM7 = 900000000 _LONGITUDE_MIN_DM7 = -1800000000 _LONGITUDE_MAX_DM7 = 1800000000 - 1 # There are 10 million dm7 in a degree _DM7_PER_DEGREE = 10000000 # --- PolyLine encoding constants --- # Do not change the precision. It matches the default in the Java implementation. _PRECISION = 7 _ENCODING_OFFSET_MINUS_ONE = 63 _FIVE_BIT_MASK = 0x1f _SIXTH_BIT_MASK = 0x20 _BIT_SHIFT = 5 _MAXIMUM_DELTA_LONGITUDE = 180 * pow(10, _PRECISION) class Boundable(object): """ A Boundable is any geometric object that can be bounded by Rectangle. """ def __init__(self): raise NotImplementedError('Boundable should not be instantiated') def bounds(self): """ Get the bounding Rectangle of this object. """ raise NotImplementedError('subclass must implement') def intersects(self, polygon): """ Check if this Boundable intersects some Polygon. """ raise NotImplementedError('subclass must implement') class Location(Boundable): """ A latitude-longitude location. This is the building block of the other geometric types. Uses the dm7 type to represent coordinates. dm7 is an integral representation of a decimal degree fixed to 7 places of precision. 7 places are enough to specify any location on Earth with submeter accuracy. Examples: 45.01 degrees -> 450_100_000 dm7 -90 degrees -> -900_000_000 dm7 150.5 degrees -> 1_505_000_000 dm7 """ def __init__(self, latitude, longitude): """ Create a new Location with a dm7 latitude and longitude. To create a Location with degrees, use the geometry.location_with_degrees() module function. """ if not isinstance(latitude, int): raise TypeError('latitude must be an integer') if not isinstance(longitude, int): raise TypeError('longitude must be an integer') if latitude > _LATITUDE_MAX_DM7 or latitude < _LATITUDE_MIN_DM7: raise ValueError('latitude {} out of range'.format(str(latitude))) if longitude > _LONGITUDE_MAX_DM7 or longitude < _LONGITUDE_MIN_DM7: raise ValueError('longitude {} out of range'.format(str(longitude))) self.latitude = latitude self.longitude = longitude def __str__(self): """ Get the wkt string representation of this Location. The pair ordering of Locations is always (LATITUDE, LONGITUDE) """ shapely_point = location_to_shapely_point_wkt_compat(self) return shapely_point.wkt def __eq__(self, other): """ Check if two Locations are equal. Equivalent Locations have equal latitude and equal longitude. """ return self.latitude == other.latitude and self.longitude == other.longitude def __ne__(self, other): """ Check if two Locations are NOT equal. """ return not (self.latitude == other.latitude and self.longitude == other.longitude) def __hash__(self): """ Compute a hashcode for this Location. """ hash_value = self.latitude * 31 hash_value = hash_value * 31 + self.longitude return hash_value def get_as_packed_int(self): """ Pack this Location into a 64 bit integer. The higher order 32 bits are the latitude, the lower order 32 bits are the longitude. """ packed = self.latitude packed = packed << 32 packed = packed | (self.longitude & 0xFFFFFFFF) return packed def bounds(self): """ Get the bounding Rectangle of this Location. """ return Rectangle(self, self) def intersects(self, polygon): """ Check if this Location intersects some Polygon. """ return polygon.fully_geometrically_encloses_location(self) def get_latitude(self): """ Get the latitude of this Location as a dm7. """ return self.latitude def get_latitude_deg(self): """ Get the latitude of this Location as a degree. """ return dm7_as_degree(self.get_latitude()) def get_longitude(self): """ Get the longitude of this Location as a dm7. """ return self.longitude def get_longitude_deg(self): """ Get the latitude of this Location as a degree. """ return dm7_as_degree(self.get_longitude()) class PolyLine(Boundable): """ A PolyLine is a set of Locations in a specific order. """ def __init__(self, location_list, deep=False): """ Create a new PolyLine given a Location list. By default, it will perform a reference copy of the Location list. Can optionally perform a deep copy of the list instead. """ if len(location_list) == 0: raise ValueError('cannot have an empty PolyLine') if deep: self.location_list = [] for point in location_list: new_point = Location(point.get_latitude(), point.get_longitude()) self.location_list.append(new_point) else: self.location_list = location_list def __str__(self): """ Get the wkt string representation of this PolyLine. """ shapely_poly = polyline_to_shapely_linestring_wkt_compat(self) return shapely_poly.wkt def __eq__(self, other): """ Check if this PolyLine is the same as another PolyLine. Compares their internal Location list. """ if len(self.location_list) != len(other.location_list): return False for point, other_point in zip(self.locations(), other.locations()): if not point == other_point: return False return True def __ne__(self, other): """ Check if this PolyLine is NOT the same as another PolyLine. """ if len(self.location_list) != len(other.location_list): return True for point, other_point in zip(self.locations(), other.locations()): if not point == other_point: return True return False def __hash__(self): """ Compute a hashcode for this PolyLine. """ hash_value = 31 for point in self.locations(): hash_value = hash_value * 31 + point.__hash__() return hash_value def compress(self): """ Transform this PolyLine into its compressed representation. The compression is based on the MapQuest compressed lat/lon encoding found here: https://developer.mapquest.com/documentation/common/encode-decode/ """ old_latitude = 0 old_longitude = 0 encoded = "" precision = pow(10, _PRECISION) last = Location(0, 0) for point in self.locations(): latitude = int(round(dm7_as_degree(point.latitude) * precision)) longitude = int(round(dm7_as_degree(point.longitude) * precision)) encoded += _encode_number(latitude - old_latitude) delta_longitude = longitude - old_longitude if delta_longitude > _MAXIMUM_DELTA_LONGITUDE: raise ValueError( 'unable to compress polyline, consecutive points {} and {} too far apart', last, point) encoded += _encode_number(delta_longitude) old_latitude = latitude old_longitude = longitude last = point if type(encoded) is str: encoded = bytes(encoded, 'utf-8') return encoded def bounds(self): """ Get the bounding Rectangle of this PolyLine. """ return bounds_locations(self.locations()) def intersects_polyline(self, polyline): """ Check if this PolyLine intersects some PolyLine. """ shapely_polyline = polyline_to_shapely_linestring(polyline) shapely_polyline_self = polyline_to_shapely_linestring(self) return shapely_polyline_self.intersects(shapely_polyline) def intersects(self, polygon): """ Check if this PolyLine intersects some Polygon. """ return polygon.overlaps_polyline(self) def get_locations_list(self): """ Get the underlying Location list for this PolyLine. """ return self.location_list def length(self): """ Get the length of this PolyLine along the surface of the Earth, in meters. """ prev_location = None sum_distance = 0 for cur_location in self.locations(): if prev_location is not None: sum_distance += location_haversine_distance(cur_location, prev_location) prev_location = cur_location return sum_distance def locations(self): """ Get a generator for the Locations in this PolyLine. """ for point in self.location_list: yield point class Polygon(PolyLine): """ A special case of PolyLine that has an extra segment between the last and first point - effectively a closed PolyLine. The Polygon does not actually re-store the last (first) Location. Instead, the API simulates its presence. """ def __init__(self, location_list, deep=False): """ Create a new Polygon given a Location list. By default, it will perform a shallow copy of the parameter list. Can optionally perform a deep copy of the list. """ super(Polygon, self).__init__(location_list, deep) def __str__(self): """ Get the wkt string representation of this Polygon. """ shapely_poly = polygon_to_shapely_polygon_wkt_compat(self) return shapely_poly.wkt def fully_geometrically_encloses_location(self, location): """ Test if this Polygon fully geometrically encloses a given Location. Will return False if the Location lies perfectly on the Polygon's boundary. """ shapely_point = location_to_shapely_point(location) shapely_poly_self = polygon_to_shapely_polygon(self) return shapely_poly_self.contains(shapely_point) def _overlaps_polygon(self, polygon): """ Test if this Polygon overlaps a given Polygon at any point. """ # TODO Shapely differentiates between overlaps and intersects # Shapely intersects() allows one to contain the other # Shapely overlaps() means they intersect, but neither contains the other # which is the right choice here? shapely_polyg_self = polygon_to_shapely_polygon(self) shapely_polyg_other = polygon_to_shapely_polygon(polygon) return shapely_polyg_self.intersects(shapely_polyg_other) def overlaps_polyline(self, polyline): """ Test if this Polygon overlaps a given PolyLine at any point. """ # TODO Shapely differentiates between overlaps and intersects # Shapely intersects() allows one to contain the other # Shapely overlaps() means they intersect, but neither contains the other # which is the right choice here? shapely_polyline = polyline_to_shapely_linestring(polyline) shapely_poly_self = polygon_to_shapely_polygon(self) return shapely_poly_self.intersects(shapely_polyline) def closed_loop(self): """ Get a generator for the Locations in this Polygon. Will generate the first item again at the end, simulating the closedness of the Polygon. """ for point in self.locations(): yield point yield self.location_list[0] def intersects(self, polygon): """ Check if this Polygon intersects some other Polygon (ie. overlaps it at any point). """ return self._overlaps_polygon(polygon) class Rectangle(Polygon): """ A special case of Polygon. """ def __init__(self, lower_left, upper_right): """ Create a new Rectangle using a lower left corner Location and an upper right corner Location. """ upper_left = Location(upper_right.get_latitude(), lower_left.get_longitude()) lower_right = Location(lower_left.get_latitude(), upper_right.get_longitude()) locations = [lower_left, upper_left, upper_right, lower_right] super(Rectangle, self).__init__(locations, deep=True) self.lower_left = lower_left self.upper_right = upper_right def get_lower_left(self): """ Get the lower left corner Location of this Rectangle. """ return self.lower_left def get_upper_right(self): """ Get the upper right corner Location of this Rectangle. """ return self.upper_right def location_with_degrees(latitude, longitude): """ Get a new Location with a latitude and longitude specified in degree values. """ latitude = degree_as_dm7(latitude) longitude = degree_as_dm7(longitude) return Location(latitude, longitude) def location_from_packed_int(packed_location): """ Decode a Location object from a packed 64 bit integer. See Location.get_as_packed_int() for more information. """ longitude = packed_location & 0xFFFFFFFF if longitude & 0x80000000 > 0: longitude = longitude - (1 << 32) latitude = (packed_location >> 32) & 0xFFFFFFFF if latitude & 0x80000000 > 0: latitude = latitude - (1 << 32) return Location(latitude, longitude) def degree_as_dm7(degree): """ Given a degree, return the equivalent dm7. Does not perform range validation. Performs integer conversion of the result. """ return int(round(_DM7_PER_DEGREE * degree)) def dm7_as_degree(dm7): """ Given a dm7, return the equivalent degree. Does not perform range validation. """ return float(dm7) / _DM7_PER_DEGREE def decompress_polyline(bytestring): """ Given a PolyLine bytestring obtained using PolyLine.compress(), decompress it and return it as a PolyLine. """ locations = _decompress_bytestring(bytestring) return PolyLine(locations) def decompress_polygon(bytestring): """ Given a PolyLine bytestring obtained using PolyLine.compress(), decompress it and return it as a Polygon. """ locations = _decompress_bytestring(bytestring) return Polygon(locations) def bounds_locations(locations): """ Build a Rectangle that bounds an iterable of Locations. """ yielded_at_least_one = False lower_lat = None upper_lat = None left_lon = None right_lon = None for location in locations: yielded_at_least_one = True latitude = location.get_latitude() longitude = location.get_longitude() if lower_lat is None or latitude < lower_lat: lower_lat = latitude if upper_lat is None or latitude > upper_lat: upper_lat = latitude if left_lon is None or longitude < left_lon: left_lon = longitude if right_lon is None or longitude > right_lon: right_lon = longitude if not yielded_at_least_one: raise ValueError('location iterable must yield at least one value') return Rectangle(Location(lower_lat, left_lon), Location(upper_lat, right_lon)) def bounds_atlasentities(entities): """ Build a Rectangle that bounds an iterable of AtlasEntities. """ yielded_at_least_one = False lower_lat = None upper_lat = None left_lon = None right_lon = None for entity in entities: yielded_at_least_one = True for location in entity.bounds().locations(): latitude = location.get_latitude() longitude = location.get_longitude() if lower_lat is None or latitude < lower_lat: lower_lat = latitude if upper_lat is None or latitude > upper_lat: upper_lat = latitude if left_lon is None or longitude < left_lon: left_lon = longitude if right_lon is None or longitude > right_lon: right_lon = longitude if not yielded_at_least_one: raise ValueError('entity iterable must yield at least one value') return Rectangle(Location(lower_lat, left_lon), Location(upper_lat, right_lon)) def location_haversine_distance(location1, location2): """ Given two locations, compute the Haversine distance between them. This is the great circle distance between the points on the sphere of the Earth. See https://en.wikipedia.org/wiki/Haversine_formula. """ mean_radius = 6371000 # meters lat1 = convert_to_radians(location1.get_latitude_deg()) lat2 = convert_to_radians(location2.get_latitude_deg()) delta_lat = lat2 - lat1 delta_lon = convert_to_radians(location2.get_longitude_deg()) - convert_to_radians( location1.get_longitude_deg()) hav = (math.sin(delta_lat / 2)** 2) + math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2)**2) constant = 2 * math.atan2(math.sqrt(hav), math.sqrt(1 - hav)) return constant * mean_radius def convert_to_radians(degree): """ Convert an angle in degrees to the equivalent angle in radians. """ return degree * (math.pi / 180) def convert_to_degrees(radian): """ Convert an angle in radians to the equivalent angle in degrees. """ return radian * (180 / math.pi) def boundable_to_shapely_box(boundable): """ Convert a pyatlas Boundable type to its Shapely Polygon representation. The Shapely Polygon will always be a rectangle. """ return polygon_to_shapely_polygon(boundable.bounds()) def polygon_to_shapely_polygon(polygon): """ Convert a Polygon to its Shapely Polygon representation. """ shapely_points = [] for location in polygon.locations(): shapely_points.append(location_to_shapely_point(location)) return shapely.geometry.Polygon(shapely.geometry.LineString(shapely_points)) def polygon_to_shapely_polygon_wkt_compat(polygon): """ Convert a Polygon to its Shapely Polygon representation but with WKT compatible coordinates. """ shapely_points = [] for location in polygon.locations(): shapely_points.append(location_to_shapely_point_wkt_compat(location)) return shapely.geometry.Polygon(shapely.geometry.LineString(shapely_points)) def location_to_shapely_point(location): """ Convert a Location to its Shapely Point representation. """ latitude = location.get_latitude() longitude = location.get_longitude() return shapely.geometry.Point(latitude, longitude) def location_to_shapely_point_wkt_compat(location): """ Convert a Location to its Shapely Point representation but with WKT compatible coordinates. """ latitude = location.get_latitude_deg() longitude = location.get_longitude_deg() return shapely.geometry.Point(longitude, latitude) def polyline_to_shapely_linestring(polyline): """ Convert a PolyLine to its Shapely LineString representation. """ shapely_points = [] for location in polyline.locations(): shapely_points.append(location_to_shapely_point(location)) return shapely.geometry.LineString(shapely_points) def polyline_to_shapely_linestring_wkt_compat(polyline): """ Convert a PolyLine to its Shapely LineString representation but with WKT compatible coordinates. """ shapely_points = [] for location in polyline.locations(): shapely_points.append(location_to_shapely_point_wkt_compat(location)) return shapely.geometry.LineString(shapely_points) def _encode_number(number): """ Encode a number as a unicode character. """ number = number << 1 if number < 0: number = ~number encoded = "" while number >= _SIXTH_BIT_MASK: code_point = (_SIXTH_BIT_MASK | number & _FIVE_BIT_MASK) + _ENCODING_OFFSET_MINUS_ONE encoded += chr(code_point) number = _urshift32(number, _BIT_SHIFT) encoded += chr(number + _ENCODING_OFFSET_MINUS_ONE) return encoded def _decompress_bytestring(bytestring): """ Reverse the compression algorithm in PolyLine.compress(). """ precision = pow(10, -1 * _PRECISION) length = len(bytestring) index = 0 latitude = 0 longitude = 0 locations = [] while index < length: shift = 0 result = 0 while True: byte_encoded = bytestring[index] - _ENCODING_OFFSET_MINUS_ONE result |= (byte_encoded & _FIVE_BIT_MASK) << shift shift += _BIT_SHIFT index += 1 if byte_encoded < _SIXTH_BIT_MASK: break if result & 1 > 0: delta_latitude = ~(_urshift32(result, 1)) else: delta_latitude = _urshift32(result, 1) latitude += delta_latitude shift = 0 result = 0 while True: byte_encoded = bytestring[index] - _ENCODING_OFFSET_MINUS_ONE result |= (byte_encoded & _FIVE_BIT_MASK) << shift shift += _BIT_SHIFT index += 1 if byte_encoded < _SIXTH_BIT_MASK: break if result & 1 > 0: delta_longitude = ~(_urshift32(result, 1)) else: delta_longitude = _urshift32(result, 1) longitude += delta_longitude latitude = latitude * precision longitude = longitude * precision # convert lat/lon to dm7 latitude = degree_as_dm7(latitude) longitude = degree_as_dm7(longitude) locations.append(Location(latitude, longitude)) return locations def _urshift32(to_shift, shift_amount): """ Perform a 32 bit unsigned right shift (drag in leading 0s). """ return (to_shift % 0x100000000) >> shift_amount ================================================ FILE: pyatlas/pyatlas/identifier_converters.py ================================================ """ This module defines helpful functions to extract information from Atlas identifiers. """ # Country code and way sectioned identifiers are 3 decimal digits _IDENTIFIER_SCALE = 1000 def get_osm_identifier(full_atlas_identifier): """ Get the OSM identifier from the full Atlas identifier by removing the country code and way section index. Example: Atlas ID: 222222001003 would return OSM ID: 222222 """ full_atlas_identifier = abs(full_atlas_identifier) return full_atlas_identifier // (_IDENTIFIER_SCALE * _IDENTIFIER_SCALE) def get_country_code(full_atlas_identifier): """ Get the country code from the full Atlas identifier. Example: Atlas ID: 222222001003 would return country code: 1 """ full_atlas_identifier = abs(full_atlas_identifier) return (full_atlas_identifier // _IDENTIFIER_SCALE) % _IDENTIFIER_SCALE def get_way_section_index(full_atlas_identifier): """ Get the way section index from the full Atlas identifier. Example: Atlas ID: 222222001003 would return index: 3 """ full_atlas_identifier = abs(full_atlas_identifier) return full_atlas_identifier % _IDENTIFIER_SCALE ================================================ FILE: pyatlas/pyatlas/pyatlas_globalfunc.py ================================================ """ This module contains global utility functions for pyatlas. """ def hello_atlas(): """ Print a welcome message! """ print("Hello Atlas!") ================================================ FILE: pyatlas/pyatlas/spatial_index.py ================================================ """ This module defines the spatial index class for use by the Atlas. """ import ctypes import shapely.geometry from shapely.geos import lgeos as _lgeos import pyatlas.geometry class SpatialIndex(object): """ An optimized container class for making spatial queries on AtlasEntities. The Atlas will automatically construct the SpatialIndices it needs as it receives queries, so it is unlikely you will ever need to create instances of this class manually. """ def __init__(self, parent_atlas, entity_type, initial_entities=None): """ Create a new SpatialIndex. Requires a reference to the parent Atlas of this index, as well as the EntityType of the AtlasEntities it will store. Can optionally accept an iterable of AtlasEntities with which to initialize the index. In order to start using the index, one must specify which backing tree implementation to use. """ self.tree = None self.atlas = parent_atlas self.entity_type = entity_type self.initial_entities = initial_entities def initialize_rtree(self): """ Initialize an R-tree to back this SpatialIndex. The underlying R-tree implementation is immutable, so repeated calls to add() will degrade performance. For more information, see the documentation in the _RTree class. """ self.tree = _RTree(self.initial_entities) def initialize_quadtree(self): """ Intitialize a quadtree to back this SpatialIndex. CURRENTLY UNIMPLEMENTED """ # TODO implement the quadtree raise NotImplementedError('quadtree currently not implemented') def add(self, entity): """ Insert an AtlasEntity into the index. """ if self.tree is None: raise ValueError('must select a tree implementation before using') if entity.bounds() is not None: self.tree.add(entity) else: raise ValueError('bounds cannot be None') def get(self, bounds, predicate=lambda e: True): """ Get a frozenset of AtlasEntities that are within or intersecting some bounds. Can optionally accept a matching predicate function. """ if self.tree is None: raise ValueError('must select a tree implementation before using') results = [] for item_index in self.tree.get(bounds): result = self.atlas.entity(item_index, self.entity_type) if predicate(result): results.append(result) return frozenset(results) class _RTree(object): """ A wrapper class for the _CustomSTRtree implementation, which calls into the native GEOS library using machinery from Shapely. This class stores raw AtlasEntity identifiers without any EntityType information. For this reason, users of _RTree should avoid adding AtlasEntities with different EntityTypes to the same tree. Note also that the underlying tree implementation (GEOS R-tree) this class uses is immutable. _RTree simulates mutability by maintaining a parallel list of elements, and rebuilding the underlying tree on each add. For this reason, extensive use of the add() method is not recommended. For more information on the immutability of the GEOS R-tree, check out this class in the GEOS codebase: https://github.com/OSGeo/geos/blob/master/src/index/strtree/AbstractSTRtree.cpp """ def __init__(self, initial_entities=None): """ Create a new _RTree, optionally with an iterable of initial AtlasEntities. """ self.contents = [] self.tree = None if initial_entities is not None: for entity in initial_entities: self.contents.append(entity) self._construct_tree_from_contents() def _construct_tree_from_contents(self): """ Use the tree's contents list (of AtlasEntities) to rebuild the backing _CustomSTRtree. """ contents_shapely_format = [ pyatlas.geometry.boundable_to_shapely_box(entity) for entity in self.contents ] # pack the arguments in format expected by the _CustomSTRtree hacktree_arguments = [] for entity, cont in zip(self.contents, contents_shapely_format): hacktree_arguments.append((entity.get_identifier(), cont)) self.tree = _CustomSTRtree(hacktree_arguments) def add(self, entity): """ Insert an AtlasEntity into the _RTree. The underlying _CustomSTRtree (which trivially wraps the GEOS STRtree) is immutable once "built", so this method forces a rebuild of the entire tree. The STRtree is "built" once any type of query has been made on it. """ self.contents.append(entity) self._construct_tree_from_contents() def get(self, boundable): """ Given a Boundable object, get a list of the identifiers of all AtlasEntities within the bounds of the Boundable. """ if self.tree is not None: boundable = pyatlas.geometry.boundable_to_shapely_box(boundable) return self.tree.get(boundable) else: raise ValueError('R-tree is empty') class _CustomSTRtree(object): """ Hack re-implementation of the shapely STRtree. Changes the behaviour of the STRtree to allow for insertion of the entity atlas identifier (type long). """ def __init__(self, items): """ Parameter items is a list of tuples, where each tuple lookes like: (entity_id: long, entity_geometry: shapely.geometry.polygon) """ self._n_geoms = len(items) self._tree_handle = shapely.geos.lgeos.GEOSSTRtree_create(max(4, len(items))) for item in items: _lgeos.GEOSSTRtree_insert(self._tree_handle, item[1]._geom, ctypes.py_object(int(item[0]))) geoms = [item[1] for item in items] self._geoms = list(geoms) def __del__(self): if self._tree_handle is not None: _lgeos.GEOSSTRtree_destroy(self._tree_handle) self._tree_handle = None def get(self, geom): """ Get a list of identifiers of AtlasEntities whose bounds intersect the bounds defined by the geom parameter. """ if self._n_geoms == 0: return [] result = [] def callback(item, userdata): identifier = ctypes.cast(item, ctypes.py_object).value result.append(identifier) _lgeos.GEOSSTRtree_query(self._tree_handle, geom._geom, _lgeos.GEOSQueryCallback(callback), None) return result ================================================ FILE: pyatlas/resources/CreateTestAtlas.java ================================================ package org.openstreetmap.atlas.test; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas.AtlasSerializationFormat; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.Maps; /* * Code sample that uses Java Atlas API to create the test.atlas used by Pyatlas unit tests. */ public class CreateTestAtlas { public static void main(final String[] args) { createTestAtlas(); } private static void createTestAtlas() { final PackedAtlasBuilder builder = new PackedAtlasBuilder(); final AtlasMetaData metaData = new AtlasMetaData(AtlasSize.DEFAULT, true, "testCodeVersion", "testDataVersion", "testCountry", "testShardName", Maps.hashMap("metakey", "metaval")); builder.setMetaData(metaData); // add points builder.addPoint(1, new Location(Latitude.degrees(38), Longitude.degrees(-118)), Maps.hashMap()); builder.addPoint(2, new Location(Latitude.degrees(38.01), Longitude.degrees(-118.01)), Maps.hashMap("some_tag", "some_value")); builder.addPoint(3, new Location(Latitude.degrees(38.02), Longitude.degrees(-118.01)), Maps.hashMap("key1", "value2", "key2", "value2")); builder.addPoint(4, new Location(Latitude.degrees(38), Longitude.degrees(-118.05)), Maps.hashMap("", "")); builder.addPoint(5, new Location(Latitude.degrees(38.05), Longitude.degrees(-118.03)), Maps .hashMap("key1", "value2", "key2", "value2", "key3:subkey1", "value3:subvalue1")); // add lines final List shapePoints = new ArrayList<>(); shapePoints.add(new Location(Latitude.degrees(38.02), Longitude.degrees(-118.02))); shapePoints.add(new Location(Latitude.degrees(38.03), Longitude.degrees(-118.01))); shapePoints.add(new Location(Latitude.degrees(38.06), Longitude.degrees(-118.05))); builder.addLine(1, new PolyLine(shapePoints), Maps.hashMap("key1", "value2", "key2", "value2")); shapePoints.clear(); shapePoints.add(new Location(Latitude.degrees(38.06), Longitude.degrees(-118.09))); shapePoints.add(new Location(Latitude.degrees(38.03), Longitude.degrees(-118.1))); builder.addLine(2, new PolyLine(shapePoints), Maps.hashMap("key1", "value2", "key2", "value2")); // add areas shapePoints.add(new Location(Latitude.degrees(38.1), Longitude.degrees(-118.02))); shapePoints.add(new Location(Latitude.degrees(38.2), Longitude.degrees(-118.01))); shapePoints.add(new Location(Latitude.degrees(38.09), Longitude.degrees(-118.05))); builder.addArea(1, new Polygon(shapePoints), Maps.hashMap("key1", "value2", "key2", "value2")); shapePoints.clear(); shapePoints.add(new Location(Latitude.degrees(39.1), Longitude.degrees(-118.06))); shapePoints.add(new Location(Latitude.degrees(39.2), Longitude.degrees(-118.02))); shapePoints.add(new Location(Latitude.degrees(38.09), Longitude.degrees(-118.03))); builder.addArea(2, new Polygon(shapePoints), Maps.hashMap("random key", "value2", "key2", "somenewvalue")); // add nodes builder.addNode(1, new Location(Latitude.degrees(39), Longitude.degrees(-118)), Maps.hashMap()); builder.addNode(2, new Location(Latitude.degrees(39.02), Longitude.degrees(-119.01)), Maps.hashMap("key1", "value2", "key2", "value2")); builder.addNode(3, new Location(Latitude.degrees(39), Longitude.degrees(-119.05)), Maps.hashMap("asd", "asdf")); builder.addNode(4, new Location(Latitude.degrees(39.05), Longitude.degrees(-119.03)), Maps .hashMap("key1", "value2", "key2", "value2", "key3:subkey1", "value3:subvalue1")); // add edges shapePoints.clear(); shapePoints.add(new Location(Latitude.degrees(39), Longitude.degrees(-119.05))); shapePoints.add(new Location(Latitude.degrees(39.02), Longitude.degrees(-119.01))); builder.addEdge(1, new PolyLine(shapePoints), Maps.hashMap("key1", "value2", "key2", "value2")); shapePoints.clear(); shapePoints.add(new Location(Latitude.degrees(39), Longitude.degrees(-118))); shapePoints.add(new Location(Latitude.degrees(39.05), Longitude.degrees(-119.03))); builder.addEdge(2, new PolyLine(shapePoints), Maps.hashMap("key5", "asdsavalue2", "key2", "value2")); shapePoints.clear(); shapePoints.add(new Location(Latitude.degrees(39.05), Longitude.degrees(-119.03))); shapePoints.add(new Location(Latitude.degrees(39), Longitude.degrees(-119.05))); builder.addEdge(3, new PolyLine(shapePoints), Maps.hashMap("key5", "asdsavalue2", "key2", "value2")); // add relations RelationBean bean = new RelationBean(); bean.addItem(1L, "node 1", ItemType.NODE); bean.addItem(2L, "an edge", ItemType.EDGE); builder.addRelation(1, 1, bean, Maps.hashMap("type", "road", "status", "foo")); bean = new RelationBean(); bean.addItem(1L, "a point", ItemType.POINT); bean.addItem(2L, "another point", ItemType.POINT); builder.addRelation(2, 2, bean, Maps.hashMap("key5", "qwertyvalue2", "key5", "asdvalue2")); final PackedAtlas atlas = (PackedAtlas) builder.get(); atlas.setSaveSerializationFormat(AtlasSerializationFormat.PROTOBUF); final File resource = new File("test.atlas"); System.out.println("Test atlas dumped to: " + resource.getAbsolutePath()); atlas.save(resource); } } ================================================ FILE: pyatlas/setup.py ================================================ import setuptools with open("README.md", "r") as fh: long_description = fh.read() # The version field is left blank, and is populated automatically by # the 'packagePyatlas' gradle target at build time. The target then resets # the field to blank before completing. setuptools.setup( name="pyatlas", version= author="lucaspcram", author_email="lucaspcram@gmail.com", license="BSD License", description="A simplified Python API for Atlas", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/osmlab/atlas", packages=setuptools.find_packages(exclude=("unit_tests",)), install_requires=[ 'protobuf==3.11.1', 'shapely==1.6.4' ], classifiers=( "Programming Language :: Python :: 3", "Operating System :: OS Independent", "License :: OSI Approved :: BSD License" ), ) ================================================ FILE: pyatlas/style.yapf ================================================ [style] based_on_style = pep8 column_limit = 100 ================================================ FILE: pyatlas/test.sh ================================================ #!/usr/bin/env bash # general case script abort if a command fails # this can be overridden with a custom error message using '|| err_shutdown' set -e set -o pipefail ### define utility functions ### ################################ err_shutdown() { echo "test.sh: ERROR: $1" deactivate exit 1 } ### check to prevent users from running this script directly ### ################################################################ if [ "$1" != "ranFromGradle" ]; then err_shutdown "this script should be run using the atlas gradle task 'testPyatlas'" fi ### set up variables to store directory names ### ################################################# gradle_project_root_dir="$(pwd)" pyatlas_dir="pyatlas" pyatlas_testdir="unit_tests" pyatlas_root_dir="$gradle_project_root_dir/$pyatlas_dir" venv_path="$pyatlas_root_dir/__pyatlas_venv__" ### abort the script if the pyatlas tests folder is not present ### ################################################################### if [ ! -d "$pyatlas_root_dir/$pyatlas_testdir" ]; then err_shutdown "pyatlas tests folder not found" fi ### run the tests ### ##################### # start the venv if [ ! -d "$venv_path" ]; then err_shutdown "missing $venv_path" fi # shellcheck source=/dev/null source "$venv_path/bin/activate" # grab the build version from gradle.properties and inject it into setup.py atlas_version=$(grep "version=" "$gradle_project_root_dir/gradle.properties" | cut -f2 -d "=") # GNU and BSD sed have different "in-place" flag syntax if [ "$(uname)" == "Darwin" ]; then sed -i "" "s/version=.*/version=\"$atlas_version\",/" "$pyatlas_root_dir/setup.py" elif [ "$(uname)" == "Linux" ]; then sed --in-place="" "s/version=.*/version=\"$atlas_version\",/" "$pyatlas_root_dir/setup.py" else err_shutdown "unrecognized platform $(uname)" fi # enter the pyatlas project directory so the unittest code can discover tests pushd "$pyatlas_root_dir" pip install -e "$pyatlas_root_dir" echo "Discovering and running unit tests..." echo "----------------------------------------------------------------------" python -m unittest discover -v -s "$pyatlas_testdir" || err_shutdown "a test failed, aborting early..." # get back to gradle project directory popd # reset version field in setup.py # GNU and BSD sed have different "in-place" flag syntax if [ "$(uname)" == "Darwin" ]; then sed -i "" "s/version=.*/version=/" "$pyatlas_root_dir/setup.py" elif [ "$(uname)" == "Linux" ]; then sed --in-place="" "s/version=.*/version=/" "$pyatlas_root_dir/setup.py" else err_shutdown "unrecognized platform $(uname)" fi # shutdown the venv deactivate ================================================ FILE: pyatlas/unit_tests/test_atlas.py ================================================ import unittest from pyatlas.atlas import Atlas from pyatlas import geometry from pyatlas import atlas_entities from pyatlas.geometry import Rectangle class AtlasTest(unittest.TestCase): def setUp(self): pass def test_lazy_loading(self): atlas = Atlas("resources/test.atlas") _touch_all_atlas_features(atlas) self.assertEqual(atlas.number_of_points(), 5) self.assertEqual(atlas.number_of_lines(), 2) self.assertEqual(atlas.number_of_areas(), 2) self.assertEqual(atlas.number_of_nodes(), 4) self.assertEqual(atlas.number_of_edges(), 3) self.assertEqual(atlas.number_of_relations(), 2) def test_upfront_loading(self): atlas = Atlas("resources/test.atlas", lazy_loading=False) _touch_all_atlas_features(atlas) self.assertEqual(atlas.number_of_points(), 5) self.assertEqual(atlas.number_of_lines(), 2) self.assertEqual(atlas.number_of_areas(), 2) self.assertEqual(atlas.number_of_nodes(), 4) self.assertEqual(atlas.number_of_edges(), 3) self.assertEqual(atlas.number_of_relations(), 2) def test_point_spatial_index(self): atlas = Atlas("resources/test.atlas") lower_left = geometry.location_with_degrees(37, -118.02) upper_right = geometry.location_with_degrees(39, -118) # NOTE point 1 does not show up in the results because it lies on the polygon border test_results = atlas.points_within(Rectangle(lower_left, upper_right)) self.assertEqual({atlas.point(2), atlas.point(3)}, test_results) test_results = atlas.points_within( Rectangle(lower_left, upper_right), lambda p: p.get_identifier() % 2 != 0) self.assertEqual({atlas.point(3)}, test_results) test_results = atlas.points_at(geometry.location_with_degrees(38, -118)) self.assertEqual({atlas.point(1)}, test_results) def test_line_spatial_index(self): atlas = Atlas("resources/test.atlas") test_location = geometry.location_with_degrees(38.02, -118.02) test_results = atlas.lines_containing(test_location) self.assertEqual({atlas.line(1)}, test_results) poly = Rectangle( geometry.location_with_degrees(38, -118), geometry.location_with_degrees(39, -119)) test_results = atlas.lines_intersecting(poly) self.assertEqual({atlas.line(1), atlas.line(2)}, test_results) def test_area_spatial_index(self): atlas = Atlas("resources/test.atlas") test_location = geometry.location_with_degrees(38.15, -118.03) test_results = atlas.areas_covering(test_location) self.assertEqual({atlas.area(2)}, test_results) test_results = atlas.areas_intersecting(atlas.area(2).as_polygon()) self.assertEqual({atlas.area(1), atlas.area(2)}, test_results) def test_node_spatial_index(self): atlas = Atlas("resources/test.atlas") lower_left = geometry.location_with_degrees(39, -119.04) upper_right = geometry.location_with_degrees(39.05, -119) # NOTE node 4 does not show up in results because it lies on the the polygon border test_results = atlas.nodes_within(Rectangle(lower_left, upper_right)) self.assertEqual({atlas.node(2)}, test_results) test_results = atlas.nodes_within( Rectangle(lower_left, upper_right), lambda n: n.get_identifier() == 3) self.assertEqual(frozenset(), test_results) test_results = atlas.nodes_at(geometry.location_with_degrees(39, -119.05)) self.assertEqual({atlas.node(3)}, test_results) def test_edge_spatial_index(self): atlas = Atlas("resources/test.atlas") test_location = geometry.location_with_degrees(39, -119.05) test_results = atlas.edges_containing(test_location) self.assertEqual({atlas.edge(1), atlas.edge(3)}, test_results) poly = Rectangle( geometry.location_with_degrees(38, -120), geometry.location_with_degrees(40, -117)) test_results = atlas.edges_intersecting(poly) self.assertEqual({atlas.edge(1), atlas.edge(2), atlas.edge(3)}, test_results) def test_relation_spatial_index(self): atlas = Atlas("resources/test.atlas") lower_left = geometry.location_with_degrees(37.999, -118.001) upper_right = geometry.location_with_degrees(38.001, -117.999) test_results = atlas.relations_with_entities_intersecting( Rectangle(lower_left, upper_right)) self.assertEqual({atlas.relation(2)}, test_results) def _touch_all_atlas_features(atlas): for point in atlas.points(): string = point.__str__() for line in atlas.lines(): string = line.__str__() for area in atlas.areas(): string = area.__str__() for node in atlas.nodes(): string = node.__str__() for edge in atlas.edges(): string = edge.__str__() for relation in atlas.relations(): string = relation.__str__() string = str(atlas.point(1)) string = str(atlas.line(1)) string = str(atlas.area(1)) string = str(atlas.node(1)) string = str(atlas.edge(1)) string = str(atlas.relation(1)) if __name__ == "__main__": unittest.main() ================================================ FILE: pyatlas/unit_tests/test_identifier_converters.py ================================================ import unittest from pyatlas import identifier_converters class IdentifierConvertersTest(unittest.TestCase): def setUp(self): pass def test_osm_conversion(self): atlas_id = 222222000000 osm_id = 222222 self.assertEqual(osm_id, identifier_converters.get_osm_identifier(atlas_id)) atlas_id = 123001002 osm_id = 123 self.assertEqual(osm_id, identifier_converters.get_osm_identifier(atlas_id)) atlas_id = 3101220 osm_id = 3 self.assertEqual(osm_id, identifier_converters.get_osm_identifier(atlas_id)) atlas_id = -222222000001 osm_id = 222222 self.assertEqual(osm_id, identifier_converters.get_osm_identifier(atlas_id)) def test_country_code_conversion(self): atlas_id = 222222000000 country_code = 0 self.assertEqual(country_code, identifier_converters.get_country_code(atlas_id)) atlas_id = 123001002 country_code = 1 self.assertEqual(country_code, identifier_converters.get_country_code(atlas_id)) atlas_id = 3101220 country_code = 101 self.assertEqual(country_code, identifier_converters.get_country_code(atlas_id)) atlas_id = -222222002001 country_code = 2 self.assertEqual(country_code, identifier_converters.get_country_code(atlas_id)) def test_way_section_conversion(self): atlas_id = 222222000000 way_section = 0 self.assertEqual(way_section, identifier_converters.get_way_section_index(atlas_id)) atlas_id = 123001002 way_section = 2 self.assertEqual(way_section, identifier_converters.get_way_section_index(atlas_id)) atlas_id = 3101220 way_section = 220 self.assertEqual(way_section, identifier_converters.get_way_section_index(atlas_id)) atlas_id = -222222002001 way_section = 1 self.assertEqual(way_section, identifier_converters.get_way_section_index(atlas_id)) ================================================ FILE: pyatlas/unit_tests/test_location.py ================================================ import unittest from pyatlas import geometry from pyatlas.geometry import Location, Rectangle class LocationTest(unittest.TestCase): def setUp(self): pass def test_location_packing(self): testlocation = Location(1, 1) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) testlocation = Location(1, -3) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) testlocation = Location(-3, -3) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) testlocation = Location(-900000000, 450000000) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) testlocation = Location(900000000, -1800000000) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) testlocation = Location(900000000, 1800000000 - 1) self.assertEqual(testlocation, geometry.location_from_packed_int(testlocation.get_as_packed_int())) def test_location_conversion(self): loc_deg = 45.0 loc_dm7 = 450000000 self.assertEqual(loc_deg, geometry.dm7_as_degree(loc_dm7)) self.assertEqual(loc_dm7, geometry.degree_as_dm7(loc_deg)) loc_deg = 90 loc_dm7 = 900000000 self.assertEqual(loc_deg, geometry.dm7_as_degree(loc_dm7)) self.assertEqual(loc_dm7, geometry.degree_as_dm7(loc_deg)) loc_deg = -30 loc_dm7 = -300000000 self.assertEqual(loc_deg, geometry.dm7_as_degree(loc_dm7)) self.assertEqual(loc_dm7, geometry.degree_as_dm7(loc_deg)) loc_deg = -180 loc_dm7 = -1800000000 self.assertEqual(loc_deg, geometry.dm7_as_degree(loc_dm7)) self.assertEqual(loc_dm7, geometry.degree_as_dm7(loc_deg)) loc_deg = 179.9999 loc_dm7 = 1799999000 self.assertEqual(loc_deg, geometry.dm7_as_degree(loc_dm7)) self.assertEqual(loc_dm7, geometry.degree_as_dm7(loc_deg)) def test_location_bounds(self): testlocation = Location(450000000, 450000000) testlocationbounds = Rectangle(testlocation, testlocation) bound = testlocation.bounds() self.assertEqual(bound, testlocationbounds) if __name__ == "__main__": unittest.main() ================================================ FILE: pyatlas/unit_tests/test_polygon_converters.py ================================================ import unittest from pyatlas import geometry from pyatlas.geometry import Location from pyatlas.geometry import Polygon from pyatlas.geometry import PolyLine import shapely.geometry class PolygonConvertersTest(unittest.TestCase): def setUp(self): pass def test_boundable_to_shapely_box(self): loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] bounds = Polygon(loclist).bounds() shapely_box = geometry.boundable_to_shapely_box(bounds) test_against = shapely.geometry.LineString([(0, 0), (450000000, 0), (450000000, 450000000), (0, 450000000)]) test_against = shapely.geometry.Polygon(test_against) self.assertTrue(shapely_box, test_against) def test_polygon_to_shapely_polygon(self): loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] polygon = Polygon(loclist) shapely_poly = geometry.polygon_to_shapely_polygon(polygon) test_against = shapely.geometry.LineString([(0, 0), (400000000, 0), (350000000, 300000000), (450000000, 450000000), (1000, 450000000)]) test_against = shapely.geometry.Polygon(test_against) self.assertTrue(shapely_poly, test_against) def test_location_to_shapely_point(self): l1 = Location(0, 0) l2 = Location(1000, 2000) l3 = Location(50000, -1000000) p1 = geometry.location_to_shapely_point(l1) p2 = geometry.location_to_shapely_point(l2) p3 = geometry.location_to_shapely_point(l3) self.assertEqual(shapely.geometry.Point(0, 0), p1) self.assertEqual(shapely.geometry.Point(1000, 2000), p2) self.assertEqual(shapely.geometry.Point(50000, -1000000), p3) def test_polyline_to_shapely_linestring(self): polyline1 = PolyLine([Location(-1000, -1000), Location(0, 0), Location(5000, 8000)]) linestring1 = geometry.polyline_to_shapely_linestring(polyline1) test_against = shapely.geometry.LineString([(-1000, -1000), (0, 0), (5000, 8000)]) self.assertEqual(linestring1, test_against) ================================================ FILE: pyatlas/unit_tests/test_polyline_polygon.py ================================================ import unittest from pyatlas import geometry from pyatlas.geometry import Location, PolyLine, Polygon, Rectangle class PolyLinePolygonTest(unittest.TestCase): def setUp(self): pass def test_polyline_compression(self): loclist = [Location(1, 1), Location(2, 2), Location(5, 5)] correct_polyline = PolyLine(loclist, deep=True) test_polyline = geometry.decompress_polyline(correct_polyline.compress()) self.assertEqual(correct_polyline, test_polyline) loclist = [ Location(382117269, -1193153616), Location(382117927, -1193152951), Location(382116912, -1193151049), Location(382116546, -1193151382), Location(382116134, -1193150734), Location(382115440, -1193151494) ] correct_polyline = PolyLine(loclist, deep=True) test_polyline = geometry.decompress_polyline(correct_polyline.compress()) self.assertEqual(correct_polyline, test_polyline) def test_polygon_closedness(self): loclist = [ Location(382117269, -1193153616), Location(382117927, -1193152951), Location(382116912, -1193151049), Location(382116546, -1193151382), Location(382116134, -1193150734), Location(382115440, -1193151494) ] correct_polygon = Polygon(loclist, deep=True) closed_list = [] for point in correct_polygon.closed_loop(): closed_list.append(point) self.assertEqual(closed_list[0], closed_list[len(closed_list) - 1]) def test_poly_bounds(self): # create a lopsided PolyLine to test bounding box loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] expected_rect = Rectangle(Location(0, 0), Location(450000000, 450000000)) computed_rect = PolyLine(loclist).bounds() self.assertEqual(expected_rect, computed_rect) # now test again but with a Polygon loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] expected_rect = Rectangle(Location(0, 0), Location(450000000, 450000000)) computed_rect = Polygon(loclist).bounds() self.assertEqual(expected_rect, computed_rect) def test_poly_length(self): loclist = [Location(0, 0), Location(5000000, 5000000)] polyline = PolyLine(loclist, deep=True) self.assertEqual(78626, int(polyline.length())) loclist2 = [Location(0, 0), Location(20000000, 20000000)] polyline = PolyLine(loclist2, deep=True) self.assertEqual(314474, int(polyline.length())) def test_fully_geometrically_encloses_location(self): loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] polygon = Polygon(loclist) point = Location(1200, 1500) self.assertTrue(polygon.fully_geometrically_encloses_location(point)) point = Location(0, 0) self.assertFalse(polygon.fully_geometrically_encloses_location(point)) point = Location(-34, -1) self.assertFalse(polygon.fully_geometrically_encloses_location(point)) def test_overlaps_polyline(self): loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] loclist2 = [Location(1, 1), Location(2000, 3000)] polygon = Polygon(loclist) polyline0 = PolyLine(loclist2) self.assertTrue(polygon.overlaps_polyline(polyline0)) def test_intersects_polygon(self): loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] loclist2 = [Location(1, 1), Location(10, 10), Location(2000, 3000)] polygon = Polygon(loclist) polygon2 = Polygon(loclist2) self.assertTrue(polygon.intersects(polygon2)) def test_intersects_polyline(self): loclist = [ Location(-3, -3), Location(-2, -2), Location(-1, -1), Location(0, 0), Location(1, 1), Location(2, 2), Location(3, 3) ] loclist2 = [ Location(-3, 3), Location(-2, 2), Location(-1, 1), Location(0, 0), Location(-1, 1), Location(-2, 2), Location(-3, 3) ] polyline = PolyLine(loclist) polyline2 = PolyLine(loclist2) self.assertTrue(polyline.intersects_polyline(polyline2)) if __name__ == "__main__": unittest.main() ================================================ FILE: pyatlas/unit_tests/test_rectangle.py ================================================ import unittest from pyatlas.atlas import Atlas from pyatlas import geometry from pyatlas.geometry import Location, Rectangle class RectangleTest(unittest.TestCase): def setUp(self): pass def test_rectangle_construction(self): rect = Rectangle(Location(0, 0), Location(450000000, 450000000)) loop = [] for point in rect.closed_loop(): loop.append(point) # first and last points should be the same self.assertEqual(loop[0], loop[len(loop) - 1]) # test consistency self.assertEqual(loop[0], Location(0, 0)) self.assertEqual(loop[1], Location(450000000, 0)) self.assertEqual(loop[2], Location(450000000, 450000000)) self.assertEqual(loop[3], Location(0, 450000000)) self.assertEqual(loop[4], Location(0, 0)) def test_location_bounding_calculation(self): loclist = [ Location(0, 0), Location(450000000, 0), Location(450000000, 450000000), Location(0, 450000000) ] expected_rect = Rectangle(Location(0, 0), Location(450000000, 450000000)) computed_rect = geometry.bounds_locations(loclist) self.assertEqual(expected_rect, computed_rect) # create a lopsided polygon to test bounding box loclist = [ Location(0, 0), Location(400000000, 0), Location(350000000, 300000000), Location(450000000, 450000000), Location(1000, 450000000) ] expected_rect = Rectangle(Location(0, 0), Location(450000000, 450000000)) computed_rect = geometry.bounds_locations(loclist) self.assertEqual(expected_rect, computed_rect) def test_entity_bounding_calculation_on_relations(self): atlas = Atlas("resources/test.atlas") relation = atlas.relation(1) expected_rect = Rectangle( Location(390000000, -1190300000), Location(390500000, -1180000000)) computed_rect = geometry.bounds_atlasentities([relation]) self.assertEqual(expected_rect, computed_rect) relation = atlas.relation(2) expected_rect = Rectangle( Location(380000000, -1180100000), Location(380100000, -1180000000)) computed_rect = geometry.bounds_atlasentities([relation]) self.assertEqual(expected_rect, computed_rect) if __name__ == "__main__": unittest.main() ================================================ FILE: pyatlas/unit_tests/test_spatial_index.py ================================================ import unittest from pyatlas import geometry from pyatlas.geometry import Rectangle from pyatlas.atlas import Atlas from pyatlas.spatial_index import SpatialIndex from pyatlas.spatial_index import _RTree from pyatlas.atlas_entities import EntityType class SpatialIndexTest(unittest.TestCase): def setUp(self): pass def test_rtree(self): # The bounding box defined in this test should only encompass # test Points 1, 2, and 3 from the test atlas atlas = Atlas("resources/test.atlas") tree = _RTree(atlas.points()) lower_left = geometry.location_with_degrees(37, -118.02) upper_right = geometry.location_with_degrees(39, -118) test_results = [] for element in tree.get(Rectangle(lower_left, upper_right)): test_results.append(element) self.assertEqual({1, 2, 3}, set(test_results)) def test_basic_spatial_index_operations(self): atlas = Atlas("resources/test.atlas") index = SpatialIndex(atlas, EntityType.POINT, atlas.points()) index.initialize_rtree() lower_left = geometry.location_with_degrees(37, -118.02) upper_right = geometry.location_with_degrees(39, -118) test_results = index.get(Rectangle(lower_left, upper_right)) self.assertEqual({atlas.point(2), atlas.point(3), atlas.point(1)}, test_results) test_results = index.get( Rectangle(lower_left, upper_right), lambda p: p.get_identifier() == 2) self.assertEqual({atlas.point(2)}, test_results) ================================================ FILE: pyatlas/venv.sh ================================================ #!/usr/bin/env bash # general case script abort if a command fails # this can be overridden with a custom error message using '|| err_shutdown' set -e set -o pipefail ### define utility functions ### ################################ err_shutdown() { echo "venv.sh: ERROR: $1" deactivate exit 1 } ### check to prevent users from running this script directly ### ################################################################ if [ "$1" != "ranFromGradle" ]; then err_shutdown "this script should be run using the atlas gradle task 'formatPyatlas'" fi ### set up variables to store directory names ### ################################################# gradle_project_root_dir="$(pwd)" pyatlas_dir="pyatlas" pyatlas_srcdir="pyatlas" pyatlas_root_dir="$gradle_project_root_dir/$pyatlas_dir" venv_path="$pyatlas_root_dir/__pyatlas_venv__" ### abort the script if the pyatlas source folder is not present ### #################################################################### if [ ! -d "$pyatlas_root_dir/$pyatlas_srcdir" ]; then err_shutdown "pyatlas source folder not found" fi ### exit the script if the venv folder already exists ### ######################################################### if [ -d "$venv_path" ]; then echo "INFO: $venv_path exists. Proceeding. Run './gradlew cleanPyatlas' to refresh." exit 0 fi ### determine if virtualenv is installed ### ############################################ if command -v virtualenv; then virtualenv_command="$(command -v virtualenv)" else err_shutdown "'command -v virtualenv' returned non-zero exit status" fi ### set up the virtual environment ### ###################################### echo "Setting up pyatlas venv..." venv_path="$pyatlas_root_dir/__pyatlas_venv__" if ! ${virtualenv_command} --python=python3 "$venv_path"; then err_shutdown "virtualenv command returned non-zero exit status" fi ================================================ FILE: pyatlas/yapf_format.py ================================================ import sys import os from yapf.yapflib.yapf_api import FormatFile from yapf.yapflib.yapf_api import FormatCode # NOTE: If a source file does not end with a newline, the formatter will # complain, but fail to actually fix the issue. To resolve, manually # insert a newline at the end of the file. def main(argv): srcdir = argv[1] mode = argv[2] violation_detected = False for file_to_style in os.listdir(srcdir): if file_to_style.endswith('.py'): filepath = os.path.join(srcdir, file_to_style) if mode == "CHECK": if detect_formatting_violation(filepath): print(str(argv[0]) + ": ERROR: formatting violation detected in " + str(file_to_style)) violation_detected = True elif mode == "APPLY": if detect_formatting_violation(filepath): print(str(file_to_style) + ": found issue, reformatting...") FormatFile(filepath, in_place=True, style_config='style.yapf') violation_detected = True else: print("ERROR: invalid mode " + str(mode)) exit(1) if mode == 'CHECK' and violation_detected: exit(1) elif not violation_detected: print(str(argv[0]) + " INFO: all formatting for targets in " + str(argv[1]) + " OK!") def detect_formatting_violation(filepath): original = read_file_contents(filepath) reformatted = FormatFile(filepath, style_config='style.yapf') if original != reformatted[0]: print(FormatCode(original, filename=filepath, print_diff=True, style_config='style.yapf')[0]) return True return False def read_file_contents(filepath): with open(filepath, 'r') as srcfile: filecontents = srcfile.read() return filecontents if __name__ == "__main__": if len(sys.argv) < 3: print("usage: " + str(sys.argv[0]) + " ") exit(1) main(sys.argv) ================================================ FILE: sample/Readme.md ================================================ # Sample Project - Copy the `sample` project folder locally, and open it as a Gradle project in your favorite IDE. - Grab one or more Atlas files from the links at the top of [that](/src/main/java/org/openstreetmap/atlas/geography/atlas#using-atlas) readme - Open the [`Sample.java`](src/main/java/org/openstreetmap/atlas/sample/Sample.java) file, and take a look at what it is doing. - Run the `Sample` class with the following switch: ``` -atlas=/path/to/some.atlas ``` ================================================ FILE: sample/build.gradle ================================================ plugins { id 'java' } repositories { // For geotools maven { url "http://repo.osgeo.org/repository/release/" } mavenCentral() } dependencies { compile 'org.slf4j:slf4j-api:1.7.12' compile 'org.slf4j:slf4j-log4j12:1.7.12' compile 'log4j:log4j:1.2.17' compile 'org.openstreetmap.atlas:atlas:6.3.0' } ================================================ FILE: sample/settings.gradle ================================================ rootProject.name = 'atlas-sample' ================================================ FILE: sample/src/main/java/org/openstreetmap/atlas/sample/Sample.java ================================================ package org.openstreetmap.atlas.sample; import java.nio.file.Path; import java.util.Comparator; import java.util.Random; import java.util.SortedSet; import java.util.TreeSet; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class Sample extends Command { private static final Logger logger = LoggerFactory.getLogger(Sample.class); private static final int VALENCE_CUTOFF = 4; private static final int PRINT_CUTOFF = 10; private static final Switch ATLAS = new Switch<>("atlas", "Path to the Atlas file to load", value -> new File(Path.of(value)), Optionality.REQUIRED); private static final Switch SHARDING = new Switch<>("sharding", "Sharding tree to load", Sharding::forString, Optionality.OPTIONAL, "geohash@8"); public static void main(final String[] args) { new Sample().runWithoutQuitting(args); } @Override protected int onRun(final CommandMap command) { // Get resource from command final Resource atlasResource = (Resource) command.get(ATLAS); logger.info(""); logger.info("Loading {}", atlasResource); // Load Atlas object in memory final Time start = Time.now(); final Atlas atlas = new AtlasResourceLoader().load(atlasResource); // Print Atlas summary logger.info(""); logger.info("Loaded Atlas in {}: {}", start.elapsedSince(), atlas); // Count the number of dual carriageway Edges: final long numberOfReverseEdges = Iterables.size(atlas.edges(Edge::hasReverseEdge)); logger.info(""); logger.info("The Atlas has {} dual carriageway edges.", numberOfReverseEdges); // Count the number of Nodes with valence >= VALENCE_CUTOFF (At least VALENCE_CUTOFF edges // connected), and print the most southern ones final Iterable valencePlusNodes = atlas .nodes(node -> node.valence() >= VALENCE_CUTOFF); final long numberOfValencePlusNodes = Iterables.size(valencePlusNodes); logger.info(""); logger.info("The Atlas has {} nodes with valence greater than or equal to {}.", numberOfValencePlusNodes, VALENCE_CUTOFF); final Comparator latitudeComparator = (node1, node2) -> { final Latitude latitude1 = node1.getLocation().getLatitude(); final Latitude latitude2 = node2.getLocation().getLatitude(); return Double.compare(latitude1.asDegrees(), latitude2.asDegrees()); }; final SortedSet sortedValencePlusNodes = new TreeSet<>(latitudeComparator); valencePlusNodes.forEach(sortedValencePlusNodes::add); logger.info(""); logger.info("{} southernmost nodes with valence greater than or equal to {}:", PRINT_CUTOFF, VALENCE_CUTOFF); sortedValencePlusNodes.stream().limit(PRINT_CUTOFF) .forEach(node -> logger.info("{}", node)); // Take a random edge and see what sharding box it intersects final Sharding sharding = (Sharding) command.get(SHARDING); final Edge randomEdge = Iterables.stream(atlas.edges()).collectToList() .get(new Random().nextInt((int) atlas.numberOfEdges())); final Iterable shards = sharding.shardsIntersecting(randomEdge.asPolyLine()); logger.info(""); logger.info("Shards intersecting Edge {}:", randomEdge.getIdentifier()); shards.forEach(shard -> logger.info("{} with shape \"{}\"", shard, shard.toWkt())); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(ATLAS, SHARDING); } } ================================================ FILE: sample/src/main/resources/log4j.properties ================================================ log4j.rootLogger=INFO, stdout # Direct log messages to stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n ================================================ FILE: scripts/log4j-atlas/log4j.properties ================================================ log4j.rootLogger=ERROR, stdout # Direct log messages to stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n ================================================ FILE: settings.gradle ================================================ rootProject.name = 'atlas' ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/PolygonPerformanceTest.java ================================================ package org.openstreetmap.atlas.geography; import java.util.List; import java.util.Random; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; import org.openstreetmap.atlas.utilities.scalars.Duration; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link Polygon #fullyGeometricallyEncloses(Location)} performance test. * * @author mkalender */ public class PolygonPerformanceTest { private static final Logger logger = LoggerFactory.getLogger(PolygonPerformanceTest.class); private static final int size = 5_000; private static final Random random = new Random(); private static void testCoverPerformanceHelper(final int size, final Rectangle polygonBounds, final Function locationRetrievalFunction, final Function extraAssertFunction) throws Exception { final int iteration = 1_000; final int polygonPointBound = 100; final int polygonMinPointCount = 3; final List randomPolygons = Stream .generate(() -> Polygon.random( random.nextInt(polygonPointBound) + polygonMinPointCount, polygonBounds)) .limit(size).collect(Collectors.toList()); double overallTotalTimeWithBoundCheck = 0; double overallTotalTimeWithoutBoundCheck = 0; for (int i = 0; i < iteration; i++) { double totalTimeWithBoundCheck = 0; final double totalTimeWithoutBoundCheck = 0; for (int j = 0; j < size; j++) { final Polygon polygon = randomPolygons.get(j); // Use location retrieval method, but send a copy of Polygon final Location location = locationRetrievalFunction .apply(new Polygon(polygon.getPoints())); // Do enclosure check final Time timer = Time.now(); final boolean resultWithBoundCheck = polygon.fullyGeometricallyEncloses(location); final Duration duration = timer.elapsedSince(); totalTimeWithBoundCheck += duration.asMilliseconds(); // Do extra checks Assert.assertTrue(extraAssertFunction.apply(resultWithBoundCheck)); } overallTotalTimeWithBoundCheck += totalTimeWithBoundCheck; overallTotalTimeWithoutBoundCheck += totalTimeWithoutBoundCheck; } logger.info( "Overall time with bound check: {} ms, without bound check: {} ms (iteration: {}) \n", overallTotalTimeWithBoundCheck, overallTotalTimeWithoutBoundCheck, iteration); } @Ignore @Test public void testPerformance() throws Exception { // Rectangles to test final Rectangle waState = Rectangle.forCorners(Location.forString("45.53714, -123.70605"), Location.forString("48.93693, -117.13623")); final Rectangle waStateNorthBorder = Rectangle.forCorners( Location.forString("45.53714, -123.70605"), Location.forString("45.53814, -117.13623")); final Rectangle coState = Rectangle.forCorners(Location.forString("36.87962, -108.7207"), Location.forString("40.6473, -102.30469")); // Case 1 logger.info("Testing locations completely outside of the polygon's bounding box"); logger.info("Size: {}, polygon bounds: {}, location bounds: {}", size, waState, coState); testCoverPerformanceHelper(size, waState, polygon -> Location.random(coState), result -> !result); // Case 2 logger.info("Testing locations on the bounding box border"); logger.info("Size: {}, polygon bounds: {}, location bounds: {}", size, waState, waStateNorthBorder); testCoverPerformanceHelper(size, waState, polygon -> Location.random(waStateNorthBorder), result -> true); // Case 3 logger.info("Testing locations within the bounding box but outside of the polygon"); logger.info("Size: {}, polygon bounds: {}, location bounds: {}", size, waState, waState); testCoverPerformanceHelper(size, waState, polygon -> { // Find a location that is in the same bounding box, but outside polygon Location randomLocation; do { randomLocation = Location.random(waState); } while (polygon.fullyGeometricallyEncloses(randomLocation)); return randomLocation; }, result -> !result); // Case 4 logger.info("Testing locations point on the polygon"); logger.info("Size: {}, polygon bounds: {}", size, waState); testCoverPerformanceHelper(size, waState, polygon -> { final List polygonPoints = polygon.getPoints(); final int aIndex = random.nextInt(polygonPoints.size()); return polygonPoints.get(aIndex); }, result -> true); // Case 5 logger.info("Testing locations inside the polygon"); logger.info("Size: {}, polygon bounds: {}", size, waState); testCoverPerformanceHelper(size, waState, polygon -> { // Find a location that is inside the polygon Location randomLocation; do { randomLocation = Location.random(waState); } while (!polygon.fullyGeometricallyEncloses(randomLocation)); return randomLocation; }, result -> result); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/AtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.io.InputStream; import java.util.function.Supplier; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.OsmPbfIngestIntegrationTest; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author tony */ public class AtlasIntegrationTest { private static final Logger logger = LoggerFactory.getLogger(AtlasIntegrationTest.class); public static Atlas loadCuba() { final Time start = Time.now(); final Supplier supplier = () -> AtlasIntegrationTest.class .getResourceAsStream("CUB_7-37-56.txt.gz"); final InputStreamResource resource = new InputStreamResource(supplier); resource.setDecompressor(Decompressor.GZIP); final Atlas result = new TextAtlasBuilder().read(resource); logger.info("Loaded a Cuba slice in {}", start.elapsedSince()); return result; } protected Atlas loadBahamas(final Polygon polygon) { final String path = OsmPbfIngestIntegrationTest.class.getResource("BHS-6-18-27.pbf") .getPath(); final AtlasLoadingOption loadingOption = AtlasLoadingOption.createOptionWithOnlySectioning() .setLoadWaysSpanningCountryBoundaries(false); final Atlas atlas = new RawAtlasGenerator(new File(path), loadingOption, MultiPolygon.forPolygon(polygon)).build(); return new AtlasSectionProcessor(atlas, loadingOption).run(); } protected Atlas loadBelizeRaw(final Polygon polygon, final AtlasLoadingOption atlasLoadingOption) { final String path = OsmPbfIngestIntegrationTest.class .getResource("BLZ_raw_08242015.osm.pbf").getPath(); Atlas atlas = new RawAtlasGenerator(new File(path), atlasLoadingOption, MultiPolygon.forPolygon(polygon)).build(); if (atlasLoadingOption.isCountrySlicing()) { atlas = new RawAtlasSlicer(atlasLoadingOption, atlas).slice(); } if (atlasLoadingOption.isWaySectioning()) { atlas = new AtlasSectionProcessor(atlas, atlasLoadingOption).run(); } return atlas; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/SubAtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; /** * @author matthieun */ public class SubAtlasIntegrationTest extends AtlasIntegrationTest { @Test public void testSubCuba() { final Atlas cuba = loadCuba(); final Atlas sub = cuba .subAtlas(Rectangle.forCorners(Location.forString("20.049468, -74.368043"), Location.forString("20.382402, -74.077814")), AtlasCutType.SOFT_CUT) .orElseThrow(() -> new CoreException("SubAtlas was not present.")); Assert.assertEquals(523, sub.metaData().getSize().getNodeNumber()); Assert.assertEquals(1344, sub.metaData().getSize().getEdgeNumber()); Assert.assertEquals(223, sub.metaData().getSize().getAreaNumber()); Assert.assertEquals(18, sub.metaData().getSize().getLineNumber()); Assert.assertEquals(28, sub.metaData().getSize().getPointNumber()); Assert.assertEquals(2, sub.metaData().getSize().getRelationNumber()); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/builder/text/TextAtlasBuilderIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.text; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class TextAtlasBuilderIntegrationTest { private static final Logger logger = LoggerFactory .getLogger(TextAtlasBuilderIntegrationTest.class); @Test public void testLoad() { final Atlas atlas = AtlasIntegrationTest.loadCuba(); logger.info("{}", atlas.metaData()); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/delta/AtlasDeltaGeoJsonIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.delta; import org.hamcrest.CoreMatchers; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; /** * @author hallahan */ public class AtlasDeltaGeoJsonIntegrationTest { private Atlas before; private Atlas after; private AtlasDelta delta; /** * Tries parsing the GeoJSON string. We then check a few things about it, such as if it has the * applicable diff properties. Also, we count the number of features. */ @Test public void parseGeoJson() { final String geoJsonStr = this.delta.toGeoJson(); final JsonObject geoJson = new JsonParser().parse(geoJsonStr).getAsJsonObject(); final JsonArray features = geoJson.getAsJsonArray("features"); int idx = 0; for (; idx < features.size(); ++idx) { final JsonObject feature = features.get(idx).getAsJsonObject(); final JsonObject properties = feature.getAsJsonObject("properties"); final JsonElement diffVal = properties.get("diff"); Assert.assertNotNull(diffVal); final JsonElement diffReasonVal = properties.get("diff:reason"); Assert.assertNotNull(diffReasonVal); final JsonElement diffTypeVal = properties.get("diff:type"); Assert.assertNotNull(diffTypeVal); // a diff property should be before or after. Assert.assertThat(diffVal.getAsString(), CoreMatchers.anyOf(CoreMatchers.is("BEFORE"), CoreMatchers.is("AFTER"))); // Make sure we have a reason and a type. Assert.assertTrue(diffReasonVal.getAsString().length() > 0); Assert.assertTrue(diffTypeVal.getAsString().length() > 0); } Assert.assertEquals(47646, idx); } @Before public void readAtlases() { this.before = new TextAtlasBuilder() .read(new InputStreamResource(() -> AtlasDeltaIntegrationTest.class .getResourceAsStream("DMA_9-168-233-base.txt.gz")) .withDecompressor(Decompressor.GZIP).withName("DMA_9-168-233-base.txt.gz")); this.after = new TextAtlasBuilder() .read(new InputStreamResource(() -> AtlasDeltaIntegrationTest.class .getResourceAsStream("DMA_9-168-233-alter.txt.gz")) .withDecompressor(Decompressor.GZIP) .withName("DMA_9-168-233-alter.txt.gz")); this.delta = new AtlasDelta(this.before, this.after, false).generate(); } /** * This is a basic test that should start failing if you change what the delta GeoJSON looks * like. */ @Test public void testGeoJson() { final String geoJson = this.delta.toGeoJson(); Assert.assertEquals(22424431, geoJson.length()); } @Test public void testRelationsGeoJson() { final String relationsGeoJson = this.delta.toRelationsGeoJson(); Assert.assertEquals(454484, relationsGeoJson.length()); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/delta/AtlasDeltaIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.delta; import java.util.SortedSet; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class AtlasDeltaIntegrationTest { private static final Logger logger = LoggerFactory.getLogger(AtlasDeltaIntegrationTest.class); @Test public void testDiff() { final Atlas before = new TextAtlasBuilder() .read(new InputStreamResource(() -> AtlasDeltaIntegrationTest.class .getResourceAsStream("DMA_9-168-233-base.txt.gz")) .withDecompressor(Decompressor.GZIP).withName("DMA_9-168-233-base.txt.gz")); final Atlas after = new TextAtlasBuilder() .read(new InputStreamResource(() -> AtlasDeltaIntegrationTest.class .getResourceAsStream("DMA_9-168-233-alter.txt.gz")) .withDecompressor(Decompressor.GZIP) .withName("DMA_9-168-233-alter.txt.gz")); final AtlasDelta delta = new AtlasDelta(before, after, true).generate(); final SortedSet differences = delta.getDifferences(); final long size = differences.size(); final long sizeNodes = differences.stream() .filter(diff -> diff.getItemType() == ItemType.NODE).count(); final long sizeEdges = differences.stream() .filter(diff -> diff.getItemType() == ItemType.EDGE).count(); final long sizeAreas = differences.stream() .filter(diff -> diff.getItemType() == ItemType.AREA).count(); final long sizeLines = differences.stream() .filter(diff -> diff.getItemType() == ItemType.LINE).count(); final long sizePoints = differences.stream() .filter(diff -> diff.getItemType() == ItemType.POINT).count(); final long sizeRelations = differences.stream() .filter(diff -> diff.getItemType() == ItemType.RELATION).count(); logger.info("Size of the Delta: {}", size); logger.info("Size of the Delta Nodes: {}", sizeNodes); logger.info("Size of the Delta Edges: {}", sizeEdges); logger.info("Size of the Delta Areas: {}", sizeAreas); logger.info("Size of the Delta Lines: {}", sizeLines); logger.info("Size of the Delta Points: {}", sizePoints); logger.info("Size of the Delta Relations: {}", sizeRelations); Assert.assertEquals(33475, size); Assert.assertEquals(3519, sizeNodes); Assert.assertEquals(16239, sizeEdges); Assert.assertEquals(13285, sizeAreas); Assert.assertEquals(94, sizeLines); Assert.assertEquals(324, sizePoints); Assert.assertEquals(14, sizeRelations); Assert.assertEquals(size, sizeNodes + sizeEdges + sizeAreas + sizeLines + sizePoints + sizeRelations); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicAtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Optional; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.dynamic.policy.DynamicAtlasPolicy; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.geography.sharding.SlippyTileSharding; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author matthieun */ public class DynamicAtlasIntegrationTest { private static final DynamicAtlasPolicy POLICY = new DynamicAtlasPolicy(shard -> { final String fileName = "DMA_" + shard.getName() + ".atlas.txt.gz"; return Optional.of(new TextAtlasBuilder().read(new InputStreamResource( () -> DynamicAtlasIntegrationTest.class.getResourceAsStream(fileName)) .withName(fileName).withDecompressor(Decompressor.GZIP))); }, new SlippyTileSharding(9), SlippyTile.forName("9-168-234"), Rectangle.MAXIMUM); @Test public void testAreas() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(39672, Iterables.size(atlas.areas())); } @Test public void testEdges() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(18936, Iterables.size(atlas.edges())); } @Test public void testLines() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(1572, Iterables.size(atlas.lines())); } @Test public void testNodes() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(8615, Iterables.size(atlas.nodes())); } @Test public void testPoints() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(348, Iterables.size(atlas.points())); } @Test public void testRelations() { final Atlas atlas = new DynamicAtlas(POLICY); Assert.assertEquals(2, Iterables.size(atlas.relations())); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/items/AtlasEntityTypeTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; /** * Tests Atlas's type retrieval * * @author mkalender */ public class AtlasEntityTypeTest extends AtlasIntegrationTest { @Test public void testTypeRetrieval() { final Atlas atlas = loadCuba(); for (final AtlasEntity entity : atlas.entities(entity -> true)) { if (entity instanceof Node) { Assert.assertEquals(((Node) entity).getType(), ItemType.NODE); } else if (entity instanceof Edge) { Assert.assertEquals(((Edge) entity).getType(), ItemType.EDGE); } else if (entity instanceof Area) { Assert.assertEquals(((Area) entity).getType(), ItemType.AREA); } else if (entity instanceof Line) { Assert.assertEquals(((Line) entity).getType(), ItemType.LINE); } else if (entity instanceof Point) { Assert.assertEquals(((Point) entity).getType(), ItemType.POINT); } else if (entity instanceof Relation) { Assert.assertEquals(((Relation) entity).getType(), ItemType.RELATION); } else { Assert.fail("Unknown type"); } } } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/items/complex/boundaries/ComplexBoundariesIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.boundaries; import java.util.ArrayList; import java.util.List; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author matthieun */ public class ComplexBoundariesIntegrationTest { @Rule public final ComplexBoundaryIntegrationTestRule rule = new ComplexBoundaryIntegrationTestRule(); @Test public void testComplexBoundary() { final Atlas atlas = this.rule.getAtlas(); final List badEntities = new ArrayList<>(); final ComplexBoundaryFinder finder = new ComplexBoundaryFinder(); final List result = Iterables.stream(finder.find(atlas, badEntities::add)) .collectToList(); Assert.assertEquals(710, result.size()); final List withCountry = Iterables.stream(result) .filter(boundary -> Iterables.size(boundary.getCountries()) > 0).collectToList(); Assert.assertEquals(2, withCountry.size()); Assert.assertEquals(139, badEntities.size()); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/items/complex/boundaries/ComplexBoundaryIntegrationTestRule.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.boundaries; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.testing.CoreTestRule; import org.openstreetmap.atlas.utilities.testing.TestAtlas; /** * @author matthieun */ public class ComplexBoundaryIntegrationTestRule extends CoreTestRule { @TestAtlas(loadFromTextResource = "HTI-DOM-Boundaries.atlas.txt.gz") private Atlas atlas; public Atlas getAtlas() { return this.atlas; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/multi/MultiAtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.ArrayList; import java.util.List; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.geography.atlas.packed.RandomPackedAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class MultiAtlasIntegrationTest extends AtlasIntegrationTest { private static final Logger logger = LoggerFactory.getLogger(MultiAtlasIntegrationTest.class); private MultiAtlas multi; public static void main(final String[] args) { new MultiAtlasIntegrationTest().loadTest(Boolean.valueOf(args[0]), Integer.valueOf(args[1]), Long.valueOf(args[2])); } public MultiAtlas largeMultiAtlas(final int count, final long eachSize) { final List atlases = new ArrayList<>(); for (int i = 0; i < count; i++) { final long startIdentifier = i > 0 ? i * eachSize - eachSize / 100 : 0; final long size = i > 0 ? eachSize + eachSize / 100 : eachSize; logger.info("Generating sub-atlas with identifiers starting at {} and ending at {}", startIdentifier, startIdentifier + size - 1); atlases.add(RandomPackedAtlasBuilder.generate(size, startIdentifier)); } return new MultiAtlas(atlases, false); } public void loadTest(final boolean overWrite, final int count, final long eachSize) { final File atlasFile = new File( System.getProperty("user.home") + "/projects/data/unitTest/multiatlas.atlas"); final MultiAtlas large; Time start; final Rectangle queryBounds = Location.TEST_5.boxAround(Distance.miles(1)); if (!overWrite && atlasFile.getFile().exists()) { start = Time.now(); large = MultiAtlas.load(atlasFile); logger.info("Finished Loading from atlas file {} in {}", atlasFile, start.elapsedSince()); } else { start = Time.now(); large = largeMultiAtlas(count, eachSize); logger.info("Created MultiAtlas in {}", start.elapsedSince()); start = Time.now(); large.save(atlasFile); logger.info("Finished writing to multiatlas file {} in {}", atlasFile, start.elapsedSince()); } // Edges start = Time.now(); final long edgesSize = Iterables.size(large.edgesIntersecting(queryBounds)); logger.info("Spatial queried and counted {} edges in {}", edgesSize, start.elapsedSince()); start = Time.now(); final long edgesSize2 = Iterables.size(large.edges()); logger.info("Total: Counted {} edges in {}", edgesSize2, start.elapsedSince()); // Nodes start = Time.now(); final long nodesSize = Iterables.size(large.nodesWithin(queryBounds)); logger.info("Spatial queried and counted {} nodes in {}", nodesSize, start.elapsedSince()); start = Time.now(); final long nodesSize2 = Iterables.size(large.nodes()); logger.info("Total: Counted {} nodes in {}", nodesSize2, start.elapsedSince()); } @Test public void testPolygonRetrieval() { final Rectangle bound = new Location(Latitude.degrees(25.0288172), Longitude.degrees(-77.5420233)).boxAround(Distance.miles(1)); final Atlas atlas1 = loadBahamas(bound); final Rectangle overlapBound = new Location(Latitude.degrees(25.0213741), Longitude.degrees(-77.5237397)).boxAround(Distance.miles(1)); final Atlas atlas2 = loadBahamas(overlapBound); final Rectangle noOverlapBound = new Location(Latitude.degrees(24.3973491), Longitude.degrees(-77.8862342)).boxAround(Distance.miles(1)); final Atlas atlas3 = loadBahamas(noOverlapBound); this.multi = new MultiAtlas(atlas1, atlas2, atlas3); final Polygon polygon = this.multi.area(24601488000000L).asPolygon(); Assert.assertEquals(40, Iterables.size(this.multi.edgesIntersecting(polygon))); Assert.assertEquals(8, Iterables.size(this.multi.areasIntersecting(polygon))); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/multi/MultiAtlasIntegrationTestRule.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.testing.CoreTestRule; import org.openstreetmap.atlas.utilities.testing.TestAtlas; /** * @author matthieun */ public class MultiAtlasIntegrationTestRule extends CoreTestRule { @TestAtlas(loadFromTextResource = "DEU_11-1084-708.atlas.txt.gz") private Atlas atlas1; @TestAtlas(loadFromTextResource = "DEU_11-1084-709.atlas.txt.gz") private Atlas atlas2; public Atlas getAtlas1() { return this.atlas1; } public Atlas getAtlas2() { return this.atlas2; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasClonerIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.geography.atlas.delta.AtlasDelta; /** * @author matthieun */ public class PackedAtlasClonerIntegrationTest extends AtlasIntegrationTest { @Test public void cloneRealCountry() { cloneAndCompare(loadCuba()); } @Test public void cloneTest() { for (int i = 0; i < 10; i++) { final Atlas atlas = RandomPackedAtlasBuilder.generate(1000, 0); cloneAndCompare(atlas); } } private void cloneAndCompare(final Atlas atlas) { final PackedAtlasCloner cloner = new PackedAtlasCloner(); final Atlas copy = cloner.cloneFrom(atlas); Assert.assertTrue(new AtlasDelta(atlas, copy).generate().getDifferences().isEmpty()); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.streaming.compression.Compressor; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class PackedAtlasIntegrationTest extends AtlasIntegrationTest { private static final Logger logger = LoggerFactory.getLogger(PackedAtlasIntegrationTest.class); public static void main(final String[] args) { new PackedAtlasIntegrationTest().loadTest(false, 10000); } public void loadTest(final boolean overWrite, final long size) { final File atlasFile = new File( System.getProperty("user.home") + "/projects/data/unitTest/atlas.atlas"); final PackedAtlas large; Time start = Time.now(); final Rectangle queryBoundsEdges = Location.TEST_5.boxAround(Distance.miles(1)); final Rectangle queryBoundsNodes = Location.TEST_5.boxAround(Distance.miles(10)); if (!overWrite && atlasFile.getFile().exists()) { start = Time.now(); large = PackedAtlas.load(atlasFile); logger.info("Finished Loading from atlas file {} in {}", atlasFile, start.elapsedSince()); } else { large = RandomPackedAtlasBuilder.generate(size, 1); start = Time.now(); large.save(atlasFile); logger.info("Finished writing to atlas file {} in {}", atlasFile, start.elapsedSince()); } // Queries start = Time.now(); final long edgesSize = Iterables.size(large.edgesIntersecting(queryBoundsEdges)); logger.info("Spatial queried and counted {} edges in {}", edgesSize, start.elapsedSince()); start = Time.now(); final long nodesSize = Iterables.size(large.nodesWithin(queryBoundsNodes)); logger.info("Spatial queried and counted {} nodes in {}", nodesSize, start.elapsedSince()); start = Time.now(); final long nodes = Iterables.size(large.nodes()); logger.info("Total: Counted {} nodes in {}", nodes, start.elapsedSince()); start = Time.now(); final long edgeShapePoints = Iterables.count(large.edgesIntersecting(queryBoundsEdges), edge -> (long) edge.asPolyLine().size()); logger.info("Spatial queried and counted {} edge shape points in {}", edgeShapePoints, start.elapsedSince()); start = Time.now(); final long edgesSize2 = Iterables.size(large.edges()); logger.info("Total: Counted {} edges in {}", edgesSize2, start.elapsedSince()); start = Time.now(); final long relationsSize = Iterables.size(large.relations()); logger.info("Total: Counted {} relations in {}", relationsSize, start.elapsedSince()); } @Test public void testPolygonRetrieval() { final Rectangle largerBound = new Location(Latitude.degrees(25.0288172), Longitude.degrees(-77.5420233)).boxAround(Distance.miles(1)); final Atlas atlas = loadBahamas(largerBound); final Polygon polygon = atlas.area(24601488000000L).asPolygon(); Assert.assertEquals(40, Iterables.size(atlas.edgesIntersecting(polygon))); Assert.assertEquals(8, Iterables.size(atlas.areasIntersecting(polygon))); } @Test public void testSerialization() { // Raw final ByteArrayResource resource = new ByteArrayResource() .withName("testSerializationByteArray"); final Atlas atlas = RandomPackedAtlasBuilder.generate(1000, 0); atlas.save(resource); final PackedAtlas deserialized = PackedAtlas.load(resource); Assert.assertTrue(atlas.equals(deserialized)); // With compression final ByteArrayResource compressedResource = new ByteArrayResource() .withName("testSerializationByteArrayCompressed"); compressedResource.setCompressor(Compressor.GZIP); compressedResource.setDecompressor(Decompressor.GZIP); atlas.save(compressedResource); final PackedAtlas compressedeserialized = PackedAtlas.load(compressedResource); Assert.assertTrue(atlas.equals(compressedeserialized)); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/pbf/OsmPbfIngestIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.clipping.Clip.ClipType; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.tags.BuildingTag; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.LastEditTimeTag; import org.openstreetmap.atlas.tags.LastEditUserIdentifierTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * @author tony */ public class OsmPbfIngestIntegrationTest extends AtlasIntegrationTest { @Test public void testAreaAndLine() { final Rectangle bound = Location.forString("25.0771736, -77.3597574") .boxAround(Distance.meters(300)); final Atlas atlas = loadBahamas(bound); Assert.assertEquals(28, atlas.numberOfAreas()); Assert.assertEquals(98, atlas.numberOfLines()); final Area smallArea = atlas.area(522592143000000L); Assert.assertEquals(true, BuildingTag.isBuilding(smallArea)); Assert.assertEquals(4, Iterables.size(smallArea)); Assert.assertNotNull(smallArea.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(smallArea.tag(LastEditUserIdentifierTag.KEY)); final Area bigArea = atlas.area(191814889000000L); Assert.assertEquals(false, BuildingTag.isBuilding(bigArea)); Assert.assertEquals("attraction", bigArea.getTags().get("tourism")); Assert.assertEquals("Fort Charlotte", bigArea.getTags().get("name")); Assert.assertEquals(20, Iterables.size(bigArea)); Assert.assertNotNull(bigArea.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(bigArea.tag(LastEditUserIdentifierTag.KEY)); final Line longLine = atlas.line(374341334000000L); Assert.assertEquals("drain", longLine.getTags().get("waterway")); Assert.assertEquals("Storm Water Drain", longLine.getTags().get("name")); Assert.assertEquals(24, Iterables.size(longLine)); Assert.assertNotNull(longLine.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(longLine.tag(LastEditUserIdentifierTag.KEY)); } @Test public void testEdgeAndNode() { final Rectangle bound = Location.forString("25.0693383, -77.3160218") .boxAround(Distance.feet(1)); final Atlas atlas = loadBahamas(bound); Assert.assertEquals(80, atlas.numberOfEdges()); final Edge edgeForward = atlas.edge(63423376000000L); Assert.assertEquals(63423376000000L, edgeForward.getIdentifier()); Assert.assertEquals(HighwayTag.RESIDENTIAL, edgeForward.highwayTag()); Assert.assertEquals(6, Iterables.size(edgeForward.getRawGeometry())); Assert.assertNotNull(edgeForward.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(edgeForward.tag(LastEditUserIdentifierTag.KEY)); final Edge edgeBackward = atlas.edge(-63423376000000L); Assert.assertEquals(-63423376000000L, edgeBackward.getIdentifier()); Assert.assertEquals(HighwayTag.RESIDENTIAL, edgeBackward.highwayTag()); Assert.assertEquals(6, Iterables.size(edgeBackward.getRawGeometry())); Assert.assertNotNull(edgeBackward.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(edgeBackward.tag(LastEditUserIdentifierTag.KEY)); Assert.assertEquals(Iterables.first(edgeForward.getRawGeometry()).get(), Iterables.last(edgeBackward.getRawGeometry()).get()); final Node startNodeOfForwardEdge = atlas.node(786050062000000L); Assert.assertEquals("POINT (-77.3160218 25.0693383)", startNodeOfForwardEdge.getLocation().toString()); Assert.assertEquals(3, startNodeOfForwardEdge.inEdges().size()); Assert.assertTrue( startNodeOfForwardEdge.inEdges().stream().map(edge -> edge.getIdentifier()) .collect(Collectors.toList()).contains(edgeBackward.getIdentifier())); Assert.assertEquals(3, startNodeOfForwardEdge.outEdges().size()); Assert.assertTrue( startNodeOfForwardEdge.outEdges().stream().map(edge -> edge.getIdentifier()) .collect(Collectors.toList()).contains(edgeForward.getIdentifier())); Assert.assertEquals(startNodeOfForwardEdge, edgeForward.start()); Assert.assertNotNull(startNodeOfForwardEdge.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(startNodeOfForwardEdge.tag(LastEditUserIdentifierTag.KEY)); final Node endNodeOfForwardEdge = atlas.node(4354620579000000L); Assert.assertEquals("POINT (-77.3149029 25.0691753)", endNodeOfForwardEdge.getLocation().toString()); Assert.assertEquals(1, endNodeOfForwardEdge.inEdges().size()); Assert.assertTrue(endNodeOfForwardEdge.inEdges().stream().map(edge -> edge.getIdentifier()) .collect(Collectors.toList()).contains(edgeForward.getIdentifier())); Assert.assertEquals(1, endNodeOfForwardEdge.outEdges().size()); Assert.assertTrue(endNodeOfForwardEdge.outEdges().stream().map(edge -> edge.getIdentifier()) .collect(Collectors.toList()).contains(edgeBackward.getIdentifier())); Assert.assertEquals(endNodeOfForwardEdge, edgeForward.end()); Assert.assertNotNull(endNodeOfForwardEdge.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(endNodeOfForwardEdge.tag(LastEditUserIdentifierTag.KEY)); } @Test public void testPoint() { final Rectangle bound = Location.forString("25.0735519,-77.3073068") .boxAround(Distance.inches(1)); final Atlas atlas = loadBahamas(bound); Assert.assertEquals(97, atlas.numberOfPoints()); final Point point = atlas.point(5665510971000000L); Assert.assertEquals(5665510971000000L, point.getIdentifier()); Assert.assertEquals("POINT (-77.3073068 25.0735519)", point.getLocation().toString()); Assert.assertEquals(6, point.getTags().size()); Assert.assertEquals("viewpoint", point.getTags().get("tourism")); Assert.assertNotNull(point.tag(LastEditTimeTag.KEY)); Assert.assertNotNull(point.tag(LastEditUserIdentifierTag.KEY)); } @Test public void testPolygonLoading() { final Rectangle largerBound = Rectangle.forLocated( Location.forString("25.0288172,-77.5420233"), Location.forString("25.0213741,-77.5237397")); final Atlas bigAtlas = loadBahamas(largerBound); final Polygon polygon = bigAtlas.area(24601488000000L).asPolygon(); final Atlas smallAtlas = loadBahamas(polygon); Assert.assertTrue(bigAtlas.numberOfEdges() > smallAtlas.numberOfEdges()); Assert.assertEquals(126, smallAtlas.numberOfEdges()); Assert.assertEquals(13, smallAtlas.numberOfAreas()); } @Test public void testRestrictionAndMultiPolygon() { final Rectangle bound = Location.forString("25.0812893, -77.3216682") .boxAround(Distance.kilometers(0.5)); final Atlas atlas = loadBahamas(bound); final Map relations = new HashMap<>(); atlas.relations().forEach(relation -> relations.put(relation.getIdentifier(), relation)); Assert.assertEquals(38, relations.size()); final Relation relation1 = relations.get(4309052000000L); Assert.assertEquals("restriction", relation1.getTags().get("type")); final RelationMemberList members1 = relation1.members(); Assert.assertEquals(3, members1.size()); Assert.assertEquals(1635708993000000L, members1.get(0).getEntity().getIdentifier()); Assert.assertEquals("via", members1.get(0).getRole()); Assert.assertEquals(150683353000000L, members1.get(1).getEntity().getIdentifier()); Assert.assertEquals("from", members1.get(1).getRole()); Assert.assertEquals(150683354000000L, members1.get(2).getEntity().getIdentifier()); Assert.assertEquals("to", members1.get(2).getRole()); final Relation relation2 = relations.get(1621692000000L); Assert.assertEquals("multipolygon", relation2.getTags().get("type")); final RelationMemberList members2 = relation2.members(); Assert.assertEquals(2, members2.size()); Assert.assertEquals(117365339000000L, members2.get(1).getEntity().getIdentifier()); Assert.assertEquals("outer", members2.get(1).getRole()); Assert.assertEquals(117365338000000L, members2.get(0).getEntity().getIdentifier()); Assert.assertEquals("inner", members2.get(0).getRole()); final Relation relation3 = relations.get(4309051000000L); Assert.assertEquals("restriction", relation3.getTags().get("type")); final RelationMemberList members3 = relation3.members(); Assert.assertEquals(3, members3.size()); Assert.assertEquals(719062796000000L, members3.get(0).getEntity().getIdentifier()); Assert.assertEquals("via", members3.get(0).getRole()); Assert.assertEquals(57942389000000L, members3.get(1).getEntity().getIdentifier()); Assert.assertEquals("from", members3.get(1).getRole()); Assert.assertEquals(318515851000000L, members3.get(2).getEntity().getIdentifier()); Assert.assertEquals("to", members3.get(2).getRole()); } @Test public void testRoute() { // This 250 meters range just covers part of the relation1 final Rectangle bound = Location.forString("26.0845577, -77.5369822") .boxAround(Distance.meters(250)); final Atlas atlas = loadBahamas(bound); final Map relations = new HashMap<>(); atlas.relations().forEach(relation -> relations.put(relation.getIdentifier(), relation)); Assert.assertEquals(9, relations.size()); final Relation relation1 = relations.get(1251624000000L); Assert.assertEquals("route", relation1.getTags().get("type")); Assert.assertEquals("bus", relation1.getTags().get("route")); Assert.assertEquals("Adult Tram", relation1.getTags().get("name")); Assert.assertEquals(23, relation1.members().size()); final Relation relation2 = relations.get(1245746000000L); Assert.assertEquals("route", relation2.getTags().get("type")); Assert.assertEquals("bus", relation2.getTags().get("route")); Assert.assertEquals("Tram", relation2.getTags().get("name")); Assert.assertEquals(18, relation2.members().size()); } @Test public void testWaysSpanningOutsideOfCountry() { final Resource pbf = new InputStreamResource( () -> OsmPbfIngestIntegrationTest.class.getResourceAsStream("CUB_72-111.pbf")); final CountryBoundaryMap map = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> OsmPbfIngestIntegrationTest.class .getResourceAsStream("CUB_osm_boundaries.txt.gz")) .withDecompressor(Decompressor.GZIP)); final SlippyTile tile = SlippyTile.forName("8-72-111"); final MultiPolygon boundary = new JtsPolygonToMultiPolygonConverter() .convert(map.countryBoundary("CUB").get(0)); final MultiPolygon loadingArea = tile.bounds().clip(boundary, ClipType.AND) .getClipMultiPolygon(); final AtlasLoadingOption loadingOption = AtlasLoadingOption.createOptionWithAllEnabled(map); Atlas atlas = new RawAtlasGenerator(pbf, loadingOption, loadingArea).build(); atlas = new RawAtlasSlicer(loadingOption, atlas).slice(); atlas = new AtlasSectionProcessor(atlas, loadingOption).run(); // Make sure that the big bridge over water made it to the Atlas Assert.assertNotNull(atlas.edge(308541861000000L)); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/AtlasSectionProcessorIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.AbstractIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.PaddingIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.WaySectionIdentifierFactory; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author tony */ public class AtlasSectionProcessorIntegrationTest extends AtlasIntegrationTest { private static Atlas rawAtlas; private static Atlas sectionedAtlas; @Before public void createAtlas() { final Rectangle belizeCity = Rectangle.forLocated( Location.forString("17.521983, -88.213739"), Location.forString("17.491327, -88.178071")); // Both atlases are not country-sliced if (rawAtlas == null) { rawAtlas = loadBelizeRaw(belizeCity, AtlasLoadingOption.createOptionWithNoSlicing() .setLoadWaysSpanningCountryBoundaries(false)); } if (sectionedAtlas == null) { sectionedAtlas = loadBelizeRaw(belizeCity, AtlasLoadingOption.createOptionWithNoSlicing().setWaySectioning(true) .setLoadWaysSpanningCountryBoundaries(false)); } } @Test public void testPrimaryWay() { // Primary way should be sectioned final long primaryWayShouldBeSectioned = 279042065L; final AbstractIdentifierFactory factory = new WaySectionIdentifierFactory( PaddingIdentifierFactory.pad(primaryWayShouldBeSectioned)); Assert.assertEquals(21, Iterables.size(rawAtlas.line(primaryWayShouldBeSectioned).getRawGeometry())); Assert.assertNull(sectionedAtlas.edge(factory.getReferenceIdentifier())); Assert.assertEquals(2, Iterables.size(sectionedAtlas.edge(factory.nextIdentifier()).getRawGeometry())); Assert.assertEquals(2, Iterables.size(sectionedAtlas.edge(factory.nextIdentifier()).getRawGeometry())); Assert.assertEquals(2, Iterables.size(sectionedAtlas.edge(factory.nextIdentifier()).getRawGeometry())); Assert.assertEquals(2, Iterables.size(sectionedAtlas.edge(279042065000015L).getRawGeometry())); Assert.assertEquals(5, Iterables.size(sectionedAtlas.edge(279042065000016L).getRawGeometry())); Assert.assertEquals(2, Iterables.size(sectionedAtlas.edge(279042065000017L).getRawGeometry())); Assert.assertNull(sectionedAtlas.edge(279042065000018L)); } @Test public void testSelfIntersectionRing() { // Way 295734091 will be sectioned into 295734091000001 and 295734091000002. However because // 295734091000002 is a ring, which should be further sectioned into 295734091000003 and // 295734091000004 final long selfIntersectionWay = 295734091; Assert.assertNotNull(rawAtlas.line(selfIntersectionWay)); final AbstractIdentifierFactory factory = new WaySectionIdentifierFactory( PaddingIdentifierFactory.pad(selfIntersectionWay)); Assert.assertNull(sectionedAtlas.edge(factory.getReferenceIdentifier())); Assert.assertNotNull(sectionedAtlas.edge(factory.nextIdentifier())); Assert.assertNotNull(sectionedAtlas.edge(factory.nextIdentifier())); } @Test public void testShapePointOrder() { // The identifier order of split way should be same as the order of shape point for way with // tag oneway=-1 final long reversedWay = 25977940L; final Line nonSplit = rawAtlas.line(reversedWay); Assert.assertNotNull(nonSplit); final AbstractIdentifierFactory factory = new WaySectionIdentifierFactory( PaddingIdentifierFactory.pad(reversedWay)); Assert.assertNull(sectionedAtlas.edge(factory.getReferenceIdentifier())); final Edge firstSplit = sectionedAtlas.edge(factory.nextIdentifier()); final Edge secondSplit = sectionedAtlas.edge(factory.nextIdentifier()); Assert.assertNotNull(firstSplit); Assert.assertNotNull(secondSplit); Assert.assertEquals(nonSplit.asPolyLine().last(), secondSplit.start().getLocation()); Assert.assertEquals(nonSplit.asPolyLine().first(), firstSplit.end().getLocation()); } @Test public void testTotalCounts() { Assert.assertEquals(0, rawAtlas.numberOfEdges()); Assert.assertEquals(2841, sectionedAtlas.numberOfEdges()); Assert.assertEquals(582, rawAtlas.numberOfLines()); Assert.assertEquals(9, sectionedAtlas.numberOfLines()); Assert.assertEquals(0, rawAtlas.numberOfNodes()); Assert.assertEquals(1165, sectionedAtlas.numberOfNodes()); Assert.assertEquals(336, rawAtlas.numberOfAreas()); Assert.assertEquals(336, sectionedAtlas.numberOfAreas()); Assert.assertEquals(1, rawAtlas.numberOfRelations()); Assert.assertEquals(1, sectionedAtlas.numberOfRelations()); Assert.assertEquals(5380, rawAtlas.numberOfPoints()); Assert.assertEquals(82, sectionedAtlas.numberOfPoints()); } @Test public void testWaterWay() { // Waterway (Atlas area) should not be sectioned final long waterway = 25977495L; Assert.assertNotNull(rawAtlas.area(waterway)); Assert.assertNotNull(sectionedAtlas.area(PaddingIdentifierFactory.pad(waterway))); } @Test public void testWayAcrossArea() { // Way across an area should not be sectioned final long wayAcrossArea = 194237644L; Assert.assertEquals(2, Iterables.size(rawAtlas.line(wayAcrossArea).getRawGeometry())); Assert.assertEquals(2, Iterables.size( sectionedAtlas.edge(PaddingIdentifierFactory.pad(wayAcrossArea)).getRawGeometry())); } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/raw/DynamicRawAtlasSectioningTestRule.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.testing.CoreTestRule; import org.openstreetmap.atlas.utilities.testing.TestAtlas; import org.openstreetmap.atlas.utilities.testing.TestAtlas.Line; import org.openstreetmap.atlas.utilities.testing.TestAtlas.Loc; import org.openstreetmap.atlas.utilities.testing.TestAtlas.Point; /** * {@link RawAtlasIntegrationTest} test data. * * @author mgostintsev */ public class DynamicRawAtlasSectioningTestRule extends CoreTestRule { // Fully inside 8-123-122 private static final String ONE = "7.9747091, -6.6837721"; private static final String TWO = "7.9737754, -6.6823776"; // Fully inside 8-123-123 private static final String THREE = "5.7826593, -6.5920337"; private static final String FOUR = "5.7827163, -6.5914695"; // Fully inside 7-62-61 private static final String FIVE = "6.7240498, -3.4840235"; private static final String SIX = "6.7260445, -3.4840096"; // Crossing 8-123-122 into 8-123-123 private static final String SEVEN = "7.0401813, -6.4733942"; private static final String EIGHT = "6.8870387, -6.4641103"; private static final String NINE = " 6.8832099, -6.4636300"; private static final String TEN = "6.8866444, -6.4654545"; // Crossing 8-123-122 into 7-62-61 private static final String ELEVEN = "7.6829818, -5.6379654"; private static final String TWELVE = "7.6735590, -5.5867130"; private static final String THIRTEEN = "7.6717846, -5.5812728"; private static final String FOURTEEN = "7.6681651, -5.5865950"; // Crossing 8-123-123 into 7-62-61 private static final String FIFTEEN = "5.8751274, -5.6744400"; private static final String SIXTEEN = "5.8641274, -5.5664451"; private static final String SEVENTEEN = "5.8545285, -5.5181042"; private static final String EIGHTEEN = "5.8813557, -5.5808398"; // Starting in 8-123-122, through 8-123-123, ending in 7-62-61 private static final String NINETEEN = "7.3884501, -6.4772718"; private static final String TWENTY = "7.3864604, -6.4728799"; private static final String TWENTY_ONE = "6.8787867, -6.4534416"; private static final String TWENTY_TWO = "6.8738255, -6.4553079"; private static final String TWENTY_THREE = "6.8299549, -5.2608135"; private static final String TWENTY_FOUR = "6.8298791, -5.2562899"; private static final String TWENTY_FIVE = "7.3868182, -6.4796125"; private static final String TWENTY_SIX = "6.8769819, -6.4596778"; private static final String TWENTY_SEVEN = "6.8333518, -5.2585345"; @TestAtlas(lines = { @Line(id = "541701001000", coordinates = { @Loc(value = THREE), @Loc(value = FOUR) }, tags = { "highway=primary" }), @Line(id = "541702001000", coordinates = { @Loc(value = SEVEN), @Loc(value = EIGHT), @Loc(value = NINE) }, tags = { "highway=primary" }), @Line(id = "541703001000", coordinates = { @Loc(value = EIGHT), @Loc(value = TEN) }, tags = { "highway=primary" }), @Line(id = "541704001000", coordinates = { @Loc(value = TWENTY_SIX), @Loc(value = TWENTY_ONE) }, tags = { "highway=primary" }), @Line(id = "541705001000", coordinates = { @Loc(value = FIFTEEN), @Loc(value = SIXTEEN), @Loc(value = SEVENTEEN) }, tags = { "highway=primary" }), @Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY), @Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE), @Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" }) }, points = { @Point(id = "511111003", coordinates = @Loc(value = THREE)), @Point(id = "511111004", coordinates = @Loc(value = FOUR)), @Point(id = "511111009", coordinates = @Loc(value = NINE)), @Point(id = "511111007", coordinates = @Loc(value = SEVEN)), @Point(id = "511111008", coordinates = @Loc(value = EIGHT)), @Point(id = "511111010", coordinates = @Loc(value = TEN)), @Point(id = "511111015", coordinates = @Loc(value = FIFTEEN)), @Point(id = "511111017", coordinates = @Loc(value = SEVENTEEN)), @Point(id = "511111019", coordinates = @Loc(value = NINETEEN)), @Point(id = "511111021", coordinates = @Loc(value = TWENTY_ONE)), @Point(id = "511111026", coordinates = @Loc(value = TWENTY_SIX)), @Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)), @Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR)) }) private Atlas atlasZ8x123y123; @TestAtlas(lines = { @Line(id = "541707001000", coordinates = { @Loc(value = ONE), @Loc(value = TWO) }, tags = { "highway=primary" }), @Line(id = "541702001000", coordinates = { @Loc(value = SEVEN), @Loc(value = EIGHT), @Loc(value = NINE) }, tags = { "highway=primary" }), @Line(id = "541708001000", coordinates = { @Loc(value = ELEVEN), @Loc(value = TWELVE), @Loc(value = THIRTEEN) }, tags = { "highway=primary" }), @Line(id = "541709001000", coordinates = { @Loc(value = TWENTY_FIVE), @Loc(value = TWENTY) }, tags = { "highway=primary" }), @Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY), @Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE), @Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" }) }, points = { @Point(id = "511111001", coordinates = @Loc(value = ONE)), @Point(id = "511111002", coordinates = @Loc(value = TWO)), @Point(id = "511111011", coordinates = @Loc(value = ELEVEN)), @Point(id = "511111013", coordinates = @Loc(value = THIRTEEN)), @Point(id = "511111007", coordinates = @Loc(value = SEVEN)), @Point(id = "511111009", coordinates = @Loc(value = NINE)), @Point(id = "511111019", coordinates = @Loc(value = NINETEEN)), @Point(id = "511111020", coordinates = @Loc(value = TWENTY)), @Point(id = "511111025", coordinates = @Loc(value = TWENTY_FIVE)), @Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)), @Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR)) }) private Atlas atlasZ8x123y122; @TestAtlas(lines = { @Line(id = "541710001000", coordinates = { @Loc(value = FIVE), @Loc(value = SIX) }, tags = { "highway=primary" }), @Line(id = "541711001000", coordinates = { @Loc(value = TWELVE), @Loc(value = FOURTEEN) }, tags = { "highway=primary" }), @Line(id = "541708001000", coordinates = { @Loc(value = ELEVEN), @Loc(value = TWELVE), @Loc(value = THIRTEEN) }, tags = { "highway=primary" }), @Line(id = "541712001000", coordinates = { @Loc(value = TWENTY_SEVEN), @Loc(value = TWENTY_THREE) }, tags = { "highway=primary" }), @Line(id = "541713001000", coordinates = { @Loc(value = SIXTEEN), @Loc(value = EIGHTEEN) }, tags = { "highway=primary" }), @Line(id = "541705001000", coordinates = { @Loc(value = FIFTEEN), @Loc(value = SIXTEEN), @Loc(value = SEVENTEEN) }, tags = { "highway=primary" }), @Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY), @Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE), @Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" }) }, points = { @Point(id = "511111005", coordinates = @Loc(value = FIVE)), @Point(id = "511111006", coordinates = @Loc(value = SIX)), @Point(id = "511111011", coordinates = @Loc(value = ELEVEN)), @Point(id = "511111012", coordinates = @Loc(value = TWELVE)), @Point(id = "511111013", coordinates = @Loc(value = THIRTEEN)), @Point(id = "511111014", coordinates = @Loc(value = FOURTEEN)), @Point(id = "511111015", coordinates = @Loc(value = FIFTEEN)), @Point(id = "511111016", coordinates = @Loc(value = SIXTEEN)), @Point(id = "511111017", coordinates = @Loc(value = SEVENTEEN)), @Point(id = "511111018", coordinates = @Loc(value = EIGHTEEN)), @Point(id = "511111019", coordinates = @Loc(value = NINETEEN)), @Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)), @Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR)), @Point(id = "511111023", coordinates = @Loc(value = TWENTY_THREE)), @Point(id = "511111027", coordinates = @Loc(value = TWENTY_SEVEN)) }) private Atlas atlasZ7x62y61; public Atlas getAtlasz7x62y61() { return this.atlasZ7x62y61; } public Atlas getAtlasz8x123y122() { return this.atlasZ8x123y122; } public Atlas getAtlasz8x123y123() { return this.atlasZ8x123y123; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/raw/RawAtlasIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; import org.junit.Assert; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.ShardFileOverlapsPolygonTest; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.sharding.DynamicTileSharding; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Integration tests for creating, slicing and sectioning with the raw Atlas ingest flow. * * @author mgostintsev */ public class RawAtlasIntegrationTest { private static final AtlasLoadingOption loadingOptionAll; private static final AtlasLoadingOption loadingOptionAntarctica; private static final AtlasLoadingOption loadingOptionIvoryCoast; private static final AtlasLoadingOption loadingOptionIntersectionAtEnd; private static final AtlasLoadingOption loadingOptionIntersectionAtStart; private static final AtlasLoadingOption loadingOptionIntersectionAtMiddle; private static final long LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS = 541706; private static final Logger logger = LoggerFactory.getLogger(RawAtlasIntegrationTest.class); static { final CountryBoundaryMap boundary = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> RawAtlasIntegrationTest.class .getResourceAsStream("CIV_GIN_LBR_osm_boundaries_with_grid_index.txt.gz")) .withDecompressor(Decompressor.GZIP)); loadingOptionAll = AtlasLoadingOption.createOptionWithAllEnabled(boundary); loadingOptionIvoryCoast = AtlasLoadingOption.createOptionWithAllEnabled(boundary) .setCountryCode("CIV"); // This is an OSM node that doesn't have any tags, is not a member of a relation or part of // a way. It should end up as a point in the final atlas. final String antarctica = "ATA"; // Create a fake boundary as a bounding box around the target point final Map> boundaries = new HashMap<>(); final Location targetPoint = Location.forString("-81.2022146, 51.6408578"); final List ataPolygons = new ArrayList<>(); ataPolygons .add(new JtsPolygonConverter().convert(targetPoint.boxAround(Distance.meters(1)))); boundaries.put(antarctica, ataPolygons); // Create a country boundary map with the fake Antarctica country boundary final CountryBoundaryMap antarticaBoundary = CountryBoundaryMap.fromBoundaryMap(boundaries); loadingOptionAntarctica = AtlasLoadingOption.createOptionWithAllEnabled(antarticaBoundary) .setCountryCode(antarctica); final CountryBoundaryMap intersectionAtEndBoundary = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> RawAtlasIntegrationTest.class .getResourceAsStream("layerIntersectionAtEndBoundaryMap.txt"))); loadingOptionIntersectionAtEnd = AtlasLoadingOption .createOptionWithAllEnabled(intersectionAtEndBoundary).setCountryCode("RUS"); final CountryBoundaryMap intersectionAtStartBoundary = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> RawAtlasIntegrationTest.class .getResourceAsStream("layerIntersectionAtStartBoundaryMap.txt"))); loadingOptionIntersectionAtStart = AtlasLoadingOption .createOptionWithAllEnabled(intersectionAtStartBoundary).setCountryCode("RUS"); final CountryBoundaryMap intersectionAtMiddleBoundary = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> RawAtlasIntegrationTest.class .getResourceAsStream("layerIntersectionInMiddleBoundaryMap.txt"))); loadingOptionIntersectionAtMiddle = AtlasLoadingOption .createOptionWithAllEnabled(intersectionAtMiddleBoundary).setCountryCode("SGP"); } @Rule public DynamicRawAtlasSectioningTestRule setup = new DynamicRawAtlasSectioningTestRule(); @Test public void testPbfToSlicedAtlasWithExpansion() { // Create a simple store, populated with 3 shards and the corresponding atlases final Map store = prepareShardStore(); final Function> rawAtlasFetcher = shard -> { if (store.containsKey(shard)) { return Optional.of(store.get(shard)); } else { return Optional.empty(); } }; // Create 3 atlas files, starting from each of the different shards final Atlas atlasFromz8x123y122 = generateSectionedAtlasStartingAtShard( new SlippyTile(123, 122, 8), rawAtlasFetcher); logger.info(atlasFromz8x123y122.summary()); final Atlas atlasFromz8x123y123 = generateSectionedAtlasStartingAtShard( new SlippyTile(123, 123, 8), rawAtlasFetcher); logger.info(atlasFromz8x123y123.summary()); final Atlas atlasFromz7x62y61 = generateSectionedAtlasStartingAtShard( new SlippyTile(62, 61, 7), rawAtlasFetcher); logger.info(atlasFromz7x62y61.summary()); // Let's focus on the edge spanning all 3 shards and verify it got sectioned properly final Iterable firstGroupOfEdges = atlasFromz8x123y122 .edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS); final Iterable secondGroupOfEdges = atlasFromz8x123y123 .edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS); final Iterable thirdGroupOfEdges = atlasFromz7x62y61 .edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS); // First look at absolute counts. Each shard will have two forward and reverse edges Assert.assertEquals(4, Iterables.size(firstGroupOfEdges)); Assert.assertEquals(4, Iterables.size(secondGroupOfEdges)); Assert.assertEquals(4, Iterables.size(thirdGroupOfEdges)); // Next, let's check identifier consistency final Set uniqueIdentifiers = new HashSet<>(); Iterables.stream(firstGroupOfEdges) .forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier())); Iterables.stream(secondGroupOfEdges) .forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier())); Iterables.stream(thirdGroupOfEdges) .forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier())); // There should be 4 pieces (each having a forward and reverse edge) total Assert.assertTrue(uniqueIdentifiers.size() == 8); // Validate the same edge identifiers built from different shards to test equality final Edge piece2from122 = atlasFromz8x123y122.edge(541706001002L); final Edge piece2from123 = atlasFromz8x123y123.edge(541706001002L); Assert.assertTrue(piece2from122.asPolyLine().equals(piece2from123.asPolyLine())); final Edge piece3from123 = atlasFromz8x123y123.edge(541706001003L); final Edge piece3from62 = atlasFromz7x62y61.edge(541706001003L); Assert.assertTrue(piece3from123.asPolyLine().equals(piece3from62.asPolyLine())); // Let's validate absolute number of edges in each shard Assert.assertEquals(12, atlasFromz8x123y122.numberOfEdges()); Assert.assertEquals(16, atlasFromz8x123y123.numberOfEdges()); Assert.assertEquals(20, atlasFromz7x62y61.numberOfEdges()); } @Test public void testPbfToSlicedRawAtlas() { // This PBF file contains really interesting data. 1. MultiPolygon with multiple inners and // outers spanning 3 countries (http://www.openstreetmap.org/relation/3638082) 2. Multiple // nested relations (http://www.openstreetmap.org/relation/3314886) final String pbfPath = RawAtlasIntegrationTest.class .getResource("8-122-122-trimmed.osm.pbf").getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(pbfPath)); final Atlas rawAtlas = rawAtlasGenerator.build(); Assert.assertEquals(0, rawAtlas.numberOfNodes()); Assert.assertEquals(0, rawAtlas.numberOfEdges()); Assert.assertEquals(5119, rawAtlas.numberOfAreas()); Assert.assertEquals(79634, rawAtlas.numberOfPoints()); Assert.assertEquals(2729, rawAtlas.numberOfLines()); Assert.assertEquals(24, rawAtlas.numberOfRelations()); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionAll, rawAtlas).slice(); Assert.assertEquals(0, slicedRawAtlas.numberOfNodes()); Assert.assertEquals(0, slicedRawAtlas.numberOfEdges()); Assert.assertEquals(5132, slicedRawAtlas.numberOfAreas()); Assert.assertEquals(32524, slicedRawAtlas.numberOfPoints()); Assert.assertEquals(3003, slicedRawAtlas.numberOfLines()); Assert.assertEquals(24, slicedRawAtlas.numberOfRelations()); // Assert all raw Atlas entities have a country code assertAllEntitiesHaveCountryCode(slicedRawAtlas); final Atlas ivoryCoast = new RawAtlasSlicer(loadingOptionIvoryCoast, rawAtlas).slice(); Assert.assertEquals(0, ivoryCoast.numberOfNodes()); Assert.assertEquals(0, ivoryCoast.numberOfEdges()); Assert.assertEquals(2400, ivoryCoast.numberOfAreas()); Assert.assertEquals(15075, ivoryCoast.numberOfPoints()); Assert.assertEquals(1265, ivoryCoast.numberOfLines()); Assert.assertEquals(19, ivoryCoast.numberOfRelations()); // Assert all raw Atlas entities have a country code assertAllEntitiesHaveCountryCode(ivoryCoast); // Test sectioning! final Atlas finalAtlas = new AtlasSectionProcessor(slicedRawAtlas, loadingOptionAll).run(); Assert.assertEquals(5011, finalAtlas.numberOfNodes()); Assert.assertEquals(9764, finalAtlas.numberOfEdges()); Assert.assertEquals(5132, finalAtlas.numberOfAreas()); Assert.assertEquals(184, finalAtlas.numberOfPoints()); Assert.assertEquals(376, finalAtlas.numberOfLines()); Assert.assertEquals(24, finalAtlas.numberOfRelations()); } @Ignore @Test public void testSectioningFromShard() { final String path = RawAtlasIntegrationTest.class.getResource("8-122-122-trimmed.osm.pbf") .getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path)); final Atlas rawAtlas = rawAtlasGenerator.build(); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionAll, rawAtlas).slice(); // Simple fetcher that returns the atlas from above for the corresponding shard final Map store = new HashMap<>(); store.put(new SlippyTile(122, 122, 8), slicedRawAtlas); final Function> rawAtlasFetcher = shard -> { if (store.containsKey(shard)) { return Optional.of(store.get(shard)); } else { return Optional.empty(); } }; final Atlas finalAtlas = new AtlasSectionProcessor(new SlippyTile(122, 122, 8), loadingOptionAll, new DynamicTileSharding(new File(ShardFileOverlapsPolygonTest.class.getResource( "/org/openstreetmap/atlas/geography/boundary/tree-6-14-100000.txt.gz") .getFile())), rawAtlasFetcher).run(); Assert.assertEquals(5009, finalAtlas.numberOfNodes()); Assert.assertEquals(9760, finalAtlas.numberOfEdges()); Assert.assertEquals(5128, finalAtlas.numberOfAreas()); Assert.assertEquals(184, finalAtlas.numberOfPoints()); Assert.assertEquals(271, finalAtlas.numberOfLines()); Assert.assertEquals(23, finalAtlas.numberOfRelations()); } @Ignore @Test public void testStandAloneNodeIngest() { // Create a raw atlas, slice and section it final String pbfPath = RawAtlasIntegrationTest.class.getResource("node-4353689487.pbf") .getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(pbfPath)); final Atlas rawAtlas = rawAtlasGenerator.build(); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionAntarctica, rawAtlas).slice(); final Atlas finalAtlas = new AtlasSectionProcessor(slicedRawAtlas, loadingOptionAntarctica) .run(); // Verify only a single point exists Assert.assertEquals(0, finalAtlas.numberOfNodes()); Assert.assertEquals(0, finalAtlas.numberOfEdges()); Assert.assertEquals(0, finalAtlas.numberOfAreas()); Assert.assertEquals(1, finalAtlas.numberOfPoints()); Assert.assertEquals(0, finalAtlas.numberOfLines()); Assert.assertEquals(0, finalAtlas.numberOfRelations()); } @Test public void testTwoWaysWithDifferentLayersIntersectingAtEnd() { // Based on https://www.openstreetmap.org/way/26071941 and // https://www.openstreetmap.org/way/405246856 having two different layer tag values and // having a shared node (https://www.openstreetmap.org/node/281526976) at which one of the // ways ends. This is a fairly common OSM use-case, where two roads (often ramps or links) // having different layer tags should be connected. final Location intersection = Location.forString("55.0480165, 82.9406646"); final String path = RawAtlasIntegrationTest.class .getResource("twoWaysWithDifferentLayersIntersectingAtEnd.pbf").getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path)); final Atlas rawAtlas = rawAtlasGenerator.build(); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionIntersectionAtEnd, rawAtlas) .slice(); final Atlas finalAtlas = new AtlasSectionProcessor(slicedRawAtlas, loadingOptionIntersectionAtEnd).run(); // Make sure there are exactly three edges created. Both ways are one-way and one of them // gets way-sectioned into two edges. Assert.assertEquals(3, finalAtlas.numberOfEdges()); // Make sure there are exactly 4 nodes Assert.assertEquals(4, finalAtlas.numberOfNodes()); // Explicitly check for a single node at the intersection location Assert.assertEquals(1, Iterables.size(finalAtlas.nodesAt(intersection))); // Explicitly check that the layer=0 link is connected to both the layer=-1 trunk edges Assert.assertEquals(2, finalAtlas.edge(26071941000000L).connectedEdges().size()); } @Test public void testTwoWaysWithDifferentLayersIntersectingAtStart() { // Based on https://www.openstreetmap.org/way/551411163 and partial piece of // https://www.openstreetmap.org/way/67803311 having two different layer tag values and // having a shared node (https://www.openstreetmap.org/node/5325270497) at which one of the // ways ends. This is a fairly common OSM use-case, where two roads (often ramps or links) // having different layer tags should be connected. In this case, we also check that the // trunk link is connected to the trunk at both the start and end nodes. final Location intersection = Location.forString("52.4819691, 38.7603042"); final String path = RawAtlasIntegrationTest.class .getResource("twoWaysWithDifferentLayersIntersectingAtStart.pbf").getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path)); final Atlas rawAtlas = rawAtlasGenerator.build(); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionIntersectionAtStart, rawAtlas) .slice(); final Atlas finalAtlas = new AtlasSectionProcessor(slicedRawAtlas, loadingOptionIntersectionAtStart).run(); // Make sure there are exactly six edges created. The trunk link (551411163) is // way-sectioned into 2 pieces - at an intermediate crossing, while the trunk (67803311) is // sectioned into 4 pieces - once at the start of the link, once at an intermediate crossing // and again at the end of the link. Assert.assertEquals(6, finalAtlas.numberOfEdges()); // Make sure there are exactly 6 nodes Assert.assertEquals(6, finalAtlas.numberOfNodes()); // Explicitly check for a single node at the intersection location Assert.assertEquals(1, Iterables.size(finalAtlas.nodesAt(intersection))); // Explicitly check that the layer=0 link is connected to both the layer=1 trunk edges and // its own sectioned edge Assert.assertEquals(3, finalAtlas.edge(551411163000001L).connectedEdges().size()); Assert.assertEquals(3, finalAtlas.edge(551411163000002L).connectedEdges().size()); } @Test public void testTwoWaysWithDifferentLayersIntersectingInMiddle() { // Based on https://www.openstreetmap.org/way/467880095 and // https://www.openstreetmap.org/way/28247094 having two different layer tag values and // having overlapping nodes (https://www.openstreetmap.org/node/4661272336 and // https://www.openstreetmap.org/node/5501637097) that should not be merged. final Location overlappingLocation = Location.forString("1.3248985,103.6452864"); final String path = RawAtlasIntegrationTest.class .getResource("twoWaysWithDifferentLayersIntersectingInMiddle.pbf").getPath(); final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path)); final Atlas rawAtlas = rawAtlasGenerator.build(); // Verify both points made it into the raw atlas Assert.assertTrue(Iterables.size(rawAtlas.pointsAt(overlappingLocation)) == 2); final Atlas slicedRawAtlas = new RawAtlasSlicer(loadingOptionIntersectionAtMiddle, rawAtlas) .slice(); final Atlas finalAtlas = new AtlasSectionProcessor(slicedRawAtlas, loadingOptionIntersectionAtMiddle).run(); // Make sure there is no sectioning happening between the two ways with different layer tag // values. There is a one-way overpass and a bi-directional residential street, resulting in // 3 total edges and 4 nodes (one on both ends of the two segments) Assert.assertEquals(3, finalAtlas.numberOfEdges()); Assert.assertEquals(4, finalAtlas.numberOfNodes()); // Again, verify there is no node at the duplicated location Assert.assertTrue(Iterables.size(finalAtlas.nodesAt(overlappingLocation)) == 0); Assert.assertEquals(0, finalAtlas.numberOfPoints()); } private void assertAllEntitiesHaveCountryCode(final Atlas atlas) { atlas.lines().forEach(line -> { Assert.assertTrue(Validators.hasValuesFor(line, ISOCountryTag.class)); }); atlas.points().forEach(point -> { Assert.assertTrue(Validators.hasValuesFor(point, ISOCountryTag.class)); }); atlas.relations().forEach(point -> { Assert.assertTrue(Validators.hasValuesFor(point, ISOCountryTag.class)); }); } private Atlas generateSectionedAtlasStartingAtShard(final Shard shard, final Function> rawAtlasFetcher) { return new AtlasSectionProcessor(shard, loadingOptionAll, new DynamicTileSharding(new File(ShardFileOverlapsPolygonTest.class.getResource( "/org/openstreetmap/atlas/geography/boundary/tree-6-14-100000.txt.gz") .getFile())), rawAtlasFetcher).run(); } private Map prepareShardStore() { final Map store = new HashMap<>(); store.put(new SlippyTile(62, 61, 7), this.setup.getAtlasz7x62y61()); store.put(new SlippyTile(123, 123, 8), this.setup.getAtlasz8x123y123()); store.put(new SlippyTile(123, 122, 8), this.setup.getAtlasz8x123y122()); return store; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/atlas/routing/AtlasRoutingIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.atlas.routing; import java.util.ArrayList; import java.util.List; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasIntegrationTest; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class AtlasRoutingIntegrationTest extends AtlasIntegrationTest { /** * @author matthieun */ private static final class ExpectedRoute { private final Location start; private final Location end; private final Distance expectedMaximumLength; private final int expectedMaximumSize; ExpectedRoute(final Location start, final Location end, final double expectedMaximumLengthInMeters, final int expectedMaximumSize) { this.start = start; this.end = end; this.expectedMaximumLength = Distance.meters(expectedMaximumLengthInMeters); this.expectedMaximumSize = expectedMaximumSize; } public Location getEnd() { return this.end; } public Distance getExpectedMaximumLength() { return this.expectedMaximumLength; } public int getExpectedMaximumSize() { return this.expectedMaximumSize; } public Location getStart() { return this.start; } } private static final Logger logger = LoggerFactory.getLogger(AtlasRoutingIntegrationTest.class); private static final Location ONE = Location.forString("20.245996,-74.1502787"); private static final Location TWO = Location.forString("20.0774286,-74.4971688"); private static final Location THREE = Location.forString("20.6079142,-74.7530228"); private static final Location FOUR = Location.forString("20.6649509,-74.9532838"); private static final Location FIVE = Location.forString("20.58069,-75.24285"); private static final Location SIX = Location.forString("20.13990,-75.20937"); private static final List EXPECTED_ROUTES = new ArrayList<>(); static { EXPECTED_ROUTES.add(new ExpectedRoute(ONE, TWO, 70_000, 50)); EXPECTED_ROUTES.add(new ExpectedRoute(TWO, THREE, 160_000, 150)); EXPECTED_ROUTES.add(new ExpectedRoute(THREE, FOUR, 30_000, 80)); EXPECTED_ROUTES.add(new ExpectedRoute(FOUR, FIVE, 60_000, 90)); EXPECTED_ROUTES.add(new ExpectedRoute(FIVE, SIX, 100_000, 110)); } private Atlas cuba; private Router router; @After public void destroy() { this.cuba = null; this.router = null; } @Before public void initialize() { this.cuba = loadCuba(); this.router = AStarRouter.fastComputationAndSubOptimalRoute(this.cuba, Distance.meters(100)); } @Test public void testRouting() { for (final ExpectedRoute expectedRoute : EXPECTED_ROUTES) { final Route route = route(expectedRoute.getStart(), expectedRoute.getEnd()); Assert.assertTrue(expectedRoute.getExpectedMaximumSize() >= route.size()); Assert.assertTrue(expectedRoute.getExpectedMaximumLength() .isGreaterThanOrEqualTo(route.length())); } } private Route route(final Location start, final Location end) { final Time beginning = Time.now(); final Route route = this.router.route(start, end); if (route == null) { throw new CoreException("Could not find route between {} and {}.", start, end); } logger.info("Computed route between {} and {}, with {} edges, {} long, in {}", start, end, route.size(), route.length(), beginning.elapsedSince()); return route; } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/boundary/CountryBoundaryMapArchiverIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.boundary; import java.io.IOException; import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.TemporaryFile; import org.openstreetmap.atlas.utilities.collections.StringList; /** * Test CountryBoundaryMap generation. * * @author james-gage * @author matthieun */ public class CountryBoundaryMapArchiverIntegrationTest { @Test public void testOceanBoundary() throws IOException { try (TemporaryFile temporary = File.temporary("CountryBoundaryMapArchiverTest", ".txt.gz")) { final StringList arguments = new StringList(); arguments.add("-" + CountryBoundaryMapArchiver.BOUNDARY_FILE.getName() + "=" + CountryBoundaryMapArchiverIntegrationTest.class .getResource("oceanTestBoundary.txt").getPath()); arguments.add("-" + CountryBoundaryMapArchiver.OUTPUT.getName() + "=" + temporary.getAbsolutePathString()); arguments.add( "-" + CountryBoundaryMapArchiver.OCEAN_BOUNDARY_ZOOM_LEVEL.getName() + "=3"); arguments .add("-" + CountryBoundaryMapArchiver.CREATE_SPATIAL_INDEX.getName() + "=true"); new CountryBoundaryMapArchiver().runWithoutQuitting(arguments.toArray()); // ensure that the correct number of ocean boundaries are generated final CountryBoundaryMap oceanBoundaryMap = CountryBoundaryMap.fromPlainText(temporary); Assert.assertEquals(57, oceanBoundaryMap.size()); } } } ================================================ FILE: src/integrationTest/java/org/openstreetmap/atlas/geography/boundary/CountryBoundaryMapIntegrationTest.java ================================================ package org.openstreetmap.atlas.geography.boundary; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.boundaries.ComplexBoundaryIntegrationTestRule; /** * @author matthieun */ public class CountryBoundaryMapIntegrationTest { // Use the same rule as ComplexBoundaryIntegrationTest @Rule public final ComplexBoundaryIntegrationTestRule rule = new ComplexBoundaryIntegrationTestRule(); @Test public void testCountryBoundaryMapFromAtlas() { final Atlas atlas = this.rule.getAtlas(); final CountryBoundaryMap map = CountryBoundaryMap.fromAtlas(atlas); Assert.assertEquals(2, map.allCountryNames().size()); Assert.assertEquals( "POLYGON ((-75.2288309 18.3520806, -75.2336999 18.3664654, -75.2384618 18.398702, " + "-75.2377069 18.4312495, -75.2322562 18.4610773, -75.222672 18.4886399, -75.2172678 18.4999677, " + "-75.1993364 18.5289778, -75.1765697 18.5547196, -75.1596528 18.5688721, -75.1492275 18.5767252, " + "-75.1198508 18.5933921, -75.0880291 18.6053034, -75.0546028 18.6121442, -75.0199034 18.6137166, " + "-74.9832495 18.6098363, -74.952276 18.6044011, -74.9225014 18.5946192, -74.893673 18.5801641, " + "-74.8681576 18.5623586, -74.8457299 18.5411084, -74.825893 18.5159259, -74.8084096 18.4854595, " + "-74.7966939 18.4525742, -74.7907319 18.418, -74.790449 18.3896744, -74.7920482 18.3693394, " + "-74.804815 18.320668, -74.8230994 18.285888, -74.848054 18.2551258, -74.8801877 18.2283852, " + "-74.9134925 18.2093121, -74.9498941 18.1964118, -74.9941897 18.1892548, -75.0374985 18.1907252, " + "-75.0814085 18.201251, -75.1219061 18.2195619, -75.1482084 18.2370322, -75.1721527 18.2582151, " + "-75.1923834 18.2826809, -75.2052088 18.3013364, -75.2192144 18.3256833, -75.2288309 18.3520806))", map.countryBoundary("UMI").get(0).toString()); } } ================================================ FILE: src/integrationTest/resources/org/openstreetmap/atlas/geography/atlas/raw/layerIntersectionAtEndBoundaryMap.txt ================================================ RUS||POLYGON((82.9358393 55.0504618, 82.9440158 55.0514274, 82.9463965 55.0445087, 82.9378292 55.0434533, 82.9358393 55.0504618)) ================================================ FILE: src/integrationTest/resources/org/openstreetmap/atlas/geography/atlas/raw/layerIntersectionAtStartBoundaryMap.txt ================================================ RUS||POLYGON((38.7 52.4, 38.8 52.4, 38.8 52.5, 38.7 52.5, 38.7 52.4)) ================================================ FILE: src/integrationTest/resources/org/openstreetmap/atlas/geography/atlas/raw/layerIntersectionInMiddleBoundaryMap.txt ================================================ SGP||POLYGON((103.6350863 1.3492437, 103.6863940 1.3448244, 103.6839910 1.2996620, 103.6219767 1.3022755, 103.6350863 1.3492437)) ================================================ FILE: src/integrationTest/resources/org/openstreetmap/atlas/geography/boundary/oceanTestBoundary.txt ================================================ ABC||POLYGON((-106.02905273438 -46.1865234375, -129.23217773438 -21.5771484375, -125.71655273438 15.6884765625, -99.700927734375 57.8759765625, -49.075927734375 61.3916015625, 7.174072265625 67.0166015625, 52.877197265625 52.2509765625, 93.658447265625 46.6259765625, 124.59594726563 10.7666015625, 126.00219726563 -20.1708984375, 85.220947265625 -49.7021484375, -4.075927734375 -56.7333984375, -79.310302734375 -59.5458984375, -106.02905273438 -46.1865234375))# ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/Event.java ================================================ package org.openstreetmap.atlas.event; import java.util.Date; /** * Useful base class to hold common information for {@link Event} implementations * * @author mkalender */ public abstract class Event { private final Date timestamp; /** * Default constructor */ protected Event() { this.timestamp = new Date(); } protected Date getTimestamp() { return this.timestamp; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/EventService.java ================================================ package org.openstreetmap.atlas.event; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import org.openstreetmap.atlas.utilities.threads.Pool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.eventbus.EventBus; /** * A simple in-memory publish-subscribe service built on top of {@link EventBus} * * @param * - event type that is going to be posted * @author mkalender * @author jklamer * @author Yazad Khambata */ public final class EventService implements EventServiceable { private static final Logger logger = LoggerFactory.getLogger(EventService.class); // A key-value mapping for multiple event services private static Map serviceMap = new ConcurrentHashMap<>(); // Event bus to dispatch events private final EventBus eventBus; // Container of processors on the EventService private final Collection> processors = new HashSet<>(); // Thread-safe complete indicator private final AtomicBoolean completed = new AtomicBoolean(); /** * @param key * key to retrieve {@link EventService} * @param * - event type that is going to be posted * @return {@link EventService} instance for given key */ public static EventService get(final String key) { serviceMap.putIfAbsent(key, new EventService()); return serviceMap.get(key); } private EventService() { this.eventBus = new EventBus((exception, context) -> logger .warn("An exception is thrown in EventBus.", exception)); } /** * Stops event processing {@link Pool} and posts a {@link ShutdownEvent} event */ @Override public void complete() { if (!this.completed.compareAndSet(false, true)) { logger.warn("EventService is already completed. Skipping completion."); return; } this.eventBus.post(new ShutdownEvent()); new HashSet<>(this.processors).forEach(this::unregister); } /** * Publishes/posts a new event {@link Object} * * @param event * {@link Object} to post */ @Override public void post(final T event) { if (event == null) { logger.warn("EventService received a null event. Skipping posting."); return; } if (this.completed.get()) { logger.warn("EventService is already completed. Skipping posting."); return; } this.eventBus.post(event); } /** * Registers given {@link Processor} to subscribe for events * * @param processor * {@link Processor} to register */ @Override public void register(final Processor processor) { this.eventBus.register(processor); this.processors.add(processor); } /** * Unregisters given {@link Processor} * * @param processor * {@link Processor} to unregister */ @Override public void unregister(final Processor processor) { this.eventBus.unregister(processor); this.processors.remove(processor); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/EventServiceable.java ================================================ package org.openstreetmap.atlas.event; import org.openstreetmap.atlas.utilities.threads.Pool; /** * Contract for Event pub-sub implementations. * * @param * - event type that is going to be posted * @author Yazad Khambata */ public interface EventServiceable { /** * Stops event processing {@link Pool} and posts a {@link ShutdownEvent} event */ void complete(); /** * Publishes/posts a new event {@link Object} * * @param event * {@link Object} to post */ void post(T event); /** * Registers given {@link Processor} to subscribe for events * * @param processor * {@link Processor} to register */ void register(Processor processor); /** * Unregisters given {@link Processor} * * @param processor * {@link Processor} to unregister */ void unregister(Processor processor); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/Processor.java ================================================ package org.openstreetmap.atlas.event; /** * The {@link Processor} interface provides simple hooks for implementations to handle events. * * @author mkalender * @param * Type that is going to be processed */ public interface Processor { /** * Method to process {@link ShutdownEvent}. This method will be called only once.
* Please make sure to add {@code @Subscribe} annotation to the method that is * implementing this method. * * @param event * {@link ShutdownEvent} to process */ void process(ShutdownEvent event); /** * Method to process {@link Event}. If your method can process multiple events simultaneously, * then mark your method with {@code @AllowConcurrentEvents} annotation.Please make sure * to add {@code @Subscribe} annotation to the method that is implementing this method. * * @param event * {@link Event} to process */ void process(T event); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/README.md ================================================ # EventService The [EventService](EventService.java) is a class build on top of Guava's EventBus with static keyword construction. It provides an non-blocking endpoint to send events of various types to be processed asynchronously during program execution. # When to use When you want to record activities of events during program execution and have those be processed, recorded, or control other threads on the fly. # How To Use 1. Get an [EventService](EventService.java) object with ```java public static EventService get(final String key) ``` All threads on the same JVM that use the same keyword to get an EventService have the same EventService object. 2. Extend [Event](Event.java) into a class that contains the message information that is needed. 3. Implement the [Processor](Processor.java) interface * Be sure to include a function that processes your desired event type * Make sure all [Processor](Processor.java) implementations have the @Subscribe annotation above both their specific event process method and above the ```process(ShutdownEvent)``` method 4. Construct and register all [Processor](Processor.java) objects with the EventService before sending events. 5. Post [Event](Event.java) objects to the [EventService](EventService.java) to be processed by the appropriate registered [Processor](Processor.java)s. 6. At the end of your job or JVM instance call EventService::complete to communicate to the processors that the program execution is complete. There are multiple and varied flexible use cases for this usage pattern. ================================================ FILE: src/main/java/org/openstreetmap/atlas/event/ShutdownEvent.java ================================================ package org.openstreetmap.atlas.event; /** * An {@link Event} that is posted when {@link EventService} is shutting down * * @author mkalender */ public class ShutdownEvent extends Event { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/exception/CoreException.java ================================================ package org.openstreetmap.atlas.exception; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.slf4j.helpers.MessageFormatter; /** * This is a universal Exception in Core. Can use substitutions in messages according to very simple * substitution rules. Substitutions can be made 1, 2 or more arguments. *

* For example, *

* * new CoreException("Hi {}, how are {}?", "there", "you"); * *

* is same as *

* * new CoreException("Hi there, how are you?"); * * * @author matthieun * @author tony * @author Yazad Khambata */ public class CoreException extends RuntimeException { public static final String TOKEN = CoreException.class.getSimpleName(); private static final long serialVersionUID = 5019327451085548495L; protected static final UnaryOperator REFINE_ARGUMENTS = arguments -> { if (arguments.length > 0 && arguments[arguments.length - 1] instanceof Throwable) { final Object[] result = new Object[arguments.length - 1]; for (int i = 0; i < arguments.length - 1; i++) { result[i] = arguments[i]; } return result; } else { return arguments; } }; protected static final Function> CAUSE_FROM = arguments -> arguments.length != REFINE_ARGUMENTS .apply(arguments).length ? Optional.of((Throwable) arguments[arguments.length - 1]) : Optional.empty(); public static Supplier supplier(final String message) { return () -> new CoreException(message); } public static Supplier supplier(final String message, final Object... arguments) { return () -> new CoreException(message, arguments); } public static Supplier supplier(final String message, final Throwable cause) { return () -> new CoreException(message, cause); } protected static String messageWithToken(final String message) { final String separator = "; "; return new StringBuilder(TOKEN).append(separator).append(message).toString(); } public CoreException(final String message) { super(message); } /** * Create a new CoreException with a specified message * * @param message * The message (formatted with {@link MessageFormatter#arrayFormat}) * @param arguments * The arguments (if the last argument is a {@link Throwable}, that becomes * the cause) */ public CoreException(final String message, final Object... arguments) { super(MessageFormatter.arrayFormat(message, REFINE_ARGUMENTS.apply(arguments)).getMessage(), CAUSE_FROM.apply(arguments).orElse(null)); } public CoreException(final String message, final Throwable cause) { super(message, cause); } public CoreException(final String message, final Throwable cause, final Object... arguments) { super(MessageFormatter.arrayFormat(message, arguments).getMessage(), cause); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/exception/ExceptionSearch.java ================================================ package org.openstreetmap.atlas.exception; import java.util.Optional; /** * Utility class for searching an exception cause chain for a particular exception type * * @author cstaylor * @param * the type of exception we're searching for */ public final class ExceptionSearch { private final Class target; public static ExceptionSearch find(final Class target) { if (target == null) { throw new IllegalArgumentException("target is null"); } return new ExceptionSearch<>(target); } private ExceptionSearch(final Class target) { this.target = target; } public Optional within(final Throwable source) { if (this.target == null) { throw new IllegalStateException("target is null"); } return Optional.ofNullable(within0(source)); } private T within0(final Throwable source) { if (source == null) { return null; } if (this.target.isInstance(source)) { return this.target.cast(source); } return within0(source.getCause()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/exception/LoadAtlasFromResourceException.java ================================================ package org.openstreetmap.atlas.exception; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Thrown when there's a problem loading an atlas from a packed atlas so we can quickly collect all * of the missing or damaged files. * * @author cstaylor */ public class LoadAtlasFromResourceException extends CoreException { private static final long serialVersionUID = 65439602944966080L; private final transient Resource resource; public LoadAtlasFromResourceException(final Resource resource, final String message) { super(message); this.resource = resource; } public LoadAtlasFromResourceException(final Resource resource, final String message, final Object... arguments) { super(message, arguments); this.resource = resource; } public LoadAtlasFromResourceException(final Resource resource, final String message, final Throwable cause) { super(message, cause); this.resource = resource; } public LoadAtlasFromResourceException(final Resource resource, final String message, final Throwable cause, final Object... arguments) { super(message, cause, arguments); this.resource = resource; } public Resource getResource() { return this.resource; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/exception/change/FeatureChangeMergeException.java ================================================ package org.openstreetmap.atlas.exception.change; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.slf4j.helpers.MessageFormatter; /** * A special exception for {@link FeatureChange} merge errors. * * @author lcram */ public class FeatureChangeMergeException extends CoreException { private static final long serialVersionUID = -3583945839922744755L; static final int MAXIMUM_MESSAGE_SIZE = 2000; private final List failureTypeTrace; static String truncate(final String input) { return input.substring(0, Math.min(input.length(), MAXIMUM_MESSAGE_SIZE)); } public FeatureChangeMergeException(final List failureTypeTrace, final String message) { super(truncate(message)); this.failureTypeTrace = failureTypeTrace; if (this.failureTypeTrace == null || this.failureTypeTrace.isEmpty()) { throw new CoreException("failureTypeTrace cannot be null or empty"); } } public FeatureChangeMergeException(final List failureTypeTrace, final String message, final Object... arguments) { super(truncate(MessageFormatter.arrayFormat(message, REFINE_ARGUMENTS.apply(arguments)) .getMessage()), CAUSE_FROM.apply(arguments).orElse(null)); this.failureTypeTrace = failureTypeTrace; if (this.failureTypeTrace == null || this.failureTypeTrace.isEmpty()) { throw new CoreException("failureTypeTrace cannot be null or empty"); } } public FeatureChangeMergeException(final MergeFailureType rootLevelFailure, final String message, final Object... arguments) { super(truncate(MessageFormatter.arrayFormat(message, REFINE_ARGUMENTS.apply(arguments)) .getMessage()), CAUSE_FROM.apply(arguments).orElse(null)); if (rootLevelFailure == null) { throw new CoreException("rootLevelFailure cannot be null"); } this.failureTypeTrace = new ArrayList<>(); this.failureTypeTrace.add(rootLevelFailure); } public FeatureChangeMergeException(final MergeFailureType rootLevelFailure, final String message) { super(truncate(message)); if (rootLevelFailure == null) { throw new CoreException("rootLevelFailure cannot be null"); } this.failureTypeTrace = new ArrayList<>(); this.failureTypeTrace.add(rootLevelFailure); } /** * Return the {@link MergeFailureType} at the provided stack index. If the index is out of * bounds, this will return the top level failure. * * @param index * the index to check * @return the {@link MergeFailureType} at the provided index, or the most top level failure if * the index is out of bounds */ public MergeFailureType failureAtFrameIndex(final int index) { if (index >= this.failureTypeTrace.size()) { return topLevelFailure(); } return this.failureTypeTrace.get(index); } /** * Get the number of {@link MergeFailureType}s in the failure trace. * * @return the number of {@link MergeFailureType}s in the failure trace */ public int failureTraceSize() { return this.failureTypeTrace.size(); } public List getMergeFailureTrace() { return new ArrayList<>(this.failureTypeTrace); } /** * Return the root level {@link MergeFailureType}. This is the bottom-most failure reason, and * will generally be the most specific failure reason available for a given * {@link FeatureChangeMergeException}. * * @return the root level {@link MergeFailureType} */ public MergeFailureType rootLevelFailure() { return this.failureTypeTrace.get(0); } /** * Return the top level {@link MergeFailureType}. This is the top-most failure reason, and will * generally be the most general failure reason available for a given * {@link FeatureChangeMergeException}. * * @return the top level {@link MergeFailureType} */ public MergeFailureType topLevelFailure() { return this.failureTypeTrace.get(this.failureTypeTrace.size() - 1); } /** * Check if the failure trace contains an exact failure sequence of {@link MergeFailureType}s, * in level order from root to top. E.g. suppose the failure trace list is [root: A, B, C, D, * top: E], and our provided subsequence is [A, B]. In this case, the method would return true. * Now suppose the provided subsequence is [A, C]. Now, the method will return false. This * method may be useful if callers want to check for and recover from a specific failure * sequence. * * @param subSequence * the subsequence of {@link MergeFailureType}s to check * @return if the subsequence is present */ public boolean traceContainsExactFailureSubSequence(final List subSequence) { if (subSequence.isEmpty()) { return true; } if (subSequence.size() > this.failureTypeTrace.size()) { return false; } for (int i = 0; i < this.failureTypeTrace.size(); i++) { boolean foundSubSequenceThisIteration = true; for (int j = 0, tmpI = i; j < subSequence.size(); j++, tmpI++) { // We hit the end of the failure trace early or we found a sequence mismatch if (tmpI >= this.failureTypeTrace.size() || subSequence.get(j) != this.failureTypeTrace.get(tmpI)) { foundSubSequenceThisIteration = false; break; } } if (foundSubSequenceThisIteration) { return true; } } return false; } /** * Check if this exception trace contains a {@link MergeFailureType} that matches the given * type. * * @param type * the {@link MergeFailureType} for which to check * @return true if the trace contains the provided type */ public boolean traceContainsFailureType(final MergeFailureType type) { for (final MergeFailureType currentType : this.failureTypeTrace) { if (type == currentType) { return true; } } return false; } /** * Check if the failure trace matches exact failure sequence of {@link MergeFailureType}s, in * level order from root to top. * * @param sequence * the sequence of {@link MergeFailureType}s to check * @return if the sequence is matches exactly */ public boolean traceMatchesExactFailureSequence(final List sequence) { if (sequence == null) { return false; } if (sequence.size() != this.failureTypeTrace.size()) { return false; } return this.failureTypeTrace.equals(sequence); } /** * Add a new top-level {@link MergeFailureType} to the failure trace. * * @param type * the {@link MergeFailureType} to add * @return the modified trace */ public List withNewTopLevelFailure(final MergeFailureType type) { final List newList = new ArrayList<>(this.failureTypeTrace); newList.add(type); return newList; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/exception/change/MergeFailureType.java ================================================ package org.openstreetmap.atlas.exception.change; /** * @author lcram */ public enum MergeFailureType { /* * These are all root level failures. */ AUTOFAIL_TAG_MERGE("tag Map merge failed due to autofail strategy"), AUTOFAIL_LONG_SET_MERGE("Long Set merge failed due to autofail strategy"), AUTOFAIL_LONG_SORTED_SET_MERGE("Long SortedSet merge failed due to autofail strategy"), AUTOFAIL_LOCATION_MERGE("Location merge failed due to autofail strategy"), AUTOFAIL_POLYLINE_MERGE("PolyLine merge failed due to autofail strategy"), AUTOFAIL_POLYGON_MERGE("Polygon merge failed due to autofail strategy"), AUTOFAIL_LONG_MERGE("Long merge failed due to autofail strategy"), SIMPLE_TAG_MERGE_FAIL("simpleTagMerger failed"), SIMPLE_LONG_SET_MERGE_FAIL("simpleLongSetMerger failed"), SIMPLE_LONG_SORTED_SET_MERGE_FAIL("simpleLongSortedSetMerger failed"), SIMPLE_RELATION_BEAN_MERGE_FAIL("simpleRelationBeanMerger failed"), DIFF_BASED_RELATION_BEAN_REMOVE_REMOVE_CONFLICT( "diffBasedRelationBeanMerger failed due to REMOVE/REMOVE conflict"), DIFF_BASED_RELATION_BEAN_ADD_REMOVE_CONFLICT( "diffBasedRelationBeanMerger failed due to ADD/REMOVE conflict"), DIFF_BASED_RELATION_BEAN_ADD_ADD_CONFLICT( "diffBasedRelationBeanMerger failed due to ADD/ADD conflict"), DIFF_BASED_TAG_ADD_ADD_CONFLICT("diffBasedTagMerger failed due to ADD/ADD conflict"), DIFF_BASED_TAG_ADD_REMOVE_CONFLICT("diffBasedTagMerger failed due to ADD/REMOVE conflict"), DIFF_BASED_LONG_MERGE_FAIL("diffBasedLongMerger failed"), DIFF_BASED_LOCATION_MERGE_FAIL("diffBasedLocationMerger failed"), DIFF_BASED_POLYLINE_MERGE_FAIL("diffBasedPolyLineMerger failed"), DIFF_BASED_POLYGON_MERGE_FAIL("diffBasedPolygonMerger failed"), CONFLICTING_BEFORE_VIEW_SET_ADD_REMOVE_CONFLICT( "conflictingBeforeViewSetMerger failed due to ADD/REMOVE conflict"), CONFLICTING_BEFORE_VIEW_RELATION_BEAN_ADD_REMOVE_CONFLICT( "conflictingBeforeViewRelationBeanMerger failed due to ADD/REMOVE conflict"), MUTUALLY_EXCLUSIVE_ADD_ADD_CONFLICT( "diffBasedMutuallyExclusiveMerger failed due to ADD/ADD conflict"), FEATURE_CHANGE_INVALID_ADD_REMOVE_MERGE( "left and right FeatureChanges disagreed on ChangeType"), FEATURE_CHANGE_INVALID_PROPERTIES_MERGE( "left and right FeatureChanges disagreed on ID or ItemType"), FEATURE_CHANGE_IMBALANCED_BEFORE_VIEW( "left and right FeatureChanges did not both have beforeViews"), /* * These failures occur at the next level up from root. They differentiate between * afterView/beforeView merge errors, and do not contain specific member information. */ AFTER_VIEW_NO_BEFORE_VIEW_MERGE_STRATEGY_FAILED( "the afterView merging function (that ignores beforeView) failed"), AFTER_VIEW_CONSISTENT_BEFORE_VIEW_MERGE_STRATEGY_FAILED( "the afterView merging function (that assumes consistent beforeViews) failed"), AFTER_VIEW_CONFLICTING_BEFORE_VIEW_MERGE_STRATEGY_FAILED( "the afterView merging function (that accounts for conflicting beforeViews) failed"), BEFORE_VIEW_MERGE_STRATEGY_FAILED("the beforeView merging function failed"), MISSING_BEFORE_VIEW_MERGE_STRATEGY( "beforeMembers conflict and no beforeView merging strategy provided"), MISSING_AFTER_VIEW_MERGE_STRATEGY_WITH_BEFORE_MEMBER_CONFLICT_HANDLING( "beforeMembers conflict and no beforeView-conflict-capable afterView merging strategy provided"), MISSING_AFTER_VIEW_MERGE_STRATEGY( "afterMembers conflict and no afterView merging strategy provided"), /* * The generic highest level merge failure. */ HIGHEST_LEVEL_MERGE_FAILURE("the FeatureChange merge failed"); private final String description; MergeFailureType(final String description) { this.description = description; } public String getDescription() { return this.description; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Altitude.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.ElevationTag; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * This is the height or elevation (usually defined in meters) above the Earth ellipsoid. The Earth * ellipsoid is a mathematical surface defined by a semi-major axis and a semi-minor axis. The most * common values for these two parameters are defined by the World Geodetic Standard 1984 (WGS-84). * The WGS-84 ellipsoid is intended to correspond to mean sea level. An {@link Altitude} of zero * corresponds roughly to sea level, with positive values increasing away from the Earth’s center. * Altitude values range from the center of the Earth (see {@link Distance#AVERAGE_EARTH_RADIUS}) to * positive infinity. For more detail, see * * here. *

* Please also note that this is NOT the same elevation (height above sea level) as referenced by * the {@link ElevationTag} in OSM. * * @author mgostintsev */ public final class Altitude implements Serializable { private static final long serialVersionUID = -9064525655677062110L; public static final Altitude MEAN_SEA_LEVEL = Altitude.meters(0); private final Distance distance; // The altitude will be negative in the range between the center of the earth and sea level: // [-AVERAGE_EARTH_RADIUS to 0). Even though the underlying altitude is negative, the // representation will be positive to make use of the Distance functionality. private boolean isNegative = false; public static Altitude meters(final double meters) { return new Altitude(meters); } private Altitude(final double meters) { if (meters < 0) { if (-meters > Distance.AVERAGE_EARTH_RADIUS.asMeters()) { throw new CoreException("Cannot have an altitude below the center of the Earth."); } this.isNegative = true; this.distance = Distance.meters(-meters); } else { this.distance = Distance.meters(meters); } } public double asMeters() { return this.isNegative ? -this.distance.asMeters() : this.distance.asMeters(); } @Override public boolean equals(final Object other) { if (other instanceof Altitude) { final Altitude that = (Altitude) other; return this.asMeters() == that.asMeters(); } return false; } @Override public int hashCode() { return Double.hashCode(this.asMeters()); } @Override public String toString() { return String.valueOf(this.asMeters()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/CompressedPolyLine.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import java.util.ArrayList; import java.util.List; /** * A {@link PolyLine} that is compressed using delta encoding. This is efficient when the * {@link PolyLine} has a lot of points close by. * * @author matthieun */ public class CompressedPolyLine implements Located, Serializable { /** * @author matthieun */ private static class ByteSign { private final byte[] bytes; private final boolean sign; ByteSign(final byte[] bytes, final boolean sign) { this.bytes = bytes; this.sign = sign; } public byte[] getBytes() { return this.bytes; } public boolean isSign() { return this.sign; } } private static final long serialVersionUID = -7813027521625470225L; private static final int BYTE_FULL_MASK = 0xFF; private static final int BYTE_SIZE = 8; private static final int INT_SIGN_MASK = 0x80000000; private static final int INT_NO_SIGN_MASK = 0x7FFFFFFF; private final byte[][] positions; private final boolean[] signs; public CompressedPolyLine(final byte[][] positions, final boolean[] signs) { this.positions = positions; this.signs = signs; } /** * Create a compressed version of a {@link PolyLine} * * @param polyLine * The {@link PolyLine} to compress. */ public CompressedPolyLine(final PolyLine polyLine) { final List positions = new ArrayList<>(); final List signs = new ArrayList<>(); int formerLatitude = 0; int formerLongitude = 0; for (final Location location : polyLine) { final int latitude = (int) location.getLatitude().asDm7(); final int longitude = (int) location.getLongitude().asDm7(); final int deltaLatitude = latitude - formerLatitude; final int deltaLongitude = longitude - formerLongitude; formerLatitude = latitude; formerLongitude = longitude; final ByteSign latShrink = shrink(deltaLatitude); final ByteSign lonShrink = shrink(deltaLongitude); positions.add(latShrink.getBytes()); signs.add(latShrink.isSign()); positions.add(lonShrink.getBytes()); signs.add(lonShrink.isSign()); } this.positions = new byte[positions.size()][]; for (int i = 0; i < positions.size(); i++) { this.positions[i] = positions.get(i); } this.signs = new boolean[signs.size()]; for (int i = 0; i < signs.size(); i++) { this.signs[i] = signs.get(i); } } /** * @return An expanded {@link PolyLine} */ public PolyLine asPolyLine() { boolean lat = true; int latitude = 0; int longitude = 0; final List locations = new ArrayList<>(); for (int index = 0; index < this.positions.length; index++) { final byte[] result = this.positions[index]; if (lat) { latitude += expand(result, index); lat = false; } else { longitude += expand(result, index); locations.add(new Location(Latitude.dm7(latitude), Longitude.dm7(longitude))); lat = true; } } return new PolyLine(locations); } @Override public Rectangle bounds() { return asPolyLine().bounds(); } public byte[][] getPositions() { return this.positions; } public boolean[] getSigns() { return this.signs; } @Override public String toString() { return asPolyLine().toString(); } /** * Transform an array of bytes into an int. If the bytes are 0x4A and 0x0F, with a negative sign * (from the index in the signs array) the returned int will be 0x80000F4A. * * @param result * The shrunk value * @param index * The index of the sign * @return The expanded value */ private int expand(final byte[] result, final int index) { int placeholder = 0; for (int i = 0; i < result.length; i++) { final byte byteValue = result[i]; placeholder |= byteValue & BYTE_FULL_MASK; if (i < result.length - 1) { placeholder <<= BYTE_SIZE; } } final boolean negative = this.signs[index]; if (negative) { placeholder |= INT_SIGN_MASK; } return placeholder; } /** * Browse the value, byte after byte, and keep only the bytes that have significance. So an int * 0x80000F4A will return an array of two bytes, 0x4A and 0x0F, and a negative sign. All the 0 * bytes will be thrown out. * * @param value * The value to shrink * @return The shrunk value. */ private ByteSign shrink(final int value) { // Get rid of the sign int placeholder = value & INT_NO_SIGN_MASK; final List bytes = new ArrayList<>(); while (Math.abs(placeholder) > 0) { final byte byteValue = (byte) placeholder; bytes.add(byteValue); placeholder >>>= BYTE_SIZE; } final int size = bytes.size(); final byte[] result = new byte[size]; for (int i = 0; i < size; i++) { result[i] = bytes.get(size - 1 - i); } return new ByteSign(result, value < 0); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/CompressedPolygon.java ================================================ package org.openstreetmap.atlas.geography; /** * Compressed {@link Polygon}. This simply extends {@link CompressedPolyLine} * * @author matthieun */ public class CompressedPolygon extends CompressedPolyLine { private static final long serialVersionUID = 2762361356248033855L; public CompressedPolygon(final byte[][] positions, final boolean[] signs) { super(positions, signs); } public CompressedPolygon(final Polygon polygon) { super(polygon); } public Polygon asPolygon() { return new Polygon(asPolyLine()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/GeometricObject.java ================================================ package org.openstreetmap.atlas.geography; /** * @author matthieun */ public interface GeometricObject { double SIMILARITY_THRESHOLD = 0.9999999; boolean intersects(PolyLine other); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/GeometricSurface.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.utilities.scalars.Surface; /** * An interface for all geometric surface objects * * @author jklamer */ public interface GeometricSurface extends Located, GeometryPrintable, GeometricObject { boolean fullyGeometricallyEncloses(Location location); boolean fullyGeometricallyEncloses(MultiPolygon multiPolygon); boolean fullyGeometricallyEncloses(PolyLine polyLine); boolean overlaps(MultiPolygon multiPolygon); boolean overlaps(PolyLine polyLine); /** * @return The {@link Surface} of this {@link GeometricSurface}. Not valid if the * {@link GeometricSurface} self-intersects, and/or overlaps itself * @see "http://www.mathopenref.com/coordpolygonarea2.html" */ Surface surface(); /** * @return The approximate {@link Surface} of this {@link GeometricSurface} if it were projected * onto the Earth. Not valid if the {@link GeometricSurface} self-intersects, and/or * overlaps itself. Uses "Some Algorithms for Polygons on a Sphere" paper as reference. * @see "https://trs.jpl.nasa.gov/bitstream/handle/2014/41271/07-0286.pdf" */ Surface surfaceOnSphere(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/GeometryPrintable.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.geography.geojson.GeoJson; /** * @author matthieun */ public interface GeometryPrintable extends GeoJson, WktPrintable, WkbPrintable { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Heading.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.utilities.scalars.Angle; /** * A Heading, that is between 0 included and 360 degrees excluded, using the standard 0 = north, 90 * = east, 180 = south and 270 = west. Heading's degree circle starts from 0 and increases in the * clock-wise direction. Heading's circle is shifted forward 180 degrees (with * {@link Heading#DELTA_DM7}) compared to Angle's degree circle. * * @author matthieun */ public class Heading extends Angle { private static final long serialVersionUID = -7621572408790801458L; public static final Heading NORTH = Heading.dm7(0L); public static final Heading SOUTH = Heading.dm7(1_800_000_000L); public static final Heading EAST = Heading.dm7(900_000_000L); public static final Heading WEST = Heading.dm7(2_700_000_000L); /** * Delta between {@link Angle}'s degree circle and {@link Heading}'s degree circle */ protected static final int DELTA_DM7 = 1_800_000_000; /** * @param degrees * A heading value in degrees * @return The built {@link Heading} object using the degrees value */ public static Heading degrees(final double degrees) { return dm7(Math.round(degrees * DM7_PER_DEGREE)); } /** * @param dm7 * A heading value in degree of magnitude 7 (dm7) * @return The built {@link Heading} object using the dm7 value */ public static Heading dm7(final long dm7) { // Roll dm7 value from 0->360 to the -180->180 that the angle expects. // Heading's circle is 180 degree ahead of Angle's long rollingDm7 = (dm7 - DELTA_DM7) % REVOLUTION_DM7; // After the roll operation, dm7 value could fall into (-180, -360) degrees. // This addition of 360 degrees will shift that degree back into (0, 180) if (rollingDm7 < MINIMUM_DM7) { rollingDm7 += REVOLUTION_DM7; } // After the roll operation, dm7 value could fall into [180, 360) degrees. // This subtraction of 360 degrees will shift that degree back into [-180, 0) if (rollingDm7 >= MAXIMUM_DM7) { rollingDm7 -= REVOLUTION_DM7; } return new Heading((int) rollingDm7); } /** * @param radians * A heading value in Radians * @return The built {@link Heading} object using the Radians value */ public static Heading radians(final double radians) { return dm7(Math.round(radians * DM7_PER_RADIAN)); } protected Heading(final int dm7) { super(dm7); } @Override public long asDm7() { // Override to scale back to 0->360 return super.asDm7() + DELTA_DM7; } @Override public String toString() { return String.valueOf(this.asDegrees()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Latitude.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.utilities.scalars.Angle; /** * A Latitude between -90 degrees and +90 degrees, both included. * * @author matthieun * @author tony */ public class Latitude extends Angle { private static final long serialVersionUID = -1737858321716005802L; protected static final int MINIMUM_DM7 = -900_000_000; protected static final int MAXIMUM_DM7 = 900_000_000; public static final Latitude MINIMUM = Latitude.dm7(MINIMUM_DM7); public static final Latitude ZERO = Latitude.dm7(0L); public static final Latitude MAXIMUM = Latitude.dm7(MAXIMUM_DM7); /** * @param degrees * A Latitude value in degrees * @return The built {@link Latitude} object using the degrees value */ public static Latitude degrees(final double degrees) { return dm7(Math.round(degrees * DM7_PER_DEGREE)); } /** * @param dm7 * A latitude value in degree of magnitude 7 (dm7) * @return The built {@link Latitude} object using the dm7 value */ public static Latitude dm7(final long dm7) { if (dm7 < MINIMUM_DM7 || dm7 > MAXIMUM_DM7) { throw new IllegalArgumentException("Cannot have a latitude of " + dm7 / DM7_PER_DEGREE + " degrees which is outside of " + MINIMUM_DM7 / DM7_PER_DEGREE + " degrees -> " + MAXIMUM_DM7 / DM7_PER_DEGREE + " degrees."); } return new Latitude((int) dm7); } /** * @param radians * A Latitude value in Radians * @return The built {@link Latitude} object using the Radians value */ public static Latitude radians(final double radians) { return dm7(Math.round(radians * DM7_PER_RADIAN)); } /** * If the given radian exceeds the latitude boundary, will return the boundary value. * * @param radians * The radian of latitude * @return The adjusted latitude if exceeds the boundary, otherwise the normal latitude */ public static Latitude radiansBounded(final double radians) { long dm7 = Math.round(radians * DM7_PER_RADIAN); if (dm7 < MINIMUM_DM7) { dm7 = MINIMUM_DM7; } if (dm7 > MAXIMUM_DM7) { dm7 = MAXIMUM_DM7; } return dm7(dm7); } /** * Constructor * * @param dm7 * The latitude value in dm7 */ protected Latitude(final int dm7) { super(dm7); if (dm7 < MINIMUM_DM7 || dm7 > MAXIMUM_DM7) { throw new IllegalArgumentException("Invalid Latitude microdegrees value: " + dm7); } } @Override public String toString() { return String.valueOf(this.asDegrees()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Located.java ================================================ package org.openstreetmap.atlas.geography; /** * Contract for any item that can be geographcally bound by a {@link Rectangle} * * @author matthieun * @author Yazad Khambata */ public interface Located { /** * @return The bounds around this located object */ Rectangle bounds(); /** * Return {@code true} if surface fully geometrically encloses {@code this}. *

* For backward compatibility a default implementation that fails is added. * * @param surface * - check if {@code this} is fully within the surface. * @return - {@code true} if fully within surface, false otherwise. */ default boolean within(GeometricSurface surface) { throw new UnsupportedOperationException(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Location.java ================================================ package org.openstreetmap.atlas.geography; import java.awt.Point; import java.awt.geom.Point2D; import java.io.Serializable; import java.util.Iterator; import java.util.Random; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Snapper.SnappedLocation; import org.openstreetmap.atlas.geography.converters.WkbLocationConverter; import org.openstreetmap.atlas.geography.converters.WktLocationConverter; import org.openstreetmap.atlas.geography.coordinates.EarthCenteredEarthFixedCoordinate; import org.openstreetmap.atlas.geography.coordinates.GeodeticCoordinate; import org.openstreetmap.atlas.geography.geojson.GeoJsonGeometry; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.gson.JsonObject; /** * Location on the surface of the earth * * @author matthieun * @author mgostintsev */ public class Location implements Located, Iterable, Serializable, GeometryPrintable, GeoJsonGeometry, GeometricObject { public static final String TEST_1_COORDINATES = "37.335310,-122.009566"; public static final String TEST_2_COORDINATES = "37.321628,-122.028464"; public static final String TEST_3_COORDINATES = "37.317585,-122.052138"; public static final String TEST_4_COORDINATES = "37.332451,-122.028932"; public static final String TEST_5_COORDINATES = "37.390535,-122.031007"; public static final String TEST_6_COORDINATES = "37.325440,-122.033948"; public static final String TEST_7_COORDINATES = "37.3314171,-122.0304871"; public static final String TEST_8_COORDINATES = "37.3214159,-122.0303831"; // Quick-access locations, mostly used for testing. public static final Location TEST_1 = Location.forString(TEST_1_COORDINATES); public static final Location TEST_2 = Location.forString(TEST_2_COORDINATES); public static final Location TEST_3 = Location.forString(TEST_3_COORDINATES); public static final Location TEST_4 = Location.forString(TEST_4_COORDINATES); public static final Location TEST_5 = Location.forString(TEST_5_COORDINATES); public static final Location TEST_6 = Location.forString(TEST_6_COORDINATES); public static final Location TEST_7 = Location.forString(TEST_7_COORDINATES); public static final Location TEST_8 = Location.forString(TEST_8_COORDINATES); public static final Location STEVENS_CREEK = Location.forString("37.324233,-122.003467"); public static final Location CROSSING_85_280 = Location.forString("37.332439,-122.055760"); public static final Location CROSSING_85_17 = Location.forString("37.255731,-121.955918"); public static final Location EIFFEL_TOWER = Location.forString("48.858241,2.294495"); public static final Location COLOSSEUM = Location.forString("41.890224,12.492340"); public static final Location CENTER = new Location(0L); private static final long serialVersionUID = 3770424147251047128L; private static final int INT_FULL_MASK = 0xFFFFFFFF; private static final long INT_FULL_MASK_AS_LONG = 0xFFFFFFFFL; private static final int INT_SIZE = 32; private static final int FACTOR_OF_3 = 3; private static final Random RANDOM = new Random(); private final Latitude latitude; private final Longitude longitude; /** * @param locationString * The {@link Location} as a {@link String} in "latitude(degrees),longitude(degrees)" * format * @return The corresponding {@link Location} */ public static Location forString(final String locationString) { final StringList split = StringList.split(locationString, ","); if (split.size() != 2) { throw new CoreException("Invalid Location String: {}", locationString); } final double latitude = Double.parseDouble(split.get(0)); final double longitude = Double.parseDouble(split.get(1)); return new Location(Latitude.degrees(latitude), Longitude.degrees(longitude)); } /** * @param locationString * The {@link Location} as a {@link String} in "longitude(degrees),latitude(degrees)" * format * @return The corresponding {@link Location} */ public static Location forStringLongitudeLatitude(final String locationString) { final StringList split = StringList.split(locationString, ","); if (split.size() != 2) { throw new CoreException("Invalid Location String: {}", locationString); } final double latitude = Double.parseDouble(split.get(1)); final double longitude = Double.parseDouble(split.get(0)); return new Location(Latitude.degrees(latitude), Longitude.degrees(longitude)); } /** * @param wkt * The {@link Location} as a Well Known Text (WKT) {@link String} format * @return The corresponding {@link Location} */ public static Location forWkt(final String wkt) { return new WktLocationConverter().backwardConvert(wkt); } /** * @param bounds * Bounds to constrain the result * @return A random location within the bounds */ public static Location random(final Rectangle bounds) { final int latitude = RANDOM.ints((int) bounds.lowerLeft().getLatitude().asDm7(), (int) bounds.upperRight().getLatitude().asDm7()).iterator().next(); final int longitude = RANDOM.ints((int) bounds.lowerLeft().getLongitude().asDm7(), (int) bounds.upperRight().getLongitude().asDm7()).iterator().next(); return new Location(Latitude.dm7(latitude), Longitude.dm7(longitude)); } /** * Build a {@link Location} from a {@link Latitude} and a {@link Longitude} objects. * * @param latitude * The {@link Latitude} to use * @param longitude * The {@link Longitude} to use */ public Location(final Latitude latitude, final Longitude longitude) { if (latitude == null) { throw new CoreException("Latitude is null."); } if (longitude == null) { throw new CoreException("Longitude is null."); } this.latitude = latitude; this.longitude = longitude; } /** * Copy constructor for {@link Location} * * @param other * the {@link Location} from which to copy */ public Location(final Location other) { if (other == null) { throw new CoreException("Other Location was null"); } this.latitude = other.latitude; this.longitude = other.longitude; } /** * Create a location from a dm7 latitude and dm7 longitude concatenated in a long * * @param concatenation * The first 32 bits are for the dm7 latitude, and the last 32 bits are for the dm7 * longitude. */ public Location(final long concatenation) { final int lon = (int) concatenation; final int lat = (int) (concatenation >>> INT_SIZE) & INT_FULL_MASK; this.longitude = Longitude.dm7(lon); this.latitude = Latitude.dm7(lat); } /** * @return A dm7 latitude and dm7 longitude concatenated in a long. The first 32 bits are for * the dm7 latitude, and the last 32 bits are for the dm7 longitude. */ public long asConcatenation() { long result = this.latitude.asDm7(); result <<= INT_SIZE; result |= this.longitude.asDm7() & INT_FULL_MASK_AS_LONG; return result; } @Override public JsonObject asGeoJsonGeometry() { return GeoJsonUtils.geometry(GeoJsonType.POINT, GeoJsonUtils.coordinate(this)); } @Override public Rectangle bounds() { return Rectangle.forCorners(this, this); } /** * Get a {@link Rectangle} around this {@link Location} * * @param extension * The height of the 1/2 {@link Rectangle}. The height of the total {@link Rectangle} * will be twice that. Same for width. * @return The {@link Rectangle} around this {@link Location} */ public Rectangle boxAround(final Distance extension) { final Location north = this.shiftAlongGreatCircle(Heading.NORTH, extension); final Location south = this.shiftAlongGreatCircle(Heading.SOUTH, extension); final Location east = this.shiftAlongGreatCircle(Heading.EAST, extension); final Location west = this.shiftAlongGreatCircle(Heading.WEST, extension); return Rectangle.forLocations(north, south, east, west); } /** * @param that * The other {@link Location} to compute the {@link Distance} to * @return The {@link Distance} between the two {@link Location} */ public Distance distanceTo(final Location that) { // Do a quick check on Longitude if (this.getLongitude().isCloserViaAntimeridianTo(that.getLongitude())) { // Use the method that is not annoyed by the antimeridian return haversineDistanceTo(that); } return equirectangularDistanceTo(that); } @Override public boolean equals(final Object other) { if (other instanceof Location) { final Location that = (Location) other; return this.getLatitude().equals(that.getLatitude()) && this.getLongitude().equals(that.getLongitude()); } return false; } /** * An equirectangular approximation distance between two locations, better performance but less * accurate. It is especially not able to handle distances that cross the antimeridian. It would * compute the distance all the way around the world instead. * * @param that * The other point to compute the distance to * @return The equirectangular distance * @see "http://www.movable-type.co.uk/scripts/latlong.html" */ public Distance equirectangularDistanceTo(final Location that) { // convert to radians final double lat1 = this.getLatitude().asRadians(); final double lon1 = this.getLongitude().asRadians(); final double lat2 = that.getLatitude().asRadians(); final double lon2 = that.getLongitude().asRadians(); final double xAxis = (lon2 - lon1) * Math.cos((lat1 + lat2) / 2); final double yAxis = lat2 - lat1; return Distance.AVERAGE_EARTH_RADIUS.scaleBy(Math.sqrt(xAxis * xAxis + yAxis * yAxis)); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.POINT; } /** * @return This {@link Location}'s {@link Latitude} */ public Latitude getLatitude() { return this.latitude; } /** * @return This {@link Location}'s {@link Longitude} */ public Longitude getLongitude() { return this.longitude; } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} and the other {@link Location} to test are on the same * East-West line. */ public boolean hasSameLatitudeAs(final Location other) { return this.getLatitude().equals(other.getLatitude()); } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} and the other {@link Location} to test are on the same * North-South line. */ public boolean hasSameLongitudeAs(final Location other) { return this.getLongitude().equals(other.getLongitude()); } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (this.latitude == null ? 0 : this.latitude.hashCode()); result = prime * result + (this.longitude == null ? 0 : this.longitude.hashCode()); return result; } /** * This uses the ‘haversine’ formula to calculate the great-circle distance between two * locations, more calculation but more accurate * * @param that * The other point to compute the distance to * @return The haversine distance * @see "http://www.movable-type.co.uk/scripts/latlong.html" */ public Distance haversineDistanceTo(final Location that) { // convert to radians final double lat1 = this.getLatitude().asRadians(); final double lon1 = this.getLongitude().asRadians(); final double lat2 = that.getLatitude().asRadians(); final double lon2 = that.getLongitude().asRadians(); final double deltaLat = lat2 - lat1; final double deltaLon = lon2 - lon1; final double hav = Math.pow(Math.sin(deltaLat / 2), 2) + Math.cos(lat1) * Math.cos(lat2) * Math.pow(Math.sin(deltaLon / 2), 2); final double result = 2 * Math.atan2(Math.sqrt(hav), Math.sqrt(1 - hav)); return Distance.AVERAGE_EARTH_RADIUS.scaleBy(result); } /** * This computes the initial heading (heading at the start point) of the segment on the surface * of earth between two locations * * @see "http://www.movable-type.co.uk/scripts/latlong.html" * @param that * The other point to compute the heading to * @return The heading between two points */ public Heading headingTo(final Location that) { if (this.equals(that)) { throw new CoreException("Cannot compute some heading when two points are the same."); } // convert to radians final double lat1 = this.getLatitude().asRadians(); final double lon1 = this.getLongitude().asRadians(); final double lat2 = that.getLatitude().asRadians(); final double lon2 = that.getLongitude().asRadians(); final double deltaLon = lon2 - lon1; final double yAxis = Math.sin(deltaLon) * Math.cos(lat2); final double xAxis = Math.cos(lat1) * Math.sin(lat2) - Math.sin(lat1) * Math.cos(lat2) * Math.cos(deltaLon); return Heading.radians(Math.atan2(yAxis, xAxis)); } @Override public boolean intersects(final PolyLine polyLine) { return polyLine.intersects(new Segment(this, this)); } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} is east of the other {@link Location}. */ public boolean isEastOf(final Location other) { return this.getLongitude().isGreaterThan(other.getLongitude()); } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} is east or on the same {@link Longitude} as the other * {@link Location}. */ public boolean isEastOfOrOnTheSameLatitudeAs(final Location other) { return this.getLongitude().isGreaterThanOrEqualTo(other.getLongitude()); } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} is north of the other {@link Location}. */ public boolean isNorthOf(final Location other) { return this.getLatitude().isGreaterThan(other.getLatitude()); } /** * @param other * The other {@link Location} to test * @return True if this {@link Location} is north or on the same {@link Latitude} as the other * {@link Location}. */ public boolean isNorthOfOrOnTheSameLatitudeAs(final Location other) { return this.getLatitude().isGreaterThanOrEqualTo(other.getLatitude()); } @Override public Iterator iterator() { return Iterables.from(this).iterator(); } /** * Midpoint along a Rhumb line between this point and that point * * @param that * The other point to compute the midpoint between * @return The {@link Location} of the loxodromic midpoint * @see "http://www.movable-type.co.uk/scripts/latlong.html" */ public Location loxodromicMidPoint(final Location that) { // Convert to Radians final double lat1 = this.getLatitude().asRadians(); double lon1 = this.getLongitude().asRadians(); final double lat2 = that.getLatitude().asRadians(); final double lon2 = that.getLongitude().asRadians(); // Crossing anti-meridian if (Math.abs(lon2 - lon1) > Math.PI) { lon1 += 2 * Math.PI; } final double pheta = (lat1 + lat2) / 2; final double phi1 = Math.tan(Math.PI / 4 + lat1 / 2); final double phi2 = Math.tan(Math.PI / 4 + lat2 / 2); final double phi3 = Math.tan(Math.PI / 4 + pheta / 2); double lambda = ((lon2 - lon1) * Math.log(phi3) + lon1 * Math.log(phi2) - lon2 * Math.log(phi1)) / Math.log(phi2 / phi1); // Locations on the same circle of latitude do not produce a finite lambda value above. // Locations at the same longitude (especially the antimeridian) should preserve their sign. // All other locations should be be normalized within [-180, +180). if (!Double.isFinite(lambda) || lon1 == lon2) { lambda = (lon1 + lon2) / 2; } else { lambda = (lambda + FACTOR_OF_3 * Math.PI) % (2 * Math.PI) - Math.PI; } return new Location(Latitude.radians(pheta), Longitude.radians(lambda)); } /** * The half-way point along a great-circle path between this and that point * * @param that * The other point to compute the midpoint between * @return The {@link Location} of the midpoint * @see "http://www.movable-type.co.uk/scripts/latlong.html" */ public Location midPoint(final Location that) { // Convert to Radians final double lat1 = this.getLatitude().asRadians(); final double lon1 = this.getLongitude().asRadians(); final double lat2 = that.getLatitude().asRadians(); final double lon2 = that.getLongitude().asRadians(); final double longitudeDelta = lon2 - lon1; final double xBearing = Math.cos(lat2) * Math.cos(longitudeDelta); final double yBearing = Math.cos(lat2) * Math.sin(longitudeDelta); final double pheta = Math.atan2(Math.sin(lat1) + Math.sin(lat2), Math.sqrt( (Math.cos(lat1) + xBearing) * (Math.cos(lat1) + xBearing) + yBearing * yBearing)); double lambda = lon1 + Math.atan2(yBearing, Math.cos(lat1) + xBearing); // Normalize to -180/180 lambda = (lambda + FACTOR_OF_3 * Math.PI) % (2 * Math.PI) - Math.PI; if (this.getLongitude().equals(Longitude.MAXIMUM) && that.getLongitude().equals(Longitude.MAXIMUM)) { lambda *= -1; } return new Location(Latitude.radians(pheta), Longitude.radians(lambda)); } /** * Shift a location along a great circle. Note that if the shifted location exceeds the boundary * of latitude (-90 to 90 degrees) or longitude (-180 to 180 degrees), it will use the boundary * instead * * @param initialHeading * Initial heading * @param distance * Distance along the great circle * @return The shifted location * @see "http://www.movable-type.co.uk/scripts/latlong.html" */ public Location shiftAlongGreatCircle(final Heading initialHeading, final Distance distance) { if (Distance.ZERO.equals(distance)) { return this; } // convert to radians final double latitude1 = this.getLatitude().asRadians(); final double longitude1 = this.getLongitude().asRadians(); final double bearing = initialHeading.asRadians(); final double latitude2 = Math.asin(Math.sin(latitude1) * Math.cos(distance.asMillimeters() / Distance.AVERAGE_EARTH_RADIUS.asMillimeters()) + Math.cos(latitude1) * Math.sin(distance.asMillimeters() / Distance.AVERAGE_EARTH_RADIUS.asMillimeters()) * Math.cos(bearing)); final double longitude2 = longitude1 + Math.atan2( Math.sin(bearing) * Math.sin(distance.asMillimeters() / Distance.AVERAGE_EARTH_RADIUS.asMillimeters()) * Math.cos(latitude1), Math.cos(distance.asMillimeters() / Distance.AVERAGE_EARTH_RADIUS.asMillimeters()) - Math.sin(latitude1) * Math.sin(latitude2)); return new Location(Latitude.radiansBounded(latitude2), Longitude.radiansBounded(longitude2)); } /** * Snap this {@link Location} to a {@link MultiPolygon} using a {@link Snapper} * * @param shape * The shape to snap to * @return The corresponding {@link SnappedLocation} */ public SnappedLocation snapTo(final MultiPolygon shape) { return new Snapper().snap(this, shape); } /** * Snap this {@link Location} to a {@link PolyLine} using a {@link Snapper} * * @param shape * The shape to snap to * @return The corresponding {@link SnappedLocation} */ public SnappedLocation snapTo(final PolyLine shape) { return new Snapper().snap(this, shape); } public String toCompactString() { return this.getLatitude() + "," + this.getLongitude(); } /** * @return the {@link EarthCenteredEarthFixedCoordinate} for this {@link Location}. */ public EarthCenteredEarthFixedCoordinate toEarthCenteredEarthFixedCoordinate() { return new EarthCenteredEarthFixedCoordinate(this); } /** * @return the {@link GeodeticCoordinate} for this {@link Location}. */ public GeodeticCoordinate toGeodeticCoordinate() { return new GeodeticCoordinate(this); } @Override public String toString() { return toWkt(); } @Override public byte[] toWkb() { return new WkbLocationConverter().convert(this); } @Override public String toWkt() { return new WktLocationConverter().convert(this); } @Override public boolean within(final GeometricSurface surface) { return surface.fullyGeometricallyEncloses(this); } protected Point2D asAwtPoint() { return new Point((int) getLongitude().asDm7(), (int) getLatitude().asDm7()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Longitude.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.utilities.scalars.Angle; /** * A Longitude between -180 degrees and +180 degrees, inclusive. * * @author matthieun * @author tony */ public class Longitude extends Angle { public static final Longitude MINIMUM = Longitude.dm7(MINIMUM_DM7); public static final Longitude ZERO = Longitude.dm7(0L); public static final Longitude MAXIMUM = Longitude.dm7(MAXIMUM_DM7); public static final Longitude ANTIMERIDIAN_WEST = Longitude.MINIMUM; public static final Longitude ANTIMERIDIAN_EAST = Longitude.MAXIMUM; private static final long serialVersionUID = 4222162088144937632L; private boolean isMaximumDm7 = false; /** * @param degrees * A Longitude value in degrees * @return The built {@link Longitude} object using the degrees value */ public static Longitude degrees(final double degrees) { return dm7(Math.round(degrees * DM7_PER_DEGREE)); } /** * @param dm7 * A longitude value in degree of magnitude 7 (dm7) * @return The built {@link Longitude} object using the dm7 value */ public static Longitude dm7(final long dm7) { if (dm7 < MINIMUM_DM7 || dm7 > MAXIMUM_DM7) { throw new IllegalArgumentException("Cannot have a longitude of " + dm7 / DM7_PER_DEGREE + " degrees which is outside of " + MINIMUM_DM7 / DM7_PER_DEGREE + " degrees -> " + MAXIMUM_DM7 / DM7_PER_DEGREE + " degrees."); } // This constructor depends on the Angle Constructor, which allows for overriding the // "assertDm7" method. return new Longitude((int) dm7); } /** * @param radians * A Longitude value in Radians * @return The built {@link Longitude} object using the Radians value */ public static Longitude radians(final double radians) { return dm7(Math.round(radians * DM7_PER_RADIAN)); } /** * If the given radian exceeds the longitude boundary, will return the boundary value. * * @param radians * The radian of longitude * @return The adjusted longitude if exceeds the boundary, otherwise the normal longitude */ public static Longitude radiansBounded(final double radians) { long dm7 = Math.round(radians * DM7_PER_RADIAN); if (dm7 < MINIMUM_DM7) { dm7 = MINIMUM_DM7; } if (dm7 >= MAXIMUM_DM7) { dm7 = MAXIMUM_DM7 - 1L; } return dm7(dm7); } /** * Constructor * * @param dm7 * The longitude value in dm7 */ protected Longitude(final int dm7) { super(dm7); if (dm7 == MAXIMUM_DM7) { this.isMaximumDm7 = true; } } @Override public long asDm7() { if (this.isMaximumDm7) { // This longitude was built with +180 and not -180, so make sure to return the same. return MAXIMUM_DM7; } else { return super.asDm7(); } } /** * @param that * The other {@link Longitude} to relate to * @return True if those two longitudes are closer to each other on the Antimeridian side than * on the Greenwich side. */ public boolean isCloserViaAntimeridianTo(final Longitude that) { // The difference in numerical longitude is larger than half a revolution return Math.abs(this.asDm7() - that.asDm7()) > REVOLUTION_DM7 / 2; } @Override public String toString() { return String.valueOf(this.asDegrees()); } @Override protected int assertDm7(final int dm7) { if (dm7 < MINIMUM_DM7 || dm7 > MAXIMUM_DM7) { throw new IllegalArgumentException("Longitude dm7 value " + dm7 + " is invalid."); } return dm7; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/MultiPolyLine.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.converters.WkbMultiPolyLineConverter; import org.openstreetmap.atlas.geography.converters.WktMultiPolyLineConverter; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonGeometry; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.streaming.readers.json.converters.PolyLineCoordinateConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import com.google.common.collect.Lists; import com.google.gson.JsonArray; import com.google.gson.JsonObject; /** * A MultiPolyLine is a set of distinct {@link PolyLine}s in a specific order * * @author yalimu */ public class MultiPolyLine implements Iterable, Located, Serializable, GeometryPrintable, GeoJsonGeometry { private static final long serialVersionUID = 5907807607388840698L; private final List polyLineList; /** * Create a {@link MultiPolyLine} from Well Known Text * * @param wkt * The Well Known Text * @return The {@link MultiPolyLine} */ public static MultiPolyLine wkt(final String wkt) { return new WktMultiPolyLineConverter().backwardConvert(wkt); } public MultiPolyLine(final Iterable polyLines) { this(Iterables.asList(polyLines)); } public MultiPolyLine(final List polyLines) { if (polyLines.isEmpty()) { throw new CoreException("Cannot have an empty list of PolyLine or Polygon."); } this.polyLineList = polyLines.stream().distinct().collect(Collectors.toList()); } public MultiPolyLine(final PolyLine... polyLines) { this(Iterables.iterable(polyLines)); } @Override public JsonObject asGeoJsonGeometry() { final PolyLineCoordinateConverter converter = new PolyLineCoordinateConverter(); final JsonArray coordinateArray = new JsonArray(); Iterables.stream(this).map(converter::convert).forEach(coordinateArray::add); return GeoJsonUtils.geometry(this.getGeoJsonType(), coordinateArray); } public Iterable asLocationIterableProperties() { return this.polyLineList.stream() .map(polyLine -> new GeoJsonBuilder.LocationIterableProperties(polyLine, new HashMap<>())) .collect(Collectors.toList()); } @Override public Rectangle bounds() { final List locations = Lists.newArrayList(); this.polyLineList.stream().map(PolyLine::getPoints).forEach(locations::addAll); return Rectangle.forLocations(locations); } @Override public boolean equals(final Object other) { if (!(other instanceof MultiPolyLine)) { return false; } final MultiPolyLine otherItem = (MultiPolyLine) other; return new HashSet<>(this.polyLineList).equals(new HashSet<>(otherItem.getPolyLineList())); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.MULTI_LINESTRING; } public List getPolyLineList() { return this.polyLineList; } @Override public int hashCode() { final StringBuilder stringBuilder = new StringBuilder(); for (final PolyLine polyLine : this.polyLineList) { stringBuilder.append(polyLine.hashCode()); } return stringBuilder.toString().hashCode(); } @Override public Iterator iterator() { return this.polyLineList.iterator(); } @Override public byte[] toWkb() { return new WkbMultiPolyLineConverter().convert(this); } @Override public String toWkt() { return new WktMultiPolyLineConverter().convert(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/MultiPolygon.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.locationtech.jts.algorithm.match.HausdorffSimilarityMeasure; import org.locationtech.jts.geom.prep.PreparedGeometry; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.operation.valid.IsValidOp; import org.locationtech.jts.operation.valid.TopologyValidationError; import org.openstreetmap.atlas.geography.clipping.Clip; import org.openstreetmap.atlas.geography.clipping.Clip.ClipType; import org.openstreetmap.atlas.geography.clipping.GeometryOperation; import org.openstreetmap.atlas.geography.converters.MultiPolygonStringConverter; import org.openstreetmap.atlas.geography.converters.WkbMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.WktMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsLocationConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonGeometry; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.maps.MultiMap; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Surface; import org.openstreetmap.atlas.utilities.tuples.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * Multiple {@link Polygon}s some inner, some outer. * * @author matthieun */ public class MultiPolygon implements Iterable, GeometricSurface, Serializable, GeoJsonGeometry { public static final MultiPolygon MAXIMUM = forPolygon(Rectangle.MAXIMUM); public static final MultiPolygon TEST_MULTI_POLYGON; private static final Logger logger = LoggerFactory.getLogger(MultiPolygon.class); private static final long serialVersionUID = 4198234682870043547L; private static final int SIMPLE_STRING_LENGTH = 200; private static final JtsMultiPolygonToMultiPolygonConverter JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final JtsPointConverter JTS_POINT_CONVERTER = new JtsPointConverter(); private static final JtsPolyLineConverter JTS_POLYLINE_CONVERTER = new JtsPolyLineConverter(); private static final JtsPolygonConverter JTS_POLYGON_CONVERTER = new JtsPolygonConverter(); static { final MultiMap outerToInners = new MultiMap<>(); final Polygon outer = new Polygon(Location.CROSSING_85_280, Location.CROSSING_85_17, Location.TEST_1, Location.TEST_5); final Polygon inner = new Polygon(Location.TEST_6, Location.TEST_2, Location.TEST_7); outerToInners.add(outer, inner); TEST_MULTI_POLYGON = new MultiPolygon(outerToInners); } private transient PreparedGeometry prepared; private final MultiMap outerToInners; private Rectangle bounds; /** * @param polygons * The outers of the multipolygon * @return A {@link MultiPolygon} with the provided {@link Polygon} as a single outer * {@link Polygon}, with no inner {@link Polygon} */ public static MultiPolygon forOuters(final Iterable polygons) { final MultiMap multiMap = new MultiMap<>(); polygons.forEach(polygon -> multiMap.put(polygon, Collections.emptyList())); return new MultiPolygon(multiMap); } public static MultiPolygon forOuters(final Polygon... polygons) { return MultiPolygon.forOuters(Arrays.asList(polygons)); } /** * @param polygon * A simple {@link Polygon} * @return A {@link MultiPolygon} with the provided {@link Polygon} as a single outer * {@link Polygon}, with no inner {@link Polygon} */ public static MultiPolygon forPolygon(final Polygon polygon) { final MultiMap multiMap = new MultiMap<>(); multiMap.put(polygon, new ArrayList<>()); return new MultiPolygon(multiMap); } /** * Generate a {@link MultiPolygon} from Well Known Text * * @param wkt * The {@link MultiPolygon} in well known text * @return The parsed {@link MultiPolygon} */ public static MultiPolygon wkt(final String wkt) { return new WktMultiPolygonConverter().backwardConvert(wkt); } public MultiPolygon(final MultiMap outerToInners) { this.outerToInners = outerToInners; } public GeoJsonObject asGeoJsonFeatureCollection() { final GeoJsonBuilder builder = new GeoJsonBuilder(); return builder.createFeatureCollection(Iterables.translate(outers(), outerPolygon -> builder.createOneOuterMultiPolygon( new MultiIterable<>(Collections.singleton(outerPolygon), this.outerToInners.get(outerPolygon))))); } /** * Creates a JsonObject with GeoJSON geometry representing this multi-polygon. * * @return A JsonObject with GeoJSON geometry */ @Override public JsonObject asGeoJsonGeometry() { return GeoJsonUtils.geometry(GeoJsonType.MULTI_POLYGON, GeoJsonUtils.multiPolygonToCoordinates(this)); } public Iterable asLocationIterableProperties() { final Iterable outers = Iterables.translate(outers(), polygon -> { final Map tags = new HashMap<>(); tags.put("MultiPolygon", "outer"); return new LocationIterableProperties(polygon, tags); }); final Iterable inners = Iterables.translate(inners(), polygon -> { final Map tags = new HashMap<>(); tags.put("MultiPolygon", "inner"); return new LocationIterableProperties(polygon, tags); }); return new MultiIterable<>(outers, inners); } /** * @return Optional of {@link Polygon} representation if possible */ public Optional asSimplePolygon() { if (this.isSimplePolygon()) { return outers().stream().findFirst(); } logger.warn("Trying to read complex MultiPolygon as simple Polygon"); return Optional.empty(); } @Override public Rectangle bounds() { if (this.bounds == null && !this.isEmpty()) { final Set locations = new HashSet<>(); forEach(polygon -> polygon.forEach(locations::add)); this.bounds = Rectangle.forLocations(locations); } return this.bounds; } /** * @param clipping * The {@link MultiPolygon} clipping that {@link MultiPolygon} * @param clipType * The type of clip (union, or, and or xor) * @return The {@link Clip} container, that can return the clipped {@link MultiPolygon} */ public Clip clip(final MultiPolygon clipping, final ClipType clipType) { return new Clip(clipType, this, clipping); } /** * Concatenate multiple {@link MultiPolygon}s into one. If the two {@link MultiPolygon}s happen * to have the same outer polygon, then the other's inner polygons will be added and the * current's inner polygons will be erased. * * @param other * The other {@link MultiPolygon} to concatenate. * @return The concatenated {@link MultiPolygon} */ public MultiPolygon concatenate(final MultiPolygon other) { final MultiMap result = new MultiMap<>(); result.putAll(getOuterToInners()); result.putAll(other.getOuterToInners()); return new MultiPolygon(result); } @Override public boolean equals(final Object other) { if (other instanceof MultiPolygon) { final MultiPolygon that = (MultiPolygon) other; final Set thatOuters = that.outers(); if (thatOuters.size() != this.outers().size()) { return false; } for (final Polygon outer : this.outers()) { if (!thatOuters.contains(outer)) { return false; } final List thatInners = that.innersOf(outer); if (thatInners.size() != this.innersOf(outer).size()) { return false; } for (final Polygon inner : this.innersOf(outer)) { if (!thatInners.contains(inner)) { return false; } } } return true; } return false; } /** * @param location * A {@link Location} item * @return True if the {@link MultiPolygon} contains the provided item (i.e. it is within the * outer polygons and not within the inner polygons) */ @Override public boolean fullyGeometricallyEncloses(final Location location) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } return this.prepared.covers(JTS_POINT_CONVERTER.convert(location)); } /** * Tests to see if entire surface of the provided {@link MultiPolygon} lies within this * {@link MultiPolygon} * * @param that * the provided {@link MultiPolygon} to test * @return true if the conditions are met, false otherwise */ @Override public boolean fullyGeometricallyEncloses(final MultiPolygon that) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } return this.prepared .covers(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(that)); } /** * @param polyLine * A {@link PolyLine} item * @return True if the {@link MultiPolygon} contains the provided {@link PolyLine}. */ @Override public boolean fullyGeometricallyEncloses(final PolyLine polyLine) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } if (polyLine instanceof Polygon) { return this.prepared.covers(JTS_POLYGON_CONVERTER.convert((Polygon) polyLine)); } return this.prepared.covers(JTS_POLYLINE_CONVERTER.convert(polyLine)); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.MULTI_POLYGON; } public MultiMap getOuterToInners() { return this.outerToInners; } @Override public int hashCode() { int result = 0; for (final Polygon polygon : this) { result += polygon.hashCode(); } return result; } public List inners() { return this.outerToInners.allValues(); } public List innersOf(final Polygon outer) { if (this.outerToInners.containsKey(outer)) { return this.outerToInners.get(outer); } return new ArrayList<>(); } @Override public boolean intersects(final PolyLine polyLine) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } if (polyLine instanceof Polygon) { return this.prepared.intersects(JTS_POLYGON_CONVERTER.convert((Polygon) polyLine)); } return this.prepared.intersects(JTS_POLYLINE_CONVERTER.convert(polyLine)); } /** * @return {@code true} if this {@link MultiPolygon} doesn't have any outer members */ public boolean isEmpty() { return this.outerToInners.isEmpty(); } /** * @return True if this {@link MultiPolygon} is valid according to the OGC SFS specification. * See {@code org.locationtech.jts.geom.Geometry.isValid()} */ public boolean isOGCValid() { return new JtsMultiPolygonToMultiPolygonConverter().backwardConvert(this).isValid(); } /** * @return True if this {@link MultiPolygon} is valid according to the OSM specification. OSM * allows some inners of the MultiPolygon to touch on more than a single point, to allow * for one inner to be split in multiple parts tagged differently. Example: a forest * with an inner, that is one side a meadow, and on the other side some marshland. */ public boolean isOSMValid() { final org.locationtech.jts.geom.MultiPolygon jtsMultiPolygon = new JtsMultiPolygonToMultiPolygonConverter() .backwardConvert(this); final TopologyValidationError topologyValidationError = new IsValidOp(jtsMultiPolygon) .getValidationError(); if (topologyValidationError != null) { // In this case, the geometry is not OGC valid, here we capture the // TopologyValidationError to know what to do next. if (TopologyValidationError.SELF_INTERSECTION == topologyValidationError.getErrorType()) { final Location errorLocation = new JtsLocationConverter() .backwardConvert(topologyValidationError.getCoordinate()); final Rectangle errorExpandedBoundingBox = errorLocation .boxAround(Distance.ONE_METER); return isOSMValidSelfIntersection(errorExpandedBoundingBox); } return false; } else { return true; } } public boolean isSimilarTo(final MultiPolygon other) { final double similarity = new HausdorffSimilarityMeasure().measure( JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this), JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(other)); return similarity > SIMILARITY_THRESHOLD; } /** * @return whether this Multipolygon can be represented as a {@link Polygon} */ public boolean isSimplePolygon() { return this.outers().size() == 1; } @Override public Iterator iterator() { return new MultiIterable<>(outers(), inners()).iterator(); } /** * Merge multiple {@link MultiPolygon}s into one. If the two {@link MultiPolygon}s happen to * have the same outer polygon, then the two's inner polygons will be added to the same list. * * @param other * The other {@link MultiPolygon} to merge. * @return The concatenated {@link MultiPolygon} */ public MultiPolygon merge(final MultiPolygon other) { final MultiMap result = new MultiMap<>(); result.putAll(getOuterToInners()); result.addAll(other.getOuterToInners()); return new MultiPolygon(result); } public Set outers() { return this.outerToInners.keySet(); } @Override public boolean overlaps(final MultiPolygon otherMultiPolygon) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } return this.prepared.intersects( JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(otherMultiPolygon)); } @Override public boolean overlaps(final PolyLine polyLine) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory .prepare(JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(this)); } if (polyLine instanceof Polygon) { return this.prepared.intersects(JTS_POLYGON_CONVERTER.convert((Polygon) polyLine)); } return this.prepared.intersects(JTS_POLYLINE_CONVERTER.convert(polyLine)); } public void saveAsGeoJson(final WritableResource resource) { final JsonWriter writer = new JsonWriter(resource); writer.write(asGeoJson()); writer.close(); } @Override public Surface surface() { Surface result = Surface.MINIMUM; for (final Polygon outer : this.outers()) { result = result.add(outer.surface()); } for (final Polygon inner : this.inners()) { result = result.subtract(inner.surface()); } return result; } @Override public Surface surfaceOnSphere() { Surface result = Surface.MINIMUM; for (final Polygon outer : this.outers()) { result = result.add(outer.surfaceOnSphere()); } for (final Polygon inner : this.inners()) { result = result.subtract(inner.surfaceOnSphere()); } return result; } public String toCompactString() { return new MultiPolygonStringConverter().backwardConvert(this); } public String toReadableString() { final String separator1 = "\n\t"; final String separator2 = "\n\t\t"; final StringBuilder builder = new StringBuilder(); final StringList outers = new StringList(); for (final Polygon outer : this.outers()) { final StringList inners = new StringList(); for (final Polygon inner : innersOf(outer)) { inners.add("Inner: " + inner.toCompactString()); } outers.add("Outer: " + outer.toCompactString() + separator2 + inners.join(separator2)); } builder.append(outers.join(separator1)); return builder.toString(); } public String toSimpleString() { final String string = toCompactString(); if (string.length() > SIMPLE_STRING_LENGTH + 1) { return string.substring(0, SIMPLE_STRING_LENGTH) + "..."; } return string; } @Override public String toString() { return toWkt(); } @Override public byte[] toWkb() { return new WkbMultiPolygonConverter().convert(this); } @Override public String toWkt() { return new WktMultiPolygonConverter().convert(this); } private boolean isLinear(final GeometricObject geometricObject) { return geometricObject instanceof PolyLine && !(geometricObject instanceof Polygon) || geometricObject instanceof MultiPolyLine || geometricObject instanceof Location; } /** * Given an OGC intersection location, check all the touching features to see if they are OSM * valid. * * @param errorExpandedBoundingBox * Small bounding box around the error location * @return True if the error is not an error according to OSM */ private boolean isOSMValidSelfIntersection(final Rectangle errorExpandedBoundingBox) { final List> ringsOfInterest = Iterables .stream(new MultiIterable<>( outers().stream().map(outer -> new Tuple<>(true, outer)) .collect(Collectors.toList()), inners().stream().map(inner -> new Tuple<>(false, inner)) .collect(Collectors.toList()))) .filter(ringOfInterest -> ringOfInterest.getSecond() .intersects(errorExpandedBoundingBox)) .collectToList(); final List intersections = new ArrayList<>(); for (int i = 0; i < ringsOfInterest.size(); i++) { for (int j = i + 1; j < ringsOfInterest.size(); j++) { // Make sure this is just a PolyLine final List> candidates = new ArrayList<>(); candidates.add(ringsOfInterest.get(i)); candidates.add(ringsOfInterest.get(j)); if (candidates.get(0).getFirst() || candidates.get(1).getFirst()) { // There is a self intersection between at least one outer, this is not // OSM valid return false; } GeometryOperation.intersection( candidates.stream().map(Tuple::getSecond).collect(Collectors.toList())) .ifPresent(intersections::add); } } boolean allIntersectionsArePolyLines = true; for (final GeometricObject intersection : intersections) { if (!isLinear(intersection)) { allIntersectionsArePolyLines = false; break; } } return ringsOfInterest.size() > 1 && allIntersectionsArePolyLines; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/PolyLine.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.locationtech.jts.geom.prep.PreparedGeometry; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Snapper.SnappedLocation; import org.openstreetmap.atlas.geography.clipping.Clip; import org.openstreetmap.atlas.geography.clipping.Clip.ClipType; import org.openstreetmap.atlas.geography.converters.WkbLocationConverter; import org.openstreetmap.atlas.geography.converters.WkbPolyLineConverter; import org.openstreetmap.atlas.geography.converters.WktLocationConverter; import org.openstreetmap.atlas.geography.converters.WktPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonGeometry; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.geography.matching.PolyLineMatch; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Angle; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Ratio; import org.openstreetmap.atlas.utilities.tuples.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * A PolyLine is a set of {@link Location}s in a specific order * * @author matthieun * @author mgostintsev * @author Sid */ public class PolyLine implements Collection, Located, Serializable, GeometryPrintable, GeoJsonGeometry, GeometricObject { public static final PolyLine TEST_POLYLINE = new PolyLine(Location.TEST_3, Location.TEST_7, Location.TEST_4, Location.TEST_1, Location.TEST_5); public static final PolyLine TEST_POLYLINE_2 = new PolyLine(Location.TEST_1, Location.TEST_5, Location.TEST_4, Location.TEST_3, Location.TEST_7); public static final PolyLine CENTER = new PolyLine(Location.CENTER); public static final PolyLine SIMPLE_POLYLINE = new PolyLine(Location.forString("1,1"), Location.forString("2,2")); public static final String SEPARATOR = ":"; protected static final int SIMPLE_STRING_LENGTH = 200; private static final JtsMultiPolygonToMultiPolygonConverter JTS_MULTIPOLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final JtsPolygonConverter JTS_POLYGON_CONVERTER = new JtsPolygonConverter(); private static final JtsPointConverter JTS_POINT_CONVERTER = new JtsPointConverter(); private static final JtsPolyLineConverter JTS_POLYLINE_CONVERTER = new JtsPolyLineConverter(); private static final long serialVersionUID = -3291779878869865427L; private static final Logger logger = LoggerFactory.getLogger(PolyLine.class); private static final String IMMUTABLE_POLYLINE = "A polyline is immutable"; private final List points; private transient PreparedGeometry prepared; public static GeoJsonObject asGeoJson(final Iterable> geometries) { return new GeoJsonBuilder().create(Iterables.translate(geometries, geometry -> new LocationIterableProperties(geometry, new HashMap<>()))); } /** * Generate a random {@link PolyLine} within bounds. * * @param numberPoints * The number of points in the {@link PolyLine} * @param bounds * The bounds for the points to be in * @return The random {@link PolyLine} */ public static PolyLine random(final int numberPoints, final Rectangle bounds) { final List locations = new ArrayList<>(); IntStream.range(0, numberPoints).forEach(index -> locations.add(Location.random(bounds))); return new PolyLine(locations); } public static void saveAsGeoJson(final Iterable> geometries, final WritableResource resource) { try (JsonWriter writer = new JsonWriter(resource)) { writer.write(asGeoJson(geometries).jsonObject()); } } /** * Create a {@link PolyLine} from Well Known Binary * * @param wkb * The Well Known Binary * @return The {@link PolyLine} */ public static PolyLine wkb(final byte[] wkb) { return new WkbPolyLineConverter().backwardConvert(wkb); } /** * Create a {@link PolyLine} from Well Known Text * * @param wkt * The Well Known Text * @return The {@link PolyLine} */ public static PolyLine wkt(final String wkt) { return new WktPolyLineConverter().backwardConvert(wkt); } public PolyLine(final Iterable points) { this(Iterables.asList(points)); } public PolyLine(final List points) { if (points.isEmpty()) { throw new CoreException("Cannot have an empty PolyLine or Polygon."); } this.points = new ArrayList<>(points); } public PolyLine(final Location... points) { this(Iterables.iterable(points)); } @Override public boolean add(final Location e) { throw new IllegalAccessError("Cannot add a Location to a PolyLine."); } @Override public boolean addAll(final Collection collection) { throw new IllegalAccessError("Cannot add Locations to a PolyLine."); } /** * Return a {@link List} of {@link Tuple} that contains the Angle {@link Angle} and * {@link Location} of all {@link Angle}s that are greater than or equal to the target * {@link Angle}. * * @param target * The threshold {@link Angle} used for comparison. * @return The {@link List} of {@link Tuple} that contains the {@link Angle} and * {@link Location} of all results */ public List> anglesGreaterThanOrEqualTo(final Angle target) { final List> result = new ArrayList<>(); final List segments = segments(); if (segments.isEmpty() || segments.size() == 1) { return result; } for (int i = 1; i < segments.size(); i++) { final Segment first = segments.get(i - 1); final Segment second = segments.get(i); final Optional firstHeading = first.heading(); final Optional secondHeading = second.heading(); if (firstHeading.isPresent() && secondHeading.isPresent()) { final Angle candidate = firstHeading.get().difference(secondHeading.get()); if (candidate.isGreaterThanOrEqualTo(target)) { final Tuple tuple = Tuple.createTuple(candidate, first.end()); result.add(tuple); } } } return result; } /** * Return a {@link List} of {@link Tuple} that contains the {@link Angle} and {@link Location} * of all {@link Angle}s that are less than or equal to the target {@link Angle}. * * @param target * The threshold {@link Angle} used for comparison. * @return The {@link List} of {@link Tuple} that contains the {@link Angle} and * {@link Location} of all results */ public List> anglesLessThanOrEqualTo(final Angle target) { final List> result = new ArrayList<>(); final List segments = segments(); if (segments.isEmpty() || segments.size() == 1) { return result; } for (int i = 1; i < segments.size(); i++) { final Segment first = segments.get(i - 1); final Segment second = segments.get(i); final Optional firstHeading = first.heading(); final Optional secondHeading = second.heading(); if (firstHeading.isPresent() && secondHeading.isPresent()) { final Angle candidate = firstHeading.get().difference(secondHeading.get()); if (candidate.isLessThanOrEqualTo(target)) { final Tuple tuple = Tuple.createTuple(candidate, first.end()); result.add(tuple); } } } return result; } /** * Append the given {@link PolyLine} to this one, if possible. * * @param other * The {@link PolyLine} to append * @return the new, combined {@link PolyLine} */ public PolyLine append(final PolyLine other) { if (this.last().equals(other.first())) { return new PolyLine(new MultiIterable<>(this, other.truncate(1, 0))); } else { throw new CoreException( "Cannot append {} to {} - the end and start points do not match.", other.toWkt(), this.toWkt()); } } @Override public JsonObject asGeoJsonGeometry() { return GeoJsonUtils.geometry(GeoJsonType.LINESTRING, GeoJsonUtils.locationsToCoordinates(this.points)); } /** * Return the average distance from this {@link PolyLine}'s shape points to the other shape, and * the other shape's shape points to this polyline. * * @param other * The other shape to compare to * @return The two way cost distance to the other {@link PolyLine} */ public Distance averageDistanceTo(final PolyLine other) { return averageOneWayDistanceTo(other).add(other.averageOneWayDistanceTo(this)) .scaleBy(Ratio.HALF); } /** * Return the average distance from this {@link PolyLine}'s shape points to the other shape, * using a one-way snapping. * * @param other * The other shape to compare to * @return The one way cost distance to the other {@link PolyLine} */ public Distance averageOneWayDistanceTo(final PolyLine other) { Distance costDistance = Distance.ZERO; for (final Location shapePoint : this) { costDistance = costDistance.add(shapePoint.snapTo(other).getDistance()); } return costDistance.scaleBy(1.0 / this.size()); } /** * Return a sub-{@link PolyLine} of this {@link PolyLine} * * @param start * The start location to include * @param startOccurrence * The occurrence index starting from 0 for the end location, in case of self * intersecting or ring polylines. * @param end * The end location to include * @param endOccurrence * The occurrence index starting from 0 for the end location, in case of self * intersecting or ring polylines. * @return The sub-{@link PolyLine} including start and end */ public PolyLine between(final Location start, final int startOccurrence, final Location end, final int endOccurrence) { final List result = new ArrayList<>(); boolean started = false; int startIndex = 0; int endIndex = 0; for (final Location location : this) { if (location.equals(start) && startOccurrence == startIndex++) { started = true; } if (location.equals(end) && endOccurrence == endIndex++) { if (!started) { throw new CoreException( "Found end first! {}(occurrence {}) and {}(occurrence {}) are not in order with respect to {}", start, startOccurrence, end, endOccurrence, this.toWkt()); } started = false; result.add(location); // Break here to avoid confusion with self-intersecting polylines. break; } if (started) { result.add(location); } } if (started) { throw new CoreException("(Start was {}) End {} is not in polyLine {}", start, end, this); } return new PolyLine(result); } @Override public Rectangle bounds() { return Rectangle.forLocations(this); } @Override public void clear() { throw new IllegalAccessError(IMMUTABLE_POLYLINE); } /** * Clip this feature on a {@link MultiPolygon} * * @param clipping * The {@link MultiPolygon} to clip to * @param clipType * The clip type (AND, OR, XOR or NOT). * @return The clip object containing the clipped features. */ public Clip clip(final MultiPolygon clipping, final ClipType clipType) { return new Clip(clipType, this, clipping); } /** * Clip this feature on a {@link Polygon} * * @param clipping * The {@link Polygon} to clip to * @param clipType * The clip type (AND, OR, XOR or NOT). * @return The clip object containing the clipped features. */ public Clip clip(final Polygon clipping, final ClipType clipType) { return new Clip(clipType, this, clipping); } /** * @param location * The {@link Location} to test * @return True if one of the vertices of this {@link PolyLine} is the provided {@link Location} */ public boolean contains(final Location location) { for (final Location thisLocation : this) { if (thisLocation.equals(location)) { return true; } } return false; } @Override public final boolean contains(final Object object) { if (object instanceof Location) { return contains((Location) object); } if (object instanceof Segment) { return contains((Segment) object); } throw new IllegalAccessError( "A polyline can contain a Segment or Location only. Maybe you meant \"covers\"?"); } /** * @param segment * The {@link Segment} to test * @return True if one of the segments of this {@link PolyLine} is the provided {@link Segment} */ public boolean contains(final Segment segment) { final List segments = this.segments(); for (final Segment thisSegment : segments) { if (thisSegment.equals(segment)) { return true; } } return false; } @Override public boolean containsAll(final Collection collection) { throw new IllegalAccessError(); } /** * Get the cost {@link Distance} to the set of {@link Segment}s that are connected and that * provide a new {@link PolyLine} that is the closest in shape to this {@link PolyLine} * * @param candidates * The candidate {@link PolyLine}s to match to this {@link PolyLine} * @return The best reconstructed match from this PolyLine. */ public PolyLineMatch costDistanceToOneWay(final Iterable candidates) { return new PolyLineMatch(this, Iterables.asList(candidates)); } @Override public boolean equals(final Object other) { if (other instanceof PolyLine) { final PolyLine that = (PolyLine) other; return Iterables.equals(this, that); } return false; } /** * Tests if this {@link PolyLine} has the same shape as another {@link PolyLine}. This is * different from equals as some {@link PolyLine}s that are different can still have the same * shape, by being reversed or by self intersecting in the same point for example. * * @param other * The other {@link PolyLine} to compare to * @return True if they both have the same shape */ public boolean equalsShape(final PolyLine other) { return this.overlapsShapeOf(other) && other.overlapsShapeOf(this); } /** * @return the final {@link Heading} for this {@link PolyLine}, based on the {@link Heading} of * the last {@link Segment}. */ public Optional finalHeading() { final List segments = this.segments(); return segments.size() > 0 ? segments.get(segments.size() - 1).heading() : Optional.empty(); } /** * @return The first {@link Location} of this {@link PolyLine} */ public Location first() { return size() > 0 ? get(0) : null; } /** * @param index * The index to query * @return The {@link Location} at the index provided in this {@link PolyLine} */ public Location get(final int index) { if (index < 0 || index >= size()) { throw new CoreException("Cannot get a Location with index " + index + ", which is not between 0 and " + size()); } return this.points.get(index); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.LINESTRING; } @Override public int hashCode() { int result = 0; for (final Location location : this) { result += location.hashCode(); } return result; } /** * @return The difference, if available, between the last {@link Segment}'s {@link Heading} and * the first {@link Segment}'s {@link Heading} */ public Optional headingDifference() { if (this.size() <= 1) { return Optional.empty(); } if (this.size() == 2) { return Optional.of(Angle.NONE); } else { final List segments = this.segments(); final Segment first = segments.get(0); final Segment last = segments.get(segments.size() - 1); final Optional heading1 = first.heading(); if (!heading1.isPresent()) { return Optional.empty(); } final Optional heading2 = last.heading(); if (!heading2.isPresent()) { return Optional.empty(); } return Optional.of(heading2.get().subtract(heading1.get())); } } /** * @return the initial {@link Heading} for this {@link PolyLine}, based on the {@link Heading} * of the first {@link Segment}. */ public Optional initialHeading() { final List segments = this.segments(); return segments.size() > 0 ? segments.get(0).heading() : Optional.empty(); } /** * @return All the locations in this {@link PolyLine} except the first and last. */ public Iterable innerLocations() { return this.truncate(1, 1); } public Set intersections(final PolyLine candidate) { final Set result = new HashSet<>(); if (this instanceof Segment) { result.addAll(candidate.intersections((Segment) this)); } else { final List segments = this.segments(); segments.forEach(segment -> { final Set intersections = segment.intersections(candidate); result.addAll(intersections); }); } return result; } public Set intersections(final Segment candidate) { final Set result = new HashSet<>(); final List segments = this.segments(); segments.forEach(segment -> { final Location intersection = segment.intersection(candidate); if (intersection != null) { result.add(intersection); } }); return result; } /** * Test if two {@link PolyLine}s intersect. * * @param other * The other {@link PolyLine} * @return True if this {@link PolyLine} intersects the other at least once. */ @Override public boolean intersects(final PolyLine other) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYLINE_CONVERTER.convert(this)); } return this.prepared.intersects(JTS_POLYLINE_CONVERTER.convert(other)); } public boolean isClosed() { return JTS_POLYLINE_CONVERTER.convert(this).isClosed(); } @Override public final boolean isEmpty() { return this.points.isEmpty(); } /** * @return True if this {@link PolyLine} is a single point, i.e. all the points are the same. */ public boolean isPoint() { Location firstPoint = null; for (final Location point : this.points) { if (firstPoint == null) { firstPoint = point; continue; } if (!point.equals(firstPoint)) { return false; } } return true; } public boolean isSimple() { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYLINE_CONVERTER.convert(this)); } return this.prepared.getGeometry().isSimple(); } @Override public Iterator iterator() { return this.points.iterator(); } public Location last() { return this.points.size() > 0 ? get(size() - 1) : null; } public Distance length() { Distance result = Distance.ZERO; final List segments = this.segments(); for (final Segment segment : segments) { result = result.add(segment.length()); } return result; } /** * @return The biggest Angle in this {@link PolyLine} */ public Angle maximumAngle() { final List segments = segments(); if (segments.isEmpty()) { return null; } if (segments.size() == 1) { return Angle.NONE; } Angle maximum = Angle.NONE; for (int i = 1; i < segments.size(); i++) { final Segment first = segments.get(i - 1); final Segment second = segments.get(i); final Optional firstHeading = first.heading(); final Optional secondHeading = second.heading(); if (firstHeading.isPresent() && secondHeading.isPresent()) { final Angle candidate = firstHeading.get().difference(secondHeading.get()); if (candidate.isGreaterThan(maximum)) { maximum = candidate; } } } return maximum; } /** * @return The location of the biggest Angle in this {@link PolyLine} */ public Optional maximumAngleLocation() { final List segments = segments(); if (segments.isEmpty() || segments.size() == 1) { return Optional.empty(); } Angle maximum = Angle.NONE; Location maximumAngleLocation = null; for (int i = 1; i < segments.size(); i++) { final Segment first = segments.get(i - 1); final Segment second = segments.get(i); final Optional firstHeading = first.heading(); final Optional secondHeading = second.heading(); if (firstHeading.isPresent() && secondHeading.isPresent()) { final Angle candidate = firstHeading.get().difference(secondHeading.get()); if (candidate.isGreaterThan(maximum) || maximumAngleLocation == null) { maximum = candidate; maximumAngleLocation = first.end(); } } } return Optional.ofNullable(maximumAngleLocation); } public Location middle() { return offsetFromStart(Ratio.HALF); } /** * Get the number of times a location appears in this {@link PolyLine}. Most useful for self * intersecting or ring {@link PolyLine}s. * * @param node * The location to test * @return The number of occurrences in this {@link PolyLine}. 0 if it never shows up. */ public int occurrences(final Location node) { int result = 0; for (final Location location : this) { if (location.equals(node)) { result++; } } return result; } /** * Get the offset from the start of the node's location * * @param node * The location to test * @param occurrenceIndex * In case of a self intersecting polyline (one or more locations appear more than * once), indicate the index at which this method should return the location. 0 would * be first occurrence, 1 second, etc. * @return The offset ratio from the start of the {@link PolyLine} */ public Ratio offsetFromStart(final Location node, final int occurrenceIndex) { final Distance max = this.length(); Distance candidate = Distance.ZERO; Location previous = this.first(); int index = 0; for (final Location location : this) { candidate = candidate.add(previous.distanceTo(location)); if (location.equals(node) && occurrenceIndex == index++) { return Ratio.ratio(candidate.asMeters() / max.asMeters()); } previous = location; } throw new CoreException("The location {} is not a node of the PolyLine", node); } public Location offsetFromStart(final Ratio ratio) { final Distance length = length(); final Distance stop = length.scaleBy(ratio); Distance accumulated = Distance.ZERO; final List segments = this.segments(); for (final Segment segment : segments) { if (accumulated.add(segment.length()).isGreaterThan(stop)) { // This is the proper segment final Ratio segmentRatio = Ratio.ratio( stop.substract(accumulated).asMeters() / segment.length().asMeters()); return segment.offsetFromStart(segmentRatio); } if (accumulated.add(segment.length()).equals(stop)) { return segment.end(); } accumulated = accumulated.add(segment.length()); } throw new CoreException("This exception should never be thrown."); } /** * @return The overall heading of the {@link PolyLine}: the heading between the start point and * the end point. */ public Optional overallHeading() { if (this.isPoint()) { logger.warn("Cannot compute a PolyLine's heading when it has zero length : {}", this); return Optional.empty(); } if (this.first().equals(this.last())) { if (logger.isWarnEnabled()) { logger.warn("Cannot compute overall heading when the polyline has " + "same start and end locations : {}", this.first().toWkt()); } return Optional.empty(); } return Optional.ofNullable(this.first().headingTo(this.last())); } /** * Tests if this {@link PolyLine} has at least the same shape as another {@link PolyLine}. If * this {@link PolyLine} is made up of {@link Segment}s ABC and the given {@link PolyLine} is * made up of BC, this would return true, despite the excess {@link Segment}. * * @param other * The other {@link PolyLine} to compare to * @return True if this {@link PolyLine} has at least the same shape as the other (but possibly * more) */ public boolean overlapsShapeOf(final PolyLine other) { final Set thisSegments = new HashSet<>(); final List segments = this.segments(); segments.forEach(segment -> { thisSegments.add(segment); thisSegments.add(segment.reversed()); }); final List otherSegments = other.segments(); for (final Segment otherSegment : otherSegments) { if (!thisSegments.contains(otherSegment)) { return false; } } return true; } /** * Prepends the given {@link PolyLine} to this one, if possible. * * @param other * The {@link PolyLine} to prepend * @return the new, combined {@link PolyLine} */ public PolyLine prepend(final PolyLine other) { if (this.first().equals(other.last())) { return new PolyLine(new MultiIterable<>(other, this.truncate(1, 0))); } else { throw new CoreException( "Cannot prepend {} to {} - the end and start points do not match.", other.toWkt(), this.toWkt()); } } @Override public boolean remove(final Object object) { throw new IllegalAccessError(IMMUTABLE_POLYLINE); } @Override public boolean removeAll(final Collection collection) { throw new IllegalAccessError(IMMUTABLE_POLYLINE); } @Override public boolean retainAll(final Collection collection) { throw new IllegalAccessError(IMMUTABLE_POLYLINE); } public PolyLine reversed() { final List reversed = new ArrayList<>(); for (int i = this.size() - 1; i >= 0; i--) { reversed.add(this.get(i)); } return new PolyLine(reversed); } public void saveAsGeoJson(final WritableResource resource) { final List> geometries = new ArrayList<>(); geometries.add(this); saveAsGeoJson(geometries, resource); } /** * @return All the {@link Segment}s that represent this {@link PolyLine}. If the * {@link PolyLine} is empty, then the {@link Segment} list is empty. If the * {@link PolyLine} has only one item, the {@link Segment} list contains only one * {@link Segment} made of twice the same {@link Location}. If the {@link PolyLine} has * more than one {@link Location}, then the result is a list of {@link Segment}s. Note: * This method should be used carefully. Each call to it will cause a rebuild of the * {@link List}, which can be very inefficient for long {@link PolyLine}s. To avoid * this, the caller can call segments once and cache the results. */ public List segments() { final List result = new ArrayList<>(this.size()); if (size() == 1) { result.add(new Segment(get(0), get(0))); } else if (this instanceof Segment) { result.add((Segment) this); } else { Location previous = null; for (final Location location : this) { if (previous == null) { previous = location; continue; } result.add(new Segment(previous, location)); previous = location; } } return result; } /** * Returns a Set of {@link Location} for all self-intersections, other than shape points for * this {@link PolyLine}. Separated from selfIntersects() to avoid degrading its performance. * * @return the set of locations */ public Set selfIntersections() { Set intersections = null; final boolean isPolygon = this instanceof Polygon; if (this.isSimple() && !isPolygon) { if (this.isClosed()) { intersections = new HashSet<>(); intersections.add(this.first()); return intersections; } return Collections.emptySet(); } // Exclude point-segments, so we know which segments are actually consecutive final List segments = this.segments().stream() .filter(segment -> !segment.isPoint()).collect(Collectors.toList()); // Consecutive segments should not be considered (they always have common point) for (int i = 0; i < segments.size() - 2; i++) { // For Polygons the last segment is consecutive to the first final int limit = isPolygon && i == 0 ? segments.size() - 1 : segments.size(); // Only consider 'higher' segments. // No need to do a reverse check with the 'lower' segments again. for (int j = i + 2; j < limit; j++) { final Location intersection = segments.get(i).intersection(segments.get(j)); if (intersection != null) { // Self-intersection is a low probability event. // Only allocate if needed if (intersections == null) { intersections = new HashSet<>(); } intersections.add(intersection); } } } return intersections == null ? Collections.emptySet() : intersections; } /** * @return True if the {@link PolyLine} self intersects at locations other than shape points. */ public boolean selfIntersects() { // See comments on algorithm in selfIntersections() final boolean isPolygon = this instanceof Polygon; final List segments = this.segments().stream() .filter(segment -> !segment.isPoint()).collect(Collectors.toList()); for (int i = 0; i < segments.size() - 2; i++) { final int limit = isPolygon && i == 0 ? segments.size() - 1 : segments.size(); for (int j = i + 2; j < limit; j++) { if (segments.get(i).intersection(segments.get(j)) != null) { return true; } } } return false; } public PolyLine shiftFirstAlongGreatCircle(final Heading initialHeading, final Distance distance) { return new PolyLine(new MultiIterable<>( Iterables.from(this.first().shiftAlongGreatCircle(initialHeading, distance)), this.truncate(1, 0))); } public PolyLine shiftLastAlongGreatCircle(final Heading initialHeading, final Distance distance) { return new PolyLine(new MultiIterable<>(this.truncate(0, 1), Iterables.from(this.last().shiftAlongGreatCircle(initialHeading, distance)))); } /** * Return the smaller one between the shortest distance from this {@link PolyLine}'s shape * points to the other shape, and the other shape's shape points to this polyline. * * @param other * The other shape to compare to * @return The two way shortest distance to the other {@link PolyLine} */ public Distance shortestDistanceTo(final PolyLine other) { final Distance one = shortestOneWayDistanceTo(other); final Distance two = other.shortestOneWayDistanceTo(this); return one.isLessThan(two) ? one : two; } /** * Return the shortest distance from this {@link PolyLine}'s shape points to the other shape, * using a one-way snapping. * * @param other * The other shape to compare to * @return The shortest one way cost distance to the other {@link PolyLine} */ public Distance shortestOneWayDistanceTo(final PolyLine other) { Distance shortest = Distance.MAXIMUM; for (final Location shapePoint : this) { final Distance current = shapePoint.snapTo(other).getDistance(); shortest = current.isLessThan(shortest) ? current : shortest; } return shortest; } @Override public int size() { return this.points.size(); } /** * Snap an origin {@link Location} to this {@link PolyLine} using a {@link Snapper} * * @param origin * The origin {@link Location} to snap * @return The corresponding {@link SnappedLocation} */ public SnappedLocation snapFrom(final Location origin) { return new Snapper().snap(origin, this); } @Override public Object[] toArray() { return this.points.toArray(); } @Override public T[] toArray(final T[] array) { return this.points.toArray(array); } public String toCompactString() { final StringList stringList = new StringList(); this.forEach(location -> { stringList.add(location.toCompactString()); }); return stringList.join(SEPARATOR); } public String toSimpleString() { final String string = toCompactString(); if (string.length() > SIMPLE_STRING_LENGTH + 1) { return string.substring(0, SIMPLE_STRING_LENGTH / 2) + "..." + string.substring(string.length() - SIMPLE_STRING_LENGTH / 2); } return string; } @Override public String toString() { return toWkt(); } /** * @return This {@link PolyLine} as Well Known Binary */ @Override public byte[] toWkb() { if (this.size() == 1) { // Handle a single location polyLine return new WkbLocationConverter().convert(this.first()); } return new WkbPolyLineConverter().convert(this); } /** * @return This {@link PolyLine} as Well Known Text */ @Override public String toWkt() { if (this.size() == 1) { // Handle a single location polyLine return new WktLocationConverter().convert(this.first()); } return new WktPolyLineConverter().convert(this); } /** * Truncates this {@link PolyLine} at the given start and end index. If the provided indices are * invalid, an empty Iterable will be returned. * * @param indexFromStart * The index before which to truncate from the start * @param indexFromEnd * The index after which to truncate from the end * @return all the locations in this {@link PolyLine} after truncation. */ public Iterable truncate(final int indexFromStart, final int indexFromEnd) { if (indexFromStart < 0 || indexFromEnd < 0 || indexFromStart >= this.size() || indexFromEnd >= this.size() || indexFromStart + indexFromEnd >= this.size()) { logger.debug("Invalid start index {} or end index {} supplied.", indexFromStart, indexFromEnd); return Collections.emptyList(); } return Iterables.stream(this).truncate(indexFromStart, indexFromEnd); } @Override public boolean within(final GeometricSurface surface) { return surface.fullyGeometricallyEncloses(this); } /** * @return This {@link PolyLine} without duplicate consecutive shape points. Non-consecutive * shape points will remain unchanged. */ public PolyLine withoutDuplicateConsecutiveShapePoints() { final List shapePoints = new ArrayList<>(); boolean hasDuplicates = false; final Iterator locationIterator = this.iterator(); // PolyLines are only valid if at least one point exists, so it is safe to call next() once. Location previousLocation = locationIterator.next(); shapePoints.add(previousLocation); while (locationIterator.hasNext()) { final Location currentLocation = locationIterator.next(); if (!currentLocation.equals(previousLocation)) { shapePoints.add(currentLocation); } else { hasDuplicates = true; } previousLocation = currentLocation; } return hasDuplicates ? new PolyLine(shapePoints) : this; } protected final List getPoints() { return this.points; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Polygon.java ================================================ package org.openstreetmap.atlas.geography; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; import java.util.stream.IntStream; import org.locationtech.jts.algorithm.match.HausdorffSimilarityMeasure; import org.locationtech.jts.geom.GeometryCollection; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.prep.PreparedGeometry; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.triangulate.ConformingDelaunayTriangulationBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.converters.WkbPolygonConverter; import org.openstreetmap.atlas.geography.converters.WktPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.GeometryStreamer; import org.openstreetmap.atlas.geography.converters.jts.JtsLocationConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPrecisionManager; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.scalars.Angle; import org.openstreetmap.atlas.utilities.scalars.Surface; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.gson.JsonObject; /** * A {@link Polygon} is a {@link PolyLine} with an extra {@link Segment} between the last * {@link Location} and the first {@link Location}. * * @author matthieun */ public class Polygon extends PolyLine implements GeometricSurface { public static final Polygon SILICON_VALLEY = new Polygon(Location.TEST_3, Location.TEST_7, Location.TEST_4, Location.TEST_1, Location.TEST_5); public static final Polygon SILICON_VALLEY_2 = new Polygon(Location.TEST_3, Location.TEST_7, Location.TEST_2, Location.TEST_1, Location.TEST_5); public static final Polygon TEST_BUILDING = new Polygon( Location.forString("37.3909505256542,-122.03104734420775"), Location.forString("37.39031973417266,-122.03141212463377"), Location.forString("37.390106627742895,-122.03113317489623"), Location.forString("37.39084823550426,-122.03062891960144"), Location.forString("37.3909505256542,-122.03104734420775")); public static final Polygon TEST_BUILDING_PART = new Polygon( Location.forString("37.390234491673446,-122.03111171722412"), Location.forString("37.39020252571126,-122.0311439037323"), Location.forString("37.39018121506223,-122.03110367059708"), Location.forString("37.39021104996917,-122.0310714840889"), Location.forString("37.390234491673446,-122.03111171722412")); public static final Polygon CENTER = new Polygon(Location.CENTER); private static final JtsMultiPolygonToMultiPolygonConverter JTS_MULTIPOLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final JtsPolygonConverter JTS_POLYGON_CONVERTER = new JtsPolygonConverter(); private static final JtsPointConverter JTS_POINT_CONVERTER = new JtsPointConverter(); private static final JtsPolyLineConverter JTS_POLYLINE_CONVERTER = new JtsPolyLineConverter(); private static final Logger logger = LoggerFactory.getLogger(Polygon.class); private static final long serialVersionUID = 2877026648358594354L; // Calculate sides starting from triangles private static final int MINIMUM_N_FOR_SIDE_CALCULATION = 3; private transient PreparedGeometry prepared; /** * Generate a random polygon within bounds. * * @param numberPoints * The number of points in the polygon * @param bounds * The bounds for the points to be in * @return The random {@link Polygon} */ public static Polygon random(final int numberPoints, final Rectangle bounds) { final List locations = new ArrayList<>(); IntStream.range(0, numberPoints).forEach(index -> locations.add(Location.random(bounds))); return new Polygon(locations); } /** * Generate a Polygon from Well Known Text * * @param wkt * The polygon in well known text * @return The parsed {@link Polygon} */ public static Polygon wkt(final String wkt) { return new WktPolygonConverter().backwardConvert(wkt); } public Polygon(final Iterable points) { this(Iterables.asList(points)); } public Polygon(final List points) { super(points); } public Polygon(final Location... points) { // This was Iterables.asList. `super` creates a new ArrayList, so we don't have to worry // about the backing array being modified. // This was 6% of a test run in a single validation (there were other validations run, so // this may be larger). After the new run, it was 3% (async Allocation Profiler) this(Arrays.asList(points)); } @Override public JsonObject asGeoJsonGeometry() { return GeoJsonUtils.geometry(GeoJsonType.POLYGON, GeoJsonUtils.polygonToCoordinates(this)); } /** * The segments that belong to this {@link Polygon} that are attached to this vertex * * @param vertexIndex * the index of the vertex * @return The segments that belong to this {@link Polygon} that are attached to this vertex */ public List attachedSegments(final int vertexIndex) { verifyVertexIndex(vertexIndex); final List result = new ArrayList<>(); // Previous if (vertexIndex > 0) { result.add(segmentForIndex(vertexIndex - 1)); } else { result.add(segmentForIndex(size() - 1)); } // Next result.add(segmentForIndex(vertexIndex)); return result; } /** * This will return the centroid of a given polygon. It can handle complex polygons including * multiple polygons. This will not necessarily return a location that is contained within the * original polygon. For example if you have two concentric circles forming a donut shape, one * smaller one contained within the bigger one. The centroid of that polygon will be at the * center technically outside of the polygon. This is a very different concept to a * representative point. * * @return a Location object that is the centroid of the polygon */ public Location center() { final Point point = JTS_POLYGON_CONVERTER.convert(this).getCentroid(); return new JtsLocationConverter().backwardConvert(point.getCoordinate()); } /** * @return An iterable of {@link Location}s that will return the first item again at the end. */ public Iterable closedLoop() { if (!this.first().equals(this.last())) { return new MultiIterable<>(this, Iterables.from(this.first())); } return this; } /** * Tests if this {@link Polygon} fully encloses (geometrically contains) a {@link Location} *

* Here is the definition of contains (insideness) of awt point. *

* Definition of insideness: A point is considered to lie inside a Shape if and only if: it lies * completely inside the Shape boundary or it lies exactly on the Shape boundary and the space * immediately adjacent to the point in the increasing X direction is entirely inside the * boundary or it lies exactly on a horizontal boundary segment and the space immediately * adjacent to the point in the increasing Y direction is inside the boundary. *

* In the case of a massive polygon (larger than 75% of the earth's width) the JTS definition of * covers is used instead, which will return true if the location lies within the polygon or * anywhere on the boundary. *

* * @param location * The {@link Location} to test * @return True if the {@link Polygon} contains the {@link Location} */ @Override public boolean fullyGeometricallyEncloses(final Location location) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } return this.prepared.covers(JTS_POINT_CONVERTER.convert(location)); } @Override public boolean fullyGeometricallyEncloses(final MultiPolygon multiPolygon) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } return this.prepared.covers(JTS_MULTIPOLYGON_CONVERTER.backwardConvert(multiPolygon)); } /** * Tests if this {@link Polygon} fully encloses (geometrically contains) a {@link PolyLine}. * Note: this will return false for the case when the {@link Polygon} and given {@link PolyLine} * are stacked on top of each other - i.e. have an identical shape as one another. * * @param polyLine * The {@link PolyLine} to test * @return True if this {@link Polygon} wraps (geometrically contains) the provided * {@link PolyLine} */ @Override public boolean fullyGeometricallyEncloses(final PolyLine polyLine) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } if (polyLine instanceof Polygon) { return this.prepared.covers(JTS_POLYGON_CONVERTER.convert((Polygon) polyLine)); } return this.prepared.covers(JTS_POLYLINE_CONVERTER.convert(polyLine)); } /** * Tests if this {@link Polygon} fully encloses (geometrically contains) a {@link Rectangle}. * Note: this will return false for the case when the {@link Polygon} has an identical shape as * the given {@link Rectangle}. * * @param rectangle * The {@link Rectangle} to test * @return True if this {@link Polygon} wraps (geometrically contains) the provided * {@link Rectangle} */ public boolean fullyGeometricallyEncloses(final Rectangle rectangle) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } return this.prepared.covers(JTS_POLYGON_CONVERTER.convert(rectangle)); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.POLYGON; } /** * Returns a location that is the closest point within the polygon to the centroid. The function * delegates to the Geometry class which delegates to the InteriorPointPoint class. You can see * the javadocs in the link below. * * @return location that is the closest point within the polygon to the centroid */ public Location interiorCenter() { final Point point = JTS_POLYGON_CONVERTER.convert(this).getInteriorPoint(); return new JtsLocationConverter().backwardConvert(point.getCoordinate()); } /** * @param expectedNumberOfSides * Expected number of sides * @param threshold * {@link Angle} threshold that decides whether a {@link Heading} difference between * segments should be counted towards heading change count or not * @return true if this {@link Polygon} has approximately n sides while ignoring {@link Heading} * differences between inner segments that are below given threshold. */ public boolean isApproximatelyNSided(final int expectedNumberOfSides, final Angle threshold) { // Ignore if polygon doesn't have enough inner shape points if (expectedNumberOfSides < MINIMUM_N_FOR_SIDE_CALCULATION || this.size() < expectedNumberOfSides) { return false; } // An N sided shape should have (n-1) heading changes final int expectedHeadingChangeCount = expectedNumberOfSides - 1; // Fetch segments and count them final List segments = this.segments(); final int segmentSize = segments.size(); // Index to keep track of segment to work on int segmentIndex = 0; // Keep track of heading changes int headingChangeCount = 0; // Find initial heading Optional previousHeading = Optional.empty(); while (segmentIndex < segmentSize) { // Make sure we start with some heading. Edges with single points do not have heading. previousHeading = segments.get(segmentIndex++).heading(); if (previousHeading.isPresent()) { break; } } // Make sure we start with some heading if (!previousHeading.isPresent()) { logger.trace("{} doesn't have a heading to calculate number of sides.", this); return false; } // Go over rest of the segments and count heading changes while (segmentIndex < segmentSize && headingChangeCount <= expectedHeadingChangeCount) { final Optional nextHeading = segments.get(segmentIndex++).heading(); // If heading difference is greater than threshold, then increment heading // change counter and update previous heading, which is used as reference if (nextHeading.isPresent() && previousHeading.get().difference(nextHeading.get()).isGreaterThan(threshold)) { headingChangeCount++; previousHeading = nextHeading; } } return headingChangeCount == expectedHeadingChangeCount; } /** * @return True if this {@link Polygon} is arranged clockwise, false otherwise. * @see * @see * @see */ public boolean isClockwise() { // Formula to calculate the area of triangle on a sphere is (A + B + C - Pi) * radius * // radius. // Equation (A + B + C - Pi) is called the spherical excess. We are going to divide our // polygon in triangles and then calculate the signed area of each triangle. Sum of the // areas of these triangles will be the area of this polygon double sphericalExcess = 0; Location previousLocation = null; for (final Location point : this.closedLoop()) { final Location currentLocation = point; if (previousLocation != null) { // for the sake of simplicity we are using two vertices from the polygon and the // third vertex would be North Pole. // Please refer "Spherical Trigonometry by I.Todhunter". // Section starting on page 7 and 17 for triangle identities and trigonometric // functions. // Also look on page 71 for getting the area of triangle final double latitudeOne = previousLocation.getLatitude().asRadians(); final double latitudeTwo = currentLocation.getLatitude().asRadians(); final double deltaLongitude = currentLocation.getLongitude().asRadians() - previousLocation.getLongitude().asRadians(); final double alpha = Math .sqrt((1 - Math.sin(latitudeOne)) / (1 + Math.sin(latitudeOne))) * Math.sqrt((1 - Math.sin(latitudeTwo)) / (1 + Math.sin(latitudeTwo))); // You can derive this from the formula on Page 74, point 102 of the book sphericalExcess += 2 * Math.atan2(alpha * Math.sin(deltaLongitude), 1 + alpha * Math.cos(deltaLongitude)); } previousLocation = currentLocation; } // Instead of area of polygon this method returns the spherical access as multiplying with // Earth (radius) ^ 2 is not going to change the sign of the area return sphericalExcess <= 0; } public boolean isSimilarTo(final Polygon other) { final double similarity = new HausdorffSimilarityMeasure() .measure(JTS_POLYGON_CONVERTER.convert(this), JTS_POLYGON_CONVERTER.convert(other)); return similarity > SIMILARITY_THRESHOLD; } public int nextSegmentIndex(final int currentVertexIndex) { verifyVertexIndex(currentVertexIndex); return currentVertexIndex; } public int nextVertexIndex(final int currentVertexIndex) { verifyVertexIndex(currentVertexIndex); if (currentVertexIndex == size() - 1) { return 0; } else { return currentVertexIndex + 1; } } @Override public boolean overlaps(final MultiPolygon multiPolygon) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } return this.prepared.intersects(JTS_MULTIPOLYGON_CONVERTER.backwardConvert(multiPolygon)); } /** * Tests if this {@link Polygon} intersects/overlaps the given {@link PolyLine} at any point. * This is different than the {@link #fullyGeometricallyEncloses(PolyLine)} method, in that it * doesn't require full containment within the {@link Polygon}, just any overlap. * * @param polyline * The {@link PolyLine} to test * @return True if this {@link Polygon} intersects/overlaps the given {@link PolyLine}. */ @Override public boolean overlaps(final PolyLine polyline) { if (this.prepared == null) { this.prepared = PreparedGeometryFactory.prepare(JTS_POLYGON_CONVERTER.convert(this)); } if (polyline instanceof Polygon) { return this.prepared.intersects(JTS_POLYGON_CONVERTER.convert((Polygon) polyline)); } return this.prepared.intersects(JTS_POLYLINE_CONVERTER.convert(polyline)); } public int previousSegmentIndex(final int currentVertexIndex) { return previousVertexIndex(currentVertexIndex); } public int previousVertexIndex(final int currentVertexIndex) { verifyVertexIndex(currentVertexIndex); if (currentVertexIndex == 0) { return size() - 1; } else { return currentVertexIndex - 1; } } @Override public Polygon reversed() { return new Polygon(super.reversed().getPoints()); } public Segment segmentForIndex(final int index) { if (index >= size()) { throw new CoreException("Invalid index"); } return new Segment(this.get(index), index == size() - 1 ? this.get(0) : this.get(index + 1)); } @Override public List segments() { final List result = super.segments(); // close the loop result.add(new Segment(last(), first())); return result; } /** * @return The surface of this polygon. Not valid if the polygon self-intersects, and/or * overlaps itself * @see "http://www.mathopenref.com/coordpolygonarea2.html" */ @Override public Surface surface() { long dm7Squared = 0L; final Iterator loopOnItself = loopOnItself().iterator(); if (!loopOnItself.hasNext()) { return Surface.forDm7Squared(0); } Location current = loopOnItself.next(); Location next = null; while (loopOnItself.hasNext()) { next = loopOnItself.next(); dm7Squared += (current.getLongitude().asDm7() + next.getLongitude().asDm7()) * (current.getLatitude().asDm7() - next.getLatitude().asDm7()); current = next; } return Surface.forDm7Squared(Math.abs(Math.round(dm7Squared / 2.0))); } /** * @return The approximate surface area of this polygon if it were projected onto the Earth. Not * valid if the polygon self-intersects, and/or overlaps itself. Uses "Some Algorithms * for Polygons on a Sphere" paper as reference. * @see "https://trs.jpl.nasa.gov/bitstream/handle/2014/41271/07-0286.pdf" */ @Override public Surface surfaceOnSphere() { double dm7 = 0L; final List locations = Lists.newArrayList(this.closedLoop()); if (locations.size() > 2) { double radians = 0L; for (int index = 0; index < locations.size() - 1; index++) { radians += (locations.get(index + 1).getLongitude().asRadians() - locations.get(index).getLongitude().asRadians()) * (2 + Math.sin(locations.get(index).getLatitude().asRadians()) + Math.sin(locations.get(index + 1).getLatitude().asRadians())); } radians = Math.abs(radians / 2.0); // Calculations are in Radians, convert to Degrees. dm7 = radians * ((double) Angle.DM7_PER_RADIAN * (double) Angle.DM7_PER_RADIAN); } return Surface.forDm7Squared(Math.round(dm7)); } /** * @return This {@link Polygon} as Well Known Binary */ @Override public byte[] toWkb() { return new WkbPolygonConverter().convert(this); } /** * @return This {@link Polygon} as Well Known Text */ @Override public String toWkt() { return new WktPolygonConverter().convert(this); } /** * Triangulate this {@link Polygon}, using the JTS library. * * @return All the triangles that form this {@link Polygon}. */ public List triangles() { final ConformingDelaunayTriangulationBuilder trianguler = new ConformingDelaunayTriangulationBuilder(); // Populate the delaunay triangulation builder trianguler.setSites(JTS_POLYGON_CONVERTER.convert(this)); final GeometryCollection triangleCollection = (GeometryCollection) trianguler .getTriangles(JtsPrecisionManager.getGeometryFactory()); // Get the output and convert back to Core Polygons, filter out the extraneous polygons from // the Delaunay triangulation. return Iterables.stream(GeometryStreamer.streamPolygons(triangleCollection)) .map(JTS_POLYGON_CONVERTER.revert()) .filter(polygon -> fullyGeometricallyEncloses(polygon.center())).collectToList(); } /** * Remove a vertex * * @param index * The index of the vertex to remove * @return The new {@link Polygon} without the specified vertex */ public Polygon withoutVertex(final int index) { if (index < 0 || index >= this.size()) { throw new CoreException("{} is not a vertex index of {}", index, this); } final List vertices = Iterables.asList(this); vertices.remove(index); return new Polygon(vertices); } /** * Remove a vertex * * @param vertex * The vertex to remove * @return The new {@link Polygon} without the specified vertex */ public Polygon withoutVertex(final Location vertex) { int index = 0; for (final Location location : this) { if (location.equals(vertex)) { return withoutVertex(index); } index++; } throw new CoreException("{} is not a vertex of {}", vertex, this); } private Iterable loopOnItself() { return new MultiIterable<>(this, () -> new Iterator() { private boolean read = false; @Override public boolean hasNext() { return !this.read; } @Override public Location next() { if (hasNext()) { this.read = true; return first(); } else { throw new NoSuchElementException(); } } }); } private void verifyVertexIndex(final int index) { if (index < 0 || index >= size()) { throw new CoreException("Invalid Vertex Index {}.", index); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/README.md ================================================ # Geography Package This package contains all the geography tools that make handling geographical data easy. ## Atlas package The [Atlas package](atlas) contains the `Atlas`, an in-memory representation of the OSM data. ## Boundary package All the classes that help with defining administrative boundaries, and associating them with three digit country codes. ## Sharding package Tools that help eavenly divide the world into multiple shards. `DynamicTileSharding` is basically a quad tree. ## Geography Classes ### `Latitude` A `Latitude` is an `Angle` extension which is bound between -90 degrees and 90 degrees. Like `Angle`, it can be created from degrees, radians, or a degree of magnitude 7 (dm7) long. ### `Longitude` A `Longitude` is an `Angle` extension which is bound between -180 degrees and 179.9999999 degrees. Like `Angle`, it can be created from degrees, radians, or a degree of magnitude 7 (dm7) long. Unlike OSM which allows -180 and 180 alike for values that touch the antimeridian, `Longitude` allows only -180. ### `Location` A `Location` is a set of two coordinates, one `Latitude` and one `Longitude`. ### `Heading` A `Heading` is an extension of `Angle` which represents the 0 to 360 degrees cardinal direction between North and some other direction. North has a heading of 0 degrees, East 90 degrees, South 180 degrees, and West 270 degrees. ### `PolyLine`, `Polygon` and `Segment` A `PolyLine` is a list of `Location`s. A `Polygon` is an extension of `PolyLine` where we assume that the first and last point are joined. The first and last point are not duplicated here. A `Segment` is a subset of `PolyLine` with only two points. `PolyLine`s and `Polygon`s can return the list of their `Segment`s. ### `Rectangle` and `Located` `Rectangle` is a bounding box between two `Latitude`s and `Longitude`s. Anything that is `Located` has to return bounds as a `Rectangle`. A `Rectangle` is also a `Polygon` with 4 points. ### `MultiPolygon` A `MultiPolygon` is a collection of outer `Polygon`s and their set of inner `Polygon`s. It is generally used to represent shapes with holes. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Rectangle.java ================================================ package org.openstreetmap.atlas.geography; import java.awt.geom.Rectangle2D; import java.util.Iterator; import java.util.Set; import org.locationtech.jts.geom.Envelope; import org.opengis.geometry.BoundingBox; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Angle; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Surface; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonPrimitive; /** * A rectangle on the surface of earth. It cannot span the date change line (longitude -180) * * @author matthieun */ public final class Rectangle extends Polygon { public static final Rectangle MAXIMUM = forCorners( new Location(Latitude.MINIMUM, Longitude.MINIMUM), new Location(Latitude.MAXIMUM, Longitude.MAXIMUM)); public static final Rectangle MINIMUM = forCorners(Location.CENTER, Location.CENTER); public static final Rectangle TEST_RECTANGLE = forString( "37.328167,-122.031905:37.330394,-122.029051"); public static final Rectangle TEST_RECTANGLE_2 = forString( "37.325194,-122.034281:37.325683,-122.033500"); private static final long serialVersionUID = 6940095569975683891L; private static final Logger logger = LoggerFactory.getLogger(Rectangle.class); // A rectangle stores only two locations, despite being a 4 location Polygon. private final Location lowerLeft; private final Location upperRight; public static Rectangle forBoundingBox(final BoundingBox boundingBox) { return Rectangle.forLocations( new Location(Latitude.degrees(boundingBox.getMinY()), Longitude.degrees(boundingBox.getMinX())), new Location(Latitude.degrees(boundingBox.getMaxY()), Longitude.degrees(boundingBox.getMaxX()))); } /** * Create a {@link Rectangle} from the lower left and upper right corners * * @param lowerLeft * The lower left corner * @param upperRight * The upper right corner * @return The resulting {@link Rectangle} */ public static Rectangle forCorners(final Location lowerLeft, final Location upperRight) { if (lowerLeft == null || upperRight == null) { throw new CoreException("Cannot build a Rectangle with one of the corners being null."); } // Sanity check to avoid invalid Rectangles else if (lowerLeft.isNorthOf(upperRight)) { throw new CoreException("Lower left cannot be higher than upper right."); } else if (lowerLeft.isEastOf(upperRight)) { throw new CoreException("Lower left cannot be to the right of the upper right."); } return new Rectangle(lowerLeft, upperRight); } /** * Build a {@link Rectangle} that wraps around an {@link Iterable} of {@link Located} objects. * * @param locateds * The {@link Iterable} of {@link Located} objects * @param * The type of located object. * @return The resulting {@link Rectangle} */ public static Rectangle forLocated(final Iterable locateds) { Latitude lower = null; Latitude upper = null; Longitude left = null; Longitude right = null; boolean hasOneItem = false; for (final Located located : locateds) { hasOneItem = true; for (final Location location : located.bounds()) { final Latitude latitude = location.getLatitude(); final Longitude longitude = location.getLongitude(); if (lower == null || latitude.isLessThan(lower)) { lower = latitude; } if (upper == null || latitude.isGreaterThan(upper)) { upper = latitude; } if (left == null || longitude.isLessThan(left)) { left = longitude; } if (right == null || longitude.isGreaterThan(right)) { right = longitude; } } } if (!hasOneItem) { throw new CoreException( "Rectangle.forLocated(Iterable) has to have at least one item in the Iterable"); } return forCorners(new Location(lower, left), new Location(upper, right)); } /** * Build a {@link Rectangle} that wraps around an array of {@link Located} objects. * * @param locateds * The array of {@link Located} objects * @return The resulting {@link Rectangle} */ public static Rectangle forLocated(final Located... locateds) { return forLocated(Iterables.iterable(locateds)); } /** * Build a {@link Rectangle} that wraps around an {@link Iterable} of {@link Location} objects. * * @param locations * The {@link Iterable} of {@link Location} objects * @return The resulting {@link Rectangle} */ public static Rectangle forLocations(final Iterable locations) { Latitude lower = null; Latitude upper = null; Longitude left = null; Longitude right = null; for (final Location location : locations) { final Latitude latitude = location.getLatitude(); final Longitude longitude = location.getLongitude(); if (lower == null || latitude.isLessThan(lower)) { lower = latitude; } if (upper == null || latitude.isGreaterThan(upper)) { upper = latitude; } if (left == null || longitude.isLessThan(left)) { left = longitude; } if (right == null || longitude.isGreaterThan(right)) { right = longitude; } } return forCorners(new Location(lower, left), new Location(upper, right)); } /** * Build a {@link Rectangle} that wraps around an array of {@link Location} objects. * * @param locations * The array of {@link Location} objects * @return The resulting {@link Rectangle} */ public static Rectangle forLocations(final Location... locations) { return Rectangle.forLocations(Iterables.iterable(locations)); } /** * @param rectangleString * The string definition * @return The resulting {@link Rectangle} parsed from its string definition */ public static Rectangle forString(final String rectangleString) { final StringList split = StringList.split(rectangleString, ":"); if (split.size() != 2) { throw new CoreException("Invalid Rectangle String: {}", rectangleString); } return forLocations(Location.forString(split.get(0)), Location.forString(split.get(1))); } /** * Private constructor using the two corners * * @param lowerLeft * The lower left corner * @param upperRight * The upper right corner */ private Rectangle(final Location lowerLeft, final Location upperRight) { super(lowerLeft, new Location(upperRight.getLatitude(), lowerLeft.getLongitude()), upperRight, new Location(lowerLeft.getLatitude(), upperRight.getLongitude())); this.lowerLeft = lowerLeft; this.upperRight = upperRight; } /** * @return JTS object {@link Envelope}, which is an equivalent of {@link Rectangle} */ public Envelope asEnvelope() { return new Envelope(this.lowerLeft.getLongitude().asDegrees(), this.upperRight.getLongitude().asDegrees(), this.lowerLeft.getLatitude().asDegrees(), this.upperRight.getLatitude().asDegrees()); } public JsonArray asGeoJsonBbox() { final JsonArray array = new JsonArray(); array.add(new JsonPrimitive(this.lowerLeft.getLongitude().asDegrees())); array.add(new JsonPrimitive(this.lowerLeft.getLatitude().asDegrees())); array.add(new JsonPrimitive(this.upperRight.getLongitude().asDegrees())); array.add(new JsonPrimitive(this.upperRight.getLatitude().asDegrees())); return array; } @Override public Rectangle bounds() { return this; } @Override public Location center() { return new Segment(this.lowerLeft, this.upperRight).middle(); } /** * @param that * The other {@link Rectangle} to combine * @return The {@link Rectangle} wrapping this {@link Rectangle} and the one passed as an * argument. */ public Rectangle combine(final Rectangle that) { return Rectangle.forLocations(this.lowerLeft, this.upperRight, that.lowerLeft, that.upperRight); } /** * Contract the rectangle in 4 directions as far as possible. If the distance to move the * corners would invert the rectangle then the side(s) will collapse into length 0. The most * that it can contract is to a single point in the middle. * * @param distance * to contract the four corners * @return new rectangle with contracted dimensions */ public Rectangle contract(final Distance distance) { final Location newLowerLeft = this.lowerLeft.shiftAlongGreatCircle(Heading.NORTH, distance) .shiftAlongGreatCircle(Heading.EAST, distance); final Location newUpperRight = this.upperRight .shiftAlongGreatCircle(Heading.SOUTH, distance) .shiftAlongGreatCircle(Heading.WEST, distance); final boolean tooShortHeight = newLowerLeft.getLatitude() .isGreaterThan(newUpperRight.getLatitude()); final boolean tooShortWidth = newLowerLeft.getLongitude() .isGreaterThan(newUpperRight.getLongitude()); if (tooShortHeight && tooShortWidth) { return this.center().bounds(); } else { final Location lowerRight = new Location(this.lowerLeft().getLatitude(), this.upperRight().getLongitude()); if (tooShortHeight) { final Latitude sharedLatitude = lowerRight.midPoint(this.upperRight()) .getLatitude(); return forCorners(new Location(sharedLatitude, newLowerLeft.getLongitude()), new Location(sharedLatitude, newUpperRight.getLongitude())); } else if (tooShortWidth) { final Longitude sharedLongitude = lowerRight.midPoint(this.lowerLeft()) .getLongitude(); return forCorners(new Location(newLowerLeft.getLatitude(), sharedLongitude), new Location(newUpperRight.getLatitude(), sharedLongitude)); } else { return forCorners(newLowerLeft, newUpperRight); } } } /** * Expand a given distance in all four directions * * @param distance * The {@link Distance} to expand * @return The expanded {@link Rectangle} */ public Rectangle expand(final Distance distance) { final Rectangle expandedVertically = this.expandVertically(distance); return expandedVertically.expandHorizontally(distance); } /** * Expand a given distance horizontally, on both directions * * @param distance * The {@link Distance} to expand * @return The expanded {@link Rectangle} */ public Rectangle expandHorizontally(final Distance distance) { final Location newLowerLeft = this.lowerLeft.shiftAlongGreatCircle(Heading.WEST, distance); final Location newUpperRight = this.upperRight.shiftAlongGreatCircle(Heading.EAST, distance); return forCorners(newLowerLeft, newUpperRight); } /** * Expand a given distance vertically, on both directions * * @param distance * The {@link Distance} to expand * @return The expanded {@link Rectangle} */ public Rectangle expandVertically(final Distance distance) { final double degreesLatitudeToShift = distance.asMeters() / Distance.APPROXIMATE_DISTANCE_PER_DEGREE_AT_EQUATOR.asMeters(); final long meterBuffer = 1; final Location oldLowerLeft = this.lowerLeft; Distance southShiftDistance = distance; // If the lowerLeft is about to be shifted south of the South Pole, stop it! if (oldLowerLeft.getLatitude().asDegrees() - degreesLatitudeToShift <= Latitude.MINIMUM .asDegrees()) { logger.warn( "Provided distance {} would have shifted past the South Pole, truncating southward expansion...", distance); final double degreesToHitMinimum = -1 * (Latitude.MINIMUM.asDegrees() - oldLowerLeft.getLatitude().asDegrees()); // subtract a small buffer off the distance to just miss the pole. southShiftDistance = Distance .meters((Distance.APPROXIMATE_DISTANCE_PER_DEGREE_AT_EQUATOR.asMeters() * degreesToHitMinimum) - meterBuffer); } final Location lowerLeftShiftedSouth = oldLowerLeft.shiftAlongGreatCircle(Heading.SOUTH, southShiftDistance); final Location oldUpperRight = this.upperRight; Distance northShiftDistance = distance; // If upperRight is about to be shifted north of the North Pole, stop it! if (oldUpperRight.getLatitude().asDegrees() + degreesLatitudeToShift >= Latitude.MAXIMUM .asDegrees()) { logger.warn( "Provided distance {} would have shifted past the North Pole, truncating northward expansion...", distance); final double degreesToHitMaximum = Latitude.MAXIMUM.asDegrees() - oldUpperRight.getLatitude().asDegrees(); // subtract a small buffer off the distance to just miss the pole. northShiftDistance = Distance .meters((Distance.APPROXIMATE_DISTANCE_PER_DEGREE_AT_EQUATOR.asMeters() * degreesToHitMaximum) - meterBuffer); } final Location upperRightShiftedNorth = oldUpperRight.shiftAlongGreatCircle(Heading.NORTH, northShiftDistance); return forCorners(lowerLeftShiftedSouth, upperRightShiftedNorth); } /** * Test if this rectangle fully encloses a {@link Located} item * * @param item * The item to test * @return True if this rectangle contains a {@link Located} item */ public boolean fullyGeometricallyEncloses(final Located item) { final Rectangle bounds = item instanceof Rectangle ? (Rectangle) item : item.bounds(); return this.lowerLeft().getLatitude().isLessThanOrEqualTo(bounds.lowerLeft().getLatitude()) && this.lowerLeft().getLongitude() .isLessThanOrEqualTo(bounds.lowerLeft().getLongitude()) && this.upperRight().getLatitude() .isGreaterThanOrEqualTo(bounds.upperRight().getLatitude()) && this.upperRight().getLongitude() .isGreaterThanOrEqualTo(bounds.upperRight().getLongitude()); } @Override public boolean fullyGeometricallyEncloses(final Location item) { return this.fullyGeometricallyEncloses((Located) item); } @Override public boolean fullyGeometricallyEncloses(final Rectangle item) { return this.fullyGeometricallyEncloses((Located) item); } /** * @return The height of this {@link Rectangle} */ public Angle height() { return Angle .dm7(this.upperRight.getLatitude().asDm7() - this.lowerLeft.getLatitude().asDm7()); } /** * @param other * The other {@link Rectangle} to intersect * @return The intersection of the two rectangles */ public Rectangle intersection(final Rectangle other) { if (other == null) { return null; } if (this.equals(other)) { return this; } if (this.fullyGeometricallyEncloses(other)) { return other; } if (other.fullyGeometricallyEncloses(this)) { return this; } final Set intersections = this.intersections(other); if (intersections.size() == 0) { return null; } if (intersections.size() == 1) { return Rectangle.forLocations(intersections.iterator().next()); } if (intersections.size() == 2) { final Iterator iterator = intersections.iterator(); final Location location1 = iterator.next(); final Location location2 = iterator.next(); if (!location1.getLatitude().equals(location2.getLatitude()) && !location1.getLongitude().equals(location2.getLongitude())) { return Rectangle.forLocations(location1, location2); } else { if (location1.getLatitude().equals(location2.getLatitude())) { if (this.width().isLessThanOrEqualTo(other.width())) { for (final Location missing : this) { if (other.fullyGeometricallyEncloses(missing)) { return Rectangle.forLocations(location1, location2, missing); } } } else { for (final Location missing : other) { if (this.fullyGeometricallyEncloses(missing)) { return Rectangle.forLocations(location1, location2, missing); } } } } if (location1.getLongitude().equals(location2.getLongitude())) { if (this.height().isLessThanOrEqualTo(other.height())) { for (final Location missing : this) { if (other.fullyGeometricallyEncloses(missing)) { return Rectangle.forLocations(location1, location2, missing); } } } else { for (final Location missing : other) { if (this.fullyGeometricallyEncloses(missing)) { return Rectangle.forLocations(location1, location2, missing); } } } } } } throw new CoreException("Cannot have more than 2 intersections."); } /** * @return The lower left corner {@link Location} of this {@link Rectangle} */ public Location lowerLeft() { return this.lowerLeft; } /** * @return The lower right corner {@link Location} of this {@link Rectangle} */ public Location lowerRight() { return new Location(this.lowerLeft.getLatitude(), this.upperRight.getLongitude()); } @Override public boolean overlaps(final PolyLine other) { if (other instanceof Rectangle) { final Rectangle otherRectangle = (Rectangle) other; return !(otherRectangle.lowerLeft.getLongitude() .isGreaterThan(this.upperRight.getLongitude()) || otherRectangle.upperRight.getLongitude() .isLessThan(this.lowerLeft.getLongitude()) || otherRectangle.upperRight.getLatitude() .isLessThan(this.lowerLeft.getLatitude()) || otherRectangle.lowerLeft.getLatitude() .isGreaterThan(this.upperRight.getLatitude())); } else { return super.overlaps(other); } } @Override public Surface surface() { return Surface.forAngles(height(), width()); } @Override public String toCompactString() { return this.lowerLeft.toCompactString() + ":" + this.upperRight.toCompactString(); } /** * @return The upper left corner {@link Location} of this {@link Rectangle} */ public Location upperLeft() { return new Location(this.upperRight.getLatitude(), this.lowerLeft.getLongitude()); } /** * @return The upper right corner {@link Location} of this {@link Rectangle} */ public Location upperRight() { return this.upperRight; } /** * @return The width of this {@link Rectangle} */ public Angle width() { long dm7Difference = this.upperRight.getLongitude().asDm7() - this.lowerLeft.getLongitude().asDm7(); if (dm7Difference >= Angle.REVOLUTION_DM7) { dm7Difference = Angle.REVOLUTION_DM7 - 1; } return Angle.dm7(dm7Difference); } protected Rectangle2D asAwtRectangle() { final int xAxis = (int) this.upperLeft().getLongitude().asDm7(); final int yAxis = (int) this.upperLeft().getLatitude().asDm7(); final int width = (int) (this.upperRight().getLongitude().asDm7() - this.upperLeft().getLongitude().asDm7()); final int height = (int) (this.upperLeft().getLatitude().asDm7() - this.lowerLeft().getLatitude().asDm7()); return new java.awt.Rectangle(xAxis, yAxis, width, height); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Segment.java ================================================ package org.openstreetmap.atlas.geography; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Ratio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link PolyLine} made of two {@link Location}s * * @author matthieun */ public class Segment extends PolyLine { private static final Logger logger = LoggerFactory.getLogger(Segment.class); private static final long serialVersionUID = -5796676985841139897L; /** * Convenience method to gather all {@link Location}s for a list of segments. * * @param segments * target segments * @return a list of {@link Location}s for the given segments */ public static List asList(final Iterable segments) { final List result = new ArrayList<>(); Iterables.stream(segments).forEach(segment -> { if (result.isEmpty() || !result.get(result.size() - 1).equals(segment.start())) { result.add(segment.start()); } result.add(segment.end()); }); return result; } /** * Convenience method to speed up the construction of the parent {@link PolyLine}. */ private static List asList(final Location start, final Location end) { // avoid the initial grow calls for ArrayList (there would be at least one grow call, // possibly two here) // The grow calls are ~50% of the cost for this method. // This should decrease the cost for segment creation by ~1/3. final List result = new ArrayList<>(2); result.add(start); result.add(end); return result; } /** * Ensures that numerator/denominator is within the range [0,1] without doing the division * * @param denominator * The denominator of the fraction * @param numerator * The numerator of the fraction * @return {@code true} if the fraction is in the range [0,1] */ private static boolean rangeCheck(final double denominator, final double numerator) { return denominator > 0 && (numerator < 0 || numerator > denominator) || denominator < 0 && (numerator > 0 || numerator < denominator); } /** * Ensures that numerator/denominator is within the range [0,1] without doing the division * * @param denominator * The denominator of the fraction * @param numerator * The numerator of the fraction * @return {@code true} if the fraction is in the range [0,1] */ private static boolean rangeCheck(final long denominator, final long numerator) { return denominator > 0 && (numerator < 0 || numerator > denominator) || denominator < 0 && (numerator > 0 || numerator < denominator); } public Segment(final Location start, final Location end) { super(asList(start, end)); } public Location end() { return this.last(); } @Override public boolean equals(final Object other) { if (other instanceof Segment) { final Segment that = (Segment) other; return this.start().equals(that.start()) && this.end().equals(that.end()); } return false; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (this.end() == null ? 0 : this.end().hashCode()); result = prime * result + (this.start() == null ? 0 : this.start().hashCode()); return result; } /** * @return The {@link Segment}'s {@link Heading}. In case the segment is the same start and end * locations, then the result is empty. */ public Optional heading() { if (this.isPoint()) { logger.warn( "Cannot compute a segment's heading when the segment is a point with same start and end {}", this.start()); return Optional.empty(); } return Optional.of(this.start().headingTo(this.end())); } /** * Intersection of two segments * * @param that * The other segment to intersect * @return The intersection point if any, null otherwise * @see "http://stackoverflow.com/a/1968345/1558687" */ public Location intersection(final Segment that) { final double p0X = this.start().getLongitude().asDegrees(); final double p0Y = this.start().getLatitude().asDegrees(); final double p1X = this.end().getLongitude().asDegrees(); final double p1Y = this.end().getLatitude().asDegrees(); final double p2X = that.start().getLongitude().asDegrees(); final double p2Y = that.start().getLatitude().asDegrees(); final double p3X = that.end().getLongitude().asDegrees(); final double p3Y = that.end().getLatitude().asDegrees(); final double s1X; final double s1Y; final double s2X; final double s2Y; s1X = p1X - p0X; s1Y = p1Y - p0Y; s2X = p3X - p2X; s2Y = p3Y - p2Y; final double sValue; final double tValue; sValue = (-s1Y * (p0X - p2X) + s1X * (p0Y - p2Y)) / (-s2X * s1Y + s1X * s2Y); tValue = (s2X * (p0Y - p2Y) - s2Y * (p0X - p2X)) / (-s2X * s1Y + s1X * s2Y); if (sValue >= 0 && sValue <= 1 && tValue >= 0 && tValue <= 1) { // Collision detected return new Location(Latitude.degrees(p0Y + tValue * s1Y), Longitude.degrees(p0X + tValue * s1X)); } // No collision return null; } /** * A fast method to test if two segments intersect * * @param that * The other {@link Segment} to test with * @return True if this segment intersects that segment * @see "http://www.java-gaming.org/index.php?topic=22590.0" */ public boolean intersects(final Segment that) { final long xAxis1 = this.start().getLongitude().asDm7(); final long yAxis1 = this.start().getLatitude().asDm7(); final long xAxis2 = this.end().getLongitude().asDm7(); final long yAxis2 = this.end().getLatitude().asDm7(); final long xAxis3 = that.start().getLongitude().asDm7(); final long yAxis3 = that.start().getLatitude().asDm7(); final long xAxis4 = that.end().getLongitude().asDm7(); final long yAxis4 = that.end().getLatitude().asDm7(); // Return false if either of the lines have zero length if (xAxis1 == xAxis2 && yAxis1 == yAxis2 || xAxis3 == xAxis4 && yAxis3 == yAxis4) { return false; } // Fastest method, based on Franklin Antonio's "Faster Line Segment Intersection" topic // "in Graphics Gems III" book (http://www.graphicsgems.org/) final long axAxis = xAxis2 - xAxis1; final long ayAxis = yAxis2 - yAxis1; final long bxAxis = xAxis3 - xAxis4; final long byAxis = yAxis3 - yAxis4; final long cxAxis = xAxis1 - xAxis3; final long cyAxis = yAxis1 - yAxis3; try { final long alphaNumerator = Math.subtractExact(byAxis * cxAxis, bxAxis * cyAxis); final long commonDenominator = Math.subtractExact(ayAxis * bxAxis, axAxis * byAxis); // ensures that alpha is within the range [0,1] without doing the division if (rangeCheck(commonDenominator, alphaNumerator)) { return false; } final long betaNumerator = Math.subtractExact(axAxis * cyAxis, ayAxis * cxAxis); // ensures that beta is within the range [0,1] without doing the division if (rangeCheck(commonDenominator, betaNumerator)) { return false; } if (commonDenominator == 0) { // This code wasn't in Franklin Antonio's method. It was added by Keith Woodward. // The // lines are parallel. Check if they're collinear. // see "http://mathworld.wolfram.com/Collinear.html" final long collinearityTestForP3 = xAxis1 * (yAxis2 - yAxis3) + xAxis2 * (yAxis3 - yAxis1) + xAxis3 * (yAxis1 - yAxis2); // If p3 is collinear with p1 and p2 then p4 will also be collinear, since p1-p2 is // parallel with p3-p4 if (collinearityTestForP3 == 0) { // The lines are collinear. Now check if they overlap. if ((xAxis1 >= xAxis3 && xAxis1 <= xAxis4 || xAxis1 <= xAxis3 && xAxis1 >= xAxis4 || xAxis2 >= xAxis3 && xAxis2 <= xAxis4 || xAxis2 <= xAxis3 && xAxis2 >= xAxis4 || xAxis3 >= xAxis1 && xAxis3 <= xAxis2 || xAxis3 <= xAxis1 && xAxis3 >= xAxis2) && (yAxis1 >= yAxis3 && yAxis1 <= yAxis4 || yAxis1 <= yAxis3 && yAxis1 >= yAxis4 || yAxis2 >= yAxis3 && yAxis2 <= yAxis4 || yAxis2 <= yAxis3 && yAxis2 >= yAxis4 || yAxis3 >= yAxis1 && yAxis3 <= yAxis2 || yAxis3 <= yAxis1 && yAxis3 >= yAxis2)) { return true; } } return false; } return true; } catch (final ArithmeticException overflow) { return this.intersectsApproximate(that); } } /** * @return True if this segment is exactly east west (the two latitudes are the same) */ public boolean isEastWest() { return start().hasSameLatitudeAs(end()); } /** * @return True if this segment is exactly north south (the two longitudes are the same) */ public boolean isNorthSouth() { return start().hasSameLongitudeAs(end()); } @Override public boolean isPoint() { return start().equals(end()); } @Override public Distance length() { return this.start().distanceTo(this.end()); } @Override public Location middle() { return new Location( Latitude.degrees((this.start().getLatitude().asDegrees() + this.end().getLatitude().asDegrees()) / 2.0), Longitude.degrees((this.start().getLongitude().asDegrees() + this.end().getLongitude().asDegrees()) / 2.0)); } @Override public Location offsetFromStart(final Ratio ratio) { final Optional heading = heading(); if (heading.isPresent()) { return this.start().shiftAlongGreatCircle(heading.get(), length().scaleBy(ratio)); } return this.start(); } /** * @return The same segment but pointing north if it is not already, by reversing it if it * points south. */ public Segment pointingNorth() { if (this.isEastWest()) { return this; } if (start().getLatitude().isLessThan(end().getLatitude())) { return this; } return new Segment(end(), start()); } @Override public Segment reversed() { return new Segment(end(), start()); } public Location start() { return this.first(); } /** * The Dot Product of two segments (seen as 2D space vectors) * * @param that * The other {@link Segment} * @return The Dot Product of the two segments */ protected double dotProduct(final Segment that) { final double thisLatitudeSpan = this.latitudeSpan(); final double thatLatitudeSpan = that.latitudeSpan(); final double thisLongitudeSpan = this.longitudeSpan(); final double thatLongitudeSpan = that.longitudeSpan(); return thisLatitudeSpan * thatLatitudeSpan + thisLongitudeSpan * thatLongitudeSpan; } protected double dotProductLength() { return Math.sqrt(dotProduct(this)); } protected long latitudeSpan() { return this.end().getLatitude().asDm7() - this.start().getLatitude().asDm7(); } protected long longitudeSpan() { return this.end().getLongitude().asDm7() - this.start().getLongitude().asDm7(); } /** * Implements the same function as intersects but with doubles to avoid overlflow issues. Should * only happen for cross world intersection. * * @param that * @return */ private boolean intersectsApproximate(final Segment that) { final double xAxis1 = this.start().getLongitude().asDegrees(); final double yAxis1 = this.start().getLatitude().asDegrees(); final double xAxis2 = this.end().getLongitude().asDegrees(); final double yAxis2 = this.end().getLatitude().asDegrees(); final double xAxis3 = that.start().getLongitude().asDegrees(); final double yAxis3 = that.start().getLatitude().asDegrees(); final double xAxis4 = that.end().getLongitude().asDegrees(); final double yAxis4 = that.end().getLatitude().asDegrees(); // Return false if either of the lines have zero length if (xAxis1 == xAxis2 && yAxis1 == yAxis2 || xAxis3 == xAxis4 && yAxis3 == yAxis4) { return false; } // Fastest method, based on Franklin Antonio's "Faster Line Segment Intersection" topic // "in Graphics Gems III" book (http://www.graphicsgems.org/) final double axAxis = xAxis2 - xAxis1; final double ayAxis = yAxis2 - yAxis1; final double bxAxis = xAxis3 - xAxis4; final double byAxis = yAxis3 - yAxis4; final double cxAxis = xAxis1 - xAxis3; final double cyAxis = yAxis1 - yAxis3; final double alphaNumerator = byAxis * cxAxis - bxAxis * cyAxis; final double commonDenominator = ayAxis * bxAxis - axAxis * byAxis; // ensures that alpha is within the range [0,1] without doing the division if (rangeCheck(commonDenominator, alphaNumerator)) { return false; } final double betaNumerator = axAxis * cyAxis - ayAxis * cxAxis; // ensures that beta is within the range [0,1] without doing the division if (rangeCheck(commonDenominator, betaNumerator)) { return false; } if (commonDenominator == 0) { // This code wasn't in Franklin Antonio's method. It was added by Keith Woodward. The // lines are parallel. Check if they're collinear. // see "http://mathworld.wolfram.com/Collinear.html" final double collinearityTestForP3 = xAxis1 * (yAxis2 - yAxis3) + xAxis2 * (yAxis3 - yAxis1) + xAxis3 * (yAxis1 - yAxis2); // If p3 is collinear with p1 and p2 then p4 will also be collinear, since p1-p2 is // parallel with p3-p4 if (collinearityTestForP3 == 0) { // The lines are collinear. Now check if they overlap. if ((xAxis1 >= xAxis3 && xAxis1 <= xAxis4 || xAxis1 <= xAxis3 && xAxis1 >= xAxis4 || xAxis2 >= xAxis3 && xAxis2 <= xAxis4 || xAxis2 <= xAxis3 && xAxis2 >= xAxis4 || xAxis3 >= xAxis1 && xAxis3 <= xAxis2 || xAxis3 <= xAxis1 && xAxis3 >= xAxis2) && (yAxis1 >= yAxis3 && yAxis1 <= yAxis4 || yAxis1 <= yAxis3 && yAxis1 >= yAxis4 || yAxis2 >= yAxis3 && yAxis2 <= yAxis4 || yAxis2 <= yAxis3 && yAxis2 >= yAxis4 || yAxis3 >= yAxis1 && yAxis3 <= yAxis2 || yAxis3 <= yAxis1 && yAxis3 >= yAxis2)) { return true; } } return false; } return true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/Snapper.java ================================================ package org.openstreetmap.atlas.geography; import java.util.Objects; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.common.collect.Iterables; /** * Snap a {@link Location} to a {@link PolyLine}. * * @author matthieun * @author bbreithaupt */ public class Snapper { /** * A snapped location on a shape. * * @author matthieun */ public static class SnappedLocation extends Location implements Comparable { private static final long serialVersionUID = -3283158797347353372L; private final Location origin; private final PolyLine target; public SnappedLocation(final Location origin, final Location snapped, final PolyLine target) { super(snapped.asConcatenation()); this.origin = origin; this.target = target; } @Override public int compareTo(final SnappedLocation other) { if (getDistance().isLessThan(other.getDistance())) { return -1; } else if (getDistance().equals(other.getDistance())) { return 0; } else { return 1; } } @Override public boolean equals(final Object other) { if (other instanceof SnappedLocation) { return this.origin.equals(((SnappedLocation) other).getOrigin()) && this.target.equals(((SnappedLocation) other).getTarget()); } if (other instanceof Location) { return super.equals(other); } return false; } /** * @return The distance between the origin and the snapped {@link Location} */ public Distance getDistance() { return this.origin.distanceTo(this); } public Location getOrigin() { return this.origin; } public PolyLine getTarget() { return this.target; } @Override public int hashCode() { return Objects.hash(this.origin, this.target); } } /** * Snap a point on a {@link PolyLine} * * @param origin * The point to snap * @param shape * The {@link PolyLine} to snap to * @return The resulting {@link SnappedLocation} */ public SnappedLocation snap(final Location origin, final Iterable shape) { if (shape instanceof Segment) { final Segment target = (Segment) shape; return snapSegment(origin, target); } else if (Iterables.size(shape) > 1) { final PolyLine target; if (shape instanceof PolyLine) { target = (PolyLine) shape; } else if (shape instanceof Polygon) { target = (Polygon) shape; } else { target = new PolyLine(shape); } SnappedLocation best = null; for (final Segment segment : target.segments()) { final SnappedLocation candidate = snap(origin, segment); if (best == null || candidate.getDistance().isLessThan(best.getDistance())) { best = candidate; } } // Return a SnappedLocation with the full shape return new SnappedLocation(origin, best, target); } else if (Iterables.size(shape) == 1) { // We have a single location in the Iterable final Location target = shape.iterator().next(); return new SnappedLocation(origin, target, new PolyLine(target)); } return null; } public SnappedLocation snap(final Location origin, final MultiPolygon shape) { SnappedLocation best = null; for (final Polygon member : new MultiIterable<>(shape.outers(), shape.inners())) { final SnappedLocation candidate = snap(origin, member); if (best == null || candidate.getDistance().isLessThan(best.getDistance())) { best = candidate; } } return best; } private SnappedLocation snapSegment(final Location origin, final Segment shape) { // Use the dot product to determine if the snapped point is within the segment, or at // the edge points final Segment variable = new Segment(shape.start(), origin); final double dotProduct = shape.dotProduct(variable); if (dotProduct <= 0) { return new SnappedLocation(origin, shape.start(), shape); } // Here, NOSONAR to avoid "Collections should not be passed as arguments to their own // methods (squid:S2114)" // It is triggered because Segment is also a collection. if (dotProduct >= shape.dotProduct(shape)) // NOSONAR { return new SnappedLocation(origin, shape.end(), shape); } // Find the point in the middle. // Inspired from http://www.sunshine2k.de/coding/java/PointOnLine/PointOnLine.html#step5 // The angle between the target and variable segment is alpha final double cosAlpha = dotProduct / (shape.dotProductLength() * variable.dotProductLength()); // Cos Alpha is also defined as (offset distance on target) / (variable's length) final double offsetDistance = cosAlpha * variable.dotProductLength(); final double latitudeAsDm7 = shape.start().getLatitude().asDm7() + offsetDistance / shape.dotProductLength() * shape.latitudeSpan(); final double longitudeAsDm7 = shape.start().getLongitude().asDm7() + offsetDistance / shape.dotProductLength() * shape.longitudeSpan(); final Location snapped = new Location(Latitude.dm7(Math.round(latitudeAsDm7)), Longitude.dm7(Math.round(longitudeAsDm7))); return new SnappedLocation(origin, snapped, shape); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/StringCompressedPolyLine.java ================================================ package org.openstreetmap.atlas.geography; import java.io.Serializable; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Encode a {@link PolyLine} using an algorithm derived from the MapQuest variant of the Google * Polyline Encoding Format. The encoding scheme falls back on WKB in edge cases when the algorithm * fails (such as when two consecutive points in the polyline have a longitude difference of greater * than 180 degrees). * * @see The MapQuest * algorithm * @see Google * Polyline Encoding Format * @author matthieun */ public class StringCompressedPolyLine implements Serializable { /** * @author matthieun */ public static class PolyLineCompressionException extends CoreException { private static final long serialVersionUID = -3974024747280370420L; public PolyLineCompressionException(final String message, final Object... items) { super(message, items); } } private static final long serialVersionUID = 5315700936842774861L; // dm7 private static final int PRECISION = 7; private static final Charset CHARSET = StandardCharsets.UTF_8; private static final int ENCODING_OFFSET_MINUS_ONE = 63; private static final int FIVE_BIT_MASK = 0x1f; private static final int SIXTH_BIT_MASK = 0x20; private static final int BIT_SHIFT = 5; // To allow for degree of magnitude 7 precision, two longitudes should not be too far apart. // Using 180 degrees as a limit. private static final long MAXIMUM_DELTA_LONGITUDE_IN_DEGREES = 180; private static final long MAXIMUM_DELTA_LONGITUDE = (long) (MAXIMUM_DELTA_LONGITUDE_IN_DEGREES * Math.pow(10, PRECISION)); /* * If the first byte of the encoding array is this sentinel value, then the following encoding * is WKB and not string-compressed. We use '0' as the sentinel value since the string * compression algorithm will always use printable ASCII characters. There will never be a 0 * byte in a valid string-compressed polyline. */ private static final byte WKB_SENTINEL = 0; private static final Logger logger = LoggerFactory.getLogger(StringCompressedPolyLine.class); private byte[] encoding; public StringCompressedPolyLine(final byte[] encoding) { this.encoding = encoding; } public StringCompressedPolyLine(final PolyLine polyLine) { try { this.encoding = compress(polyLine, PRECISION).getBytes(CHARSET); } catch (final PolyLineCompressionException exception) { logger.warn("Unable to use string compression", exception); this.encoding = getWkbFallback(polyLine); } catch (final Exception exception) { throw new CoreException("Could not compress polyline.", exception); } } public PolyLine asPolyLine() { if (this.encoding[0] == WKB_SENTINEL) { final byte[] strippedEncoding = new byte[this.encoding.length - 1]; System.arraycopy(this.encoding, 1, strippedEncoding, 0, strippedEncoding.length); return PolyLine.wkb(strippedEncoding); } else { String encodedString = null; try { encodedString = new String(this.encoding, CHARSET); return asPolyLine(encodedString, PRECISION); } catch (final Exception exception) { throw new CoreException( "Could not decompress polyline:\nEncoding: \'{}\'\nString: \'\'.", this.encoding, encodedString, exception); } } } public byte[] getEncoding() { return this.encoding; } @Override public String toString() { if (this.encoding[0] == WKB_SENTINEL) { final byte[] strippedEncoding = new byte[this.encoding.length - 1]; System.arraycopy(this.encoding, 1, strippedEncoding, 0, strippedEncoding.length); return PolyLine.wkb(strippedEncoding).toWkt(); } else { try { return new String(this.encoding, CHARSET); } catch (final Exception e) { throw new CoreException("Could not stringify byte array.", e); } } } private PolyLine asPolyLine(final String encoded, final int precision) { final double precision2 = Math.pow(10, -precision); final int length = encoded.length(); int index = 0; int latitude = 0; int longitude = 0; final List array = new ArrayList<>(); while (index < length) { int byteEncoded; int shift = 0; int result = 0; do { byteEncoded = Character.codePointAt(encoded, index++) - ENCODING_OFFSET_MINUS_ONE; result |= (byteEncoded & FIVE_BIT_MASK) << shift; shift += BIT_SHIFT; } while (byteEncoded >= SIXTH_BIT_MASK); final int deltaLatitude = (result & 1) > 0 ? ~(result >>> 1) : result >>> 1; latitude += deltaLatitude; shift = 0; result = 0; do { byteEncoded = Character.codePointAt(encoded, index++) - ENCODING_OFFSET_MINUS_ONE; result |= (byteEncoded & FIVE_BIT_MASK) << shift; shift += BIT_SHIFT; } while (byteEncoded >= SIXTH_BIT_MASK); final int deltalongitude = (result & 1) > 0 ? ~(result >>> 1) : result >>> 1; longitude += deltalongitude; array.add(new Location(Latitude.degrees(latitude * precision2), Longitude.degrees(longitude * precision2))); } return new PolyLine(array); } private String compress(final PolyLine points, final int precision0) { long oldLatitude = 0; long oldLongitude = 0; final StringBuilder encoded = new StringBuilder(); final double precision = Math.pow(10, precision0); Location last = Location.CENTER; for (final Location location : points) { // Round to N decimal places final long latitude = Math.round(location.getLatitude().asDegrees() * precision); final long longitude = Math.round(location.getLongitude().asDegrees() * precision); // Encode the differences between the points encoded.append(encodeNumber(latitude - oldLatitude)); final long deltaLongitude = longitude - oldLongitude; if (Math.abs(deltaLongitude) > MAXIMUM_DELTA_LONGITUDE) { throw new PolyLineCompressionException( "Unable to compress the polyLine, two consecutive points ({} and {}) are too far apart in longitude: {} degrees.", last, location, deltaLongitude / precision); } encoded.append(encodeNumber(deltaLongitude)); oldLatitude = latitude; oldLongitude = longitude; last = location; } return encoded.toString(); } private String encodeNumber(final long number0) { long number = number0 << 1; if (number < 0) { number = ~number; } final StringBuilder encoded = new StringBuilder(); while (number >= SIXTH_BIT_MASK) { encoded.append(String.valueOf(Character.toChars( (SIXTH_BIT_MASK | (int) number & FIVE_BIT_MASK) + ENCODING_OFFSET_MINUS_ONE))); number >>>= BIT_SHIFT; } encoded.append(Character.toChars((int) number + ENCODING_OFFSET_MINUS_ONE)); return encoded.toString(); } private byte[] getWkbFallback(final PolyLine polyLine) { final byte[] wkbEncoding = polyLine.toWkb(); final byte[] finalEncoding = new byte[1 + wkbEncoding.length]; finalEncoding[0] = WKB_SENTINEL; System.arraycopy(wkbEncoding, 0, finalEncoding, 1, wkbEncoding.length); return finalEncoding; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/StringCompressedPolygon.java ================================================ package org.openstreetmap.atlas.geography; /** * Compressed {@link Polygon} * * @author matthieun */ public class StringCompressedPolygon extends StringCompressedPolyLine { private static final long serialVersionUID = 7681617657249431319L; public StringCompressedPolygon(final byte[] encoding) { super(encoding); } public StringCompressedPolygon(final Polygon polygon) { super(polygon); } public Polygon asPolygon() { return new Polygon(asPolyLine()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/WkbPrintable.java ================================================ package org.openstreetmap.atlas.geography; /** * @author matthieun */ public interface WkbPrintable { byte[] toWkb(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/WktPrintable.java ================================================ package org.openstreetmap.atlas.geography; import org.openstreetmap.atlas.utilities.collections.StringList; /** * @author matthieun */ public interface WktPrintable { static String toWktCollection(final Iterable input) { final StringList wktList = new StringList(); input.forEach(wktPrintable -> wktList.add(wktPrintable.toWkt())); final StringBuilder builder = new StringBuilder(); builder.append("GEOMETRYCOLLECTION ("); builder.append(wktList.join(", ")); builder.append(")"); return builder.toString(); } String toWkt(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/AbstractAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.index.PackedSpatialIndex; import org.openstreetmap.atlas.geography.index.RTree; import org.openstreetmap.atlas.geography.index.SpatialIndex; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Abstract implementation of {@link Atlas} that covers common methods. * * @author matthieun * @author tony * @author mgostintsev */ public abstract class AbstractAtlas extends BareAtlas { protected static final long DEFAULT_NUMBER_OF_ITEMS = 1024; protected static final int HASH_MODULO_RATIO = 10; private static final long serialVersionUID = -1408393006815178776L; private static final Logger logger = LoggerFactory.getLogger(AbstractAtlas.class); // Spatial index lock objects for thread protection. Even though it looks not necessary, those // locks are static to avoid issues when deserializing Atlas files. If non static, they might // end up being null in some cases right after deserialization. private static final Object NODE_LOCK = new Object(); private static final Object EDGE_LOCK = new Object(); private static final Object AREA_LOCK = new Object(); private static final Object LINE_LOCK = new Object(); private static final Object POINT_LOCK = new Object(); private static final Object RELATION_LOCK = new Object(); // Spatial indices // Transient: Those are not serialized, and re-generated on the fly // Volatile: This is to allow double checked locking to be safe in the // this.buildXXXXXSpatialIndexIfNecessary() methods. // http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html // See: "Fixing Double-Checked Locking using Volatile" private transient volatile SpatialIndex nodeSpatialIndex; private transient volatile SpatialIndex edgeSpatialIndex; private transient volatile SpatialIndex areaSpatialIndex; private transient volatile SpatialIndex lineSpatialIndex; private transient volatile SpatialIndex pointSpatialIndex; private transient volatile SpatialIndex relationSpatialIndex; /** * Create an {@link Atlas} from an OSM protobuf and save it to a resource. Skip slicing. * * @param osmPbf * The OSM protobuf * @param atlasResource * The {@link WritableResource} to save the {@link Atlas} to * @return The created {@link Atlas} */ public static Atlas createAndSaveOsmPbf(final Resource osmPbf, final WritableResource atlasResource) { Atlas atlas = new RawAtlasGenerator(osmPbf).build(); atlas = new AtlasSectionProcessor(atlas, AtlasLoadingOption.createOptionWithNoSlicing()) .run(); atlas.save(atlasResource); return atlas; } /** * Create an {@link Atlas} from an OSM protobuf that has already been sliced and save it to a * resource * * @param osmPbf * The OSM protobuf * @param atlasResource * The {@link WritableResource} to save the {@link Atlas} to * @param boundaryMap * The {@link CountryBoundaryMap} to use for country-slicing * @return The created {@link Atlas} */ public static Atlas createAndSaveOsmPbfWithSlicing(final Resource osmPbf, final WritableResource atlasResource, final CountryBoundaryMap boundaryMap) { Atlas atlas = new RawAtlasGenerator(osmPbf).build(); final AtlasLoadingOption loadingOption = AtlasLoadingOption .createOptionWithAllEnabled(boundaryMap); atlas = new RawAtlasSlicer(loadingOption, atlas).slice(); atlas = new AtlasSectionProcessor(atlas, AtlasLoadingOption.createOptionWithAllEnabled(boundaryMap)).run(); atlas.save(atlasResource); return atlas; } /** * Create from an OSM protobuf resource. Skip slicing. * * @param resource * The OSM protobuf resource * @return The Atlas read from the pbf */ public static Atlas forOsmPbf(final Resource resource) { Atlas atlas = new RawAtlasGenerator(resource).build(); atlas = new AtlasSectionProcessor(atlas, AtlasLoadingOption.createOptionWithNoSlicing()) .run(); return atlas; } @Override public Iterable areasCovering(final Location location) { return Iterables.stream(this.getAreaSpatialIndex().get(location.bounds())).filter(area -> { final Polygon areaPolygon = area.asPolygon(); return areaPolygon.fullyGeometricallyEncloses(location); }); } @Override public Iterable areasCovering(final Location location, final Predicate matcher) { return Iterables.filter(areasCovering(location), matcher); } @Override public Iterable areasIntersecting(final GeometricSurface surface) { return Iterables.stream(this.getAreaSpatialIndex().get(surface.bounds())).filter(area -> { final Polygon areaPolygon = area.asPolygon(); return surface.overlaps(areaPolygon); }); } @Override public Iterable areasIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filterTranslate(areasIntersecting(surface), item -> item, matcher); } @Override public Iterable areasWithin(final GeometricSurface surface) { return Iterables.stream(this.getAreaSpatialIndex().get(surface.bounds())).filter(area -> { final Polygon areaPolygon = area.asPolygon(); return surface.fullyGeometricallyEncloses(areaPolygon); }); } @Override public Iterable edgesContaining(final Location location) { return Iterables.stream(this.getEdgeSpatialIndex().get(location.bounds())).filter(edge -> { final PolyLine polyline = edge.asPolyLine(); return polyline.contains(location); }); } @Override public Iterable edgesContaining(final Location location, final Predicate matcher) { return Iterables.filter(edgesContaining(location), matcher); } @Override public Iterable edgesIntersecting(final GeometricSurface surface) { return Iterables.stream(this.getEdgeSpatialIndex().get(surface.bounds())).filter(edge -> { final PolyLine polyline = edge.asPolyLine(); return surface.overlaps(polyline); }); } @Override public Iterable edgesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(edgesIntersecting(surface), matcher); } @Override public Iterable edgesWithin(final GeometricSurface surface) { return Iterables.stream(this.getEdgeSpatialIndex().get(surface.bounds())).filter(edge -> { final PolyLine polyline = edge.asPolyLine(); return surface.fullyGeometricallyEncloses(polyline); }); } public SpatialIndex getAreaSpatialIndex() { buildAreaSpatialIndexIfNecessary(); return this.areaSpatialIndex; } public SpatialIndex getEdgeSpatialIndex() { buildEdgeSpatialIndexIfNecessary(); return this.edgeSpatialIndex; } public SpatialIndex getLineSpatialIndex() { buildLineSpatialIndexIfNecessary(); return this.lineSpatialIndex; } public SpatialIndex getNodeSpatialIndex() { buildNodeSpatialIndexIfNecessary(); return this.nodeSpatialIndex; } public SpatialIndex getPointSpatialIndex() { buildPointSpatialIndexIfNecessary(); return this.pointSpatialIndex; } public SpatialIndex getRelationSpatialIndex() { buildRelationSpatialIndexIfNecessary(); return this.relationSpatialIndex; } @Override public Iterable linesContaining(final Location location) { return Iterables.stream(this.getLineSpatialIndex().get(location.bounds())).filter(line -> { final PolyLine polyline = line.asPolyLine(); return polyline.contains(location); }); } @Override public Iterable linesContaining(final Location location, final Predicate matcher) { return Iterables.filter(linesContaining(location), matcher); } @Override public Iterable linesIntersecting(final GeometricSurface surface) { return Iterables.stream(this.getLineSpatialIndex().get(surface.bounds())).filter(line -> { final PolyLine polyline = line.asPolyLine(); return surface.overlaps(polyline); }); } @Override public Iterable linesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(linesIntersecting(surface), matcher); } @Override public Iterable linesWithin(final GeometricSurface surface) { return Iterables.stream(this.getLineSpatialIndex().get(surface.bounds())).filter(line -> { final PolyLine polyline = line.asPolyLine(); return surface.fullyGeometricallyEncloses(polyline); }); } @Override public Iterable nodesAt(final Location location) { return this.getNodeSpatialIndex().get(location.bounds()); } @Override public Iterable nodesWithin(final GeometricSurface surface) { final Iterable nodes = this.getNodeSpatialIndex().get(surface.bounds()); if (surface instanceof Rectangle) { return nodes; } return Iterables.filter(nodes, node -> surface.fullyGeometricallyEncloses(node.getLocation())); } @Override public Iterable nodesWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(nodesWithin(surface), matcher); } @Override public Iterable pointsAt(final Location location) { return this.getPointSpatialIndex().get(location.bounds()); } @Override public Iterable pointsWithin(final GeometricSurface surface) { final Iterable points = this.getPointSpatialIndex().get(surface.bounds()); if (surface instanceof Rectangle) { return points; } return Iterables.filter(points, point -> surface.fullyGeometricallyEncloses(point.getLocation())); } @Override public Iterable pointsWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filterTranslate(pointsWithin(surface), item -> item, matcher); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface) { final Iterable relations = this.getRelationSpatialIndex().get(surface.bounds()); return Iterables.filter(relations, relation -> relation.intersects(surface)); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(relationsWithEntitiesIntersecting(surface), matcher); } @Override public Iterable relationsWithEntitiesWithin(final GeometricSurface surface) { final Iterable relations = this.getRelationSpatialIndex().get(surface.bounds()); return Iterables.filter(relations, relation -> relation.within(surface)); } @Override public void save(final WritableResource writableResource) { throw new CoreException( "{} does not support saving. Consider using {} instead. A {} can be had using Atlas.cloneToPackedAtlas()", this.getClass().getName(), PackedAtlas.class.getName(), PackedAtlas.class.getName()); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildAreaSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(AREA_LOCK, ItemType.AREA, this::newAreaSpatialIndex, () -> this.areaSpatialIndex, newSpatialIndex -> this.areaSpatialIndex = newSpatialIndex); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildEdgeSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(EDGE_LOCK, ItemType.EDGE, this::newEdgeSpatialIndex, () -> this.edgeSpatialIndex, newSpatialIndex -> this.edgeSpatialIndex = newSpatialIndex); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildLineSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(LINE_LOCK, ItemType.LINE, this::newLineSpatialIndex, () -> this.lineSpatialIndex, newSpatialIndex -> this.lineSpatialIndex = newSpatialIndex); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildNodeSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(NODE_LOCK, ItemType.NODE, this::newNodeSpatialIndex, () -> this.nodeSpatialIndex, newSpatialIndex -> this.nodeSpatialIndex = newSpatialIndex); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildPointSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(POINT_LOCK, ItemType.POINT, this::newPointSpatialIndex, () -> this.pointSpatialIndex, newSpatialIndex -> this.pointSpatialIndex = newSpatialIndex); } /** * This method is useful for de-serialized Atlases. When an Atlas is serialized, the indices are * not saved (transient). When de-serialized, they will be null, until this method is called. */ protected void buildRelationSpatialIndexIfNecessary() { buildSpatialIndexIfNecessary(RELATION_LOCK, ItemType.RELATION, this::newRelationSpatialIndex, () -> this.relationSpatialIndex, newSpatialIndex -> this.relationSpatialIndex = newSpatialIndex); } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewAreaSpatialIndex() { if (this.areaSpatialIndex == null) { this.areaSpatialIndex = newAreaSpatialIndex(); } return this.areaSpatialIndex; } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewEdgeSpatialIndex() { if (this.edgeSpatialIndex == null) { this.edgeSpatialIndex = newEdgeSpatialIndex(); } return this.edgeSpatialIndex; } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewLineSpatialIndex() { if (this.lineSpatialIndex == null) { this.lineSpatialIndex = newLineSpatialIndex(); } return this.lineSpatialIndex; } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewNodeSpatialIndex() { if (this.nodeSpatialIndex == null) { this.nodeSpatialIndex = newNodeSpatialIndex(); } return this.nodeSpatialIndex; } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewPointSpatialIndex() { if (this.pointSpatialIndex == null) { this.pointSpatialIndex = newPointSpatialIndex(); } return this.pointSpatialIndex; } /** * @return The spatial index as new (meaning empty). This has to be used only in the protected * constructors, to not conflict with the thread safe methods that re-build spatial * indices as needed. */ protected SpatialIndex getAsNewRelationSpatialIndex() { if (this.relationSpatialIndex == null) { this.relationSpatialIndex = newRelationSpatialIndex(); } return this.relationSpatialIndex; } /** * Implementation of double-checked locking with volatile global variable as suggested by sonar * * @see "https://rules.sonarsource.com/java/tag/multi-threading/RSPEC-2168" * @param lock * An object to lock on. Needs to be a global static variable. * @param type * The type of the Spatial Index object to create * @param newIndexSupplier * A function that returns a new built-out index * @param globalIndexSupplier * A function that returns the existing global index * @param globalIndexConsumer * A function that resets the existing global index */ @SuppressWarnings("unchecked") private void buildSpatialIndexIfNecessary(final Object lock, final ItemType type, final Supplier> newIndexSupplier, final Supplier> globalIndexSupplier, final Consumer> globalIndexConsumer) { SpatialIndex localIndex = globalIndexSupplier.get(); if (localIndex == null) { // Here lock is a global static variable. Sonar cannot see it here, hence the trailing // comment. synchronized (lock) // NOSONAR { localIndex = globalIndexSupplier.get(); if (localIndex == null) { logger.info("Re-Building {} Spatial Index...", type); final SpatialIndex temporaryIndex = newIndexSupplier.get(); Iterables.stream(this.entities(type, type.getMemberClass())) .map(entity -> (M) entity).forEach(temporaryIndex::add); globalIndexConsumer.accept(temporaryIndex); } } } } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newAreaSpatialIndex() { return newSpatialIndex((item, bounds) -> bounds.overlaps(item.asPolygon()), this::area); } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newEdgeSpatialIndex() { return newSpatialIndex((item, bounds) -> bounds.overlaps(item.asPolyLine()), this::edge); } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newLineSpatialIndex() { return newSpatialIndex((item, bounds) -> bounds.overlaps(item.asPolyLine()), this::line); } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newNodeSpatialIndex() { return newSpatialIndex((item, bounds) -> bounds.fullyGeometricallyEncloses(item), this::node); } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newPointSpatialIndex() { return newSpatialIndex((item, bounds) -> bounds.fullyGeometricallyEncloses(item), this::point); } /** * Create a new spatial index * * @return A newly created spatial index */ private SpatialIndex newRelationSpatialIndex() { return newSpatialIndex((item, bounds) -> item.intersects(bounds), this::relation); } /** * @param memberValidForBounds * A function that decides if a member is included in bounds or not. * @param memberFromIdentifier * A function that re-builds a member from its identifier. * @return A {@link SpatialIndex} tailored to the specified type */ private SpatialIndex newSpatialIndex( final BiFunction memberValidForBounds, final Function memberFromIdentifier) { return new PackedSpatialIndex(new RTree<>()) { private static final long serialVersionUID = 6569644967280192054L; @Override protected Long compress(final M item) { return item.getIdentifier(); } @Override protected boolean isValid(final M item, final Rectangle bounds) { return memberValidForBounds.apply(item, bounds); } @Override protected M restore(final Long packed) { return memberFromIdentifier.apply(packed); } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.io.Serializable; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.LongFunction; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.SnappedEdge; import org.openstreetmap.atlas.geography.atlas.items.SnappedLineItem; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.geojson.GeoJsonFeatureCollection; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.gson.JsonObject; /** * Atlas is a representation of an OpenStreetMap region in memory. It is a navigable collection of * unidirectional {@link Edge}s and {@link Node}s. It is designed to be close to the OpenStreetMap * model. It also contains a collection of non-navigable geolocated items that can be {@link Point} * s, {@link Line}s or {@link Area}s. All can be members of {@link Relation}s. * * @author matthieun * @author tony */ public interface Atlas extends Located, Iterable, Serializable, GeoJsonFeatureCollection { static Iterable entitiesMatchingId(final Long[] identifiers, final LongFunction function) { return Arrays.stream(identifiers).map(function::apply).filter(Objects::nonNull) .collect(Collectors.toSet()); } /** * @param identifier * The {@link Area}'s identifier * @return The {@link Area} that corresponds to the provided identifier */ Area area(long identifier); /** * @return All the {@link Area}s in this {@link Atlas} */ Iterable areas(); /** * A wrapper over {@link #area(long)} for multiple ids. * * @param identifiers * - The area identifiers to fetch. * @return The {@link Area}s that corresponds to the provided identifier. */ default Iterable areas(final Long... identifiers) { return entitiesMatchingId(identifiers, this::area); } /** * Return all the {@link Area}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Area}s matching the {@link Predicate}. */ Iterable areas(Predicate matcher); /** * Return all the {@link Area}s covering some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link Area}s covering the {@link Location}. */ Iterable areasCovering(Location location); /** * Return all the {@link Area}s matching a {@link Predicate} and covering some {@link Location}. * * @param matcher * The matcher to consider * @param location * The {@link Location} to consider * @return All the {@link Area}s matching the {@link Predicate} and covering the * {@link Location}. */ Iterable areasCovering(Location location, Predicate matcher); /** * Return all the {@link Area}s within and/or intersecting some surface. * * @param surface * The surface to consider * @return All the {@link Area}s within and/or intersecting the surface. */ Iterable areasIntersecting(GeometricSurface surface); /** * Return all the {@link Area}s matching a {@link Predicate}, and within and/or intersecting * some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Area}s matching the {@link Predicate}, and within and/or intersecting * the surface. */ Iterable areasIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link Area}s fully within some surface. * * @param surface * The surface to consider * @return All the {@link Area}s fully within the surface. */ Iterable areasWithin(GeometricSurface surface); /** * @param matcher * The matcher to consider * @return A {@link GeoJsonObject} that contains part the features in this {@link Atlas} which * matches the given matcher */ JsonObject asGeoJson(Predicate matcher); /** * Clone this {@link Atlas} to a {@link PackedAtlas}. Do not uses "clone" so as not to be * confused with the clone function in {@link Cloneable}, which this interface does not extend. * * @return The {@link PackedAtlas} copy of this {@link Atlas}. */ default PackedAtlas cloneToPackedAtlas() { return new PackedAtlasCloner().cloneFrom(this); } /** * @param identifier * The {@link Edge}'s identifier * @return The {@link Edge} that corresponds to the provided identifier */ Edge edge(long identifier); /** * @return All the {@link Edge}s in this {@link Atlas} */ Iterable edges(); /** * A wrapper over {@link #edge(long)} for multiple ids. * * @param identifiers * - The edge identifiers to fetch. * @return The {@link Edge}s that corresponds to the provided identifier. */ default Iterable edges(final Long... identifiers) { return entitiesMatchingId(identifiers, this::edge); } /** * Return all the {@link Edge}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Edge}s matching the {@link Predicate}. */ Iterable edges(Predicate matcher); /** * Return all the {@link Edge}s containing some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link Edge}s containing the {@link Location}. */ Iterable edgesContaining(Location location); /** * Return all the {@link Edge}s matching a {@link Predicate} and containing some * {@link Location}. * * @param matcher * The matcher to consider * @param location * The {@link Location} to consider * @return All the {@link Edge}s matching the {@link Predicate} and containing the * {@link Location}. */ Iterable edgesContaining(Location location, Predicate matcher); /** * Return all the {@link Edge}s within and/or intersecting some surface. * * @param surface * The surface to consider * @return All the {@link Edge}s within and/or intersecting the surface. */ Iterable edgesIntersecting(GeometricSurface surface); /** * Return all the {@link Edge}s matching a {@link Predicate}, and within and/or intersecting * some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Edge}s matching the {@link Predicate}, and within and/or intersecting * the surface. */ Iterable edgesIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link Edge}s fully within some surface. * * @param surface * The surface to consider * @return All the {@link Edge}s fully within the surface. */ Iterable edgesWithin(GeometricSurface surface); /** * Return all the {@link AtlasEntity}s * * @return All the {@link AtlasEntity}s */ Iterable entities(); /** * Return all the {@link AtlasEntity}s of a specific type * * @param type * The type to restrain to * @param memberClass * The class of the member * @param * The AtlasEntity type * @return All the {@link AtlasEntity}s of a specific type */ Iterable entities(ItemType type, Class memberClass); /** * Return all the {@link AtlasEntity}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link AtlasEntity}s matching the {@link Predicate}. */ Iterable entities(Predicate matcher); /** * Return all the {@link AtlasEntity}s within and/or intersecting some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link AtlasEntity}s within and/or intersecting the {@link GeometricSurface}. */ Iterable entitiesIntersecting(GeometricSurface surface); /** * Return all the {@link AtlasEntity}s matching a {@link Predicate}, and within and/or * intersecting some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link AtlasEntity}s matching the {@link Predicate}, and within and/or * intersecting the surface. */ Iterable entitiesIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link AtlasEntity}s fully within some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link AtlasEntity}s fully within the {@link GeometricSurface}. */ Iterable entitiesWithin(GeometricSurface surface); /** * Return all the {@link AtlasEntity}s matching a {@link Predicate}, and fully within some * surface. * * @param matcher * The matcher to consider * @param surface * The {@link GeometricSurface} to consider * @return All the {@link AtlasEntity}s fully within the {@link GeometricSurface}. */ Iterable entitiesWithin(GeometricSurface surface, Predicate matcher); /** * Get an entity from identifier and type * * @param identifier * the identifier * @param type * The type * @return The corresponding AtlasEntity, null if any. */ AtlasEntity entity(long identifier, ItemType type); /** * @return This Atlas' identifier */ UUID getIdentifier(); /** * @return This Atlas' optional name. If not specified, should return a String version of the * identifier. */ String getName(); /** * Return all the {@link AtlasItem}s * * @return All the {@link AtlasItem}s */ Iterable items(); /** * Return all the {@link AtlasItem}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link AtlasItem}s matching the {@link Predicate}. */ Iterable items(Predicate matcher); /** * Return all the {@link AtlasItem}s containing some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link AtlasItem}s containing the {@link Location}. */ Iterable itemsContaining(Location location); /** * Return all the {@link AtlasItem}s matching a {@link Predicate} and containing some * {@link Location}. * * @param matcher * The matcher to consider * @param location * The {@link Location} to consider * @return All the {@link AtlasItem}s matching the {@link Predicate} and containing the * {@link Location}. */ Iterable itemsContaining(Location location, Predicate matcher); /** * Return all the {@link AtlasItem}s within and/or intersecting some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link AtlasItem}s within and/or intersecting the {@link GeometricSurface}. */ Iterable itemsIntersecting(GeometricSurface surface); /** * Return all the {@link AtlasItem}s matching a {@link Predicate}, and within and/or * intersecting some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link AtlasItem}s matching the {@link Predicate}, and within and/or * intersecting the surface. */ Iterable itemsIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link AtlasItem}s fully within some surface. * * @param surface * The surface to consider * @return All the {@link AtlasItem}s fully within the surface. */ Iterable itemsWithin(GeometricSurface surface); /** * @param identifier * The {@link Line}'s identifier * @return The {@link Line} that corresponds to the provided identifier */ Line line(long identifier); /** * Return all the {@link LineItem}s * * @return All the {@link LineItem}s */ Iterable lineItems(); /** * Return all the {@link LineItem}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link LineItem}s matching the {@link Predicate}. */ Iterable lineItems(Predicate matcher); /** * Return all the {@link LineItem}s containing some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link LineItem}s containing the {@link Location}. */ Iterable lineItemsContaining(Location location); /** * Return all the {@link LineItem}s matching a {@link Predicate} and containing some * {@link Location}. * * @param matcher * The matcher to consider * @param location * The {@link Location} to consider * @return All the {@link LineItem}s matching the {@link Predicate} and containing the * {@link Location}. */ Iterable lineItemsContaining(Location location, Predicate matcher); /** * Return all the {@link LineItem}s within and/or intersecting some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link LineItem}s within and/or intersecting the {@link GeometricSurface}. */ Iterable lineItemsIntersecting(GeometricSurface surface); /** * Return all the {@link LineItem}s matching a {@link Predicate}, and within and/or intersecting * some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link LineItem}s matching the {@link Predicate}, and within and/or * intersecting the surface. */ Iterable lineItemsIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link LineItem}s fully within some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link LineItem}s within and/or intersecting the {@link GeometricSurface}. */ Iterable lineItemsWithin(GeometricSurface surface); /** * @return All the {@link Line}s in this {@link Atlas} */ Iterable lines(); /** * A wrapper over {@link #line(long)} for multiple ids. * * @param identifiers * - The line identifiers to fetch. * @return The {@link Line}s that corresponds to the provided identifier. */ default Iterable lines(final Long... identifiers) { return entitiesMatchingId(identifiers, this::line); } /** * Return all the {@link Line}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Line}s matching the {@link Predicate}. */ Iterable lines(Predicate matcher); /** * Return all the {@link Line}s containing some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link Line}s containing the location. */ Iterable linesContaining(Location location); /** * Return all the {@link Line}s matching a {@link Predicate} and containing some * {@link Location}. * * @param matcher * The matcher to consider * @param location * The {@link Location} to consider * @return All the {@link Line}s matching the {@link Predicate} and containing the * {@link Location}. */ Iterable linesContaining(Location location, Predicate matcher); /** * Return all the {@link Line}s within and/or intersecting some surface. * * @param surface * The surface to consider * @return All the {@link Line}s within and/or intersecting the surface. */ Iterable linesIntersecting(GeometricSurface surface); /** * Return all the {@link Line}s matching a {@link Predicate}, and within and/or intersecting * some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Line}s matching the {@link Predicate}, and within and/or intersecting * the surface. */ Iterable linesIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link Line}s fully within some surface. * * @param surface * The surface to consider * @return All the {@link Line}s fully within the surface. */ Iterable linesWithin(GeometricSurface surface); /** * Return all the {@link LocationItem}s * * @return All the {@link LocationItem}s */ Iterable locationItems(); /** * Return all the {@link LocationItem}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link LocationItem}s matching the {@link Predicate}. */ Iterable locationItems(Predicate matcher); /** * Return all the {@link LocationItem}s within and/or intersecting some surface. * * @param surface * The {@link GeometricSurface} to consider * @return All the {@link LocationItem}s within and/or intersecting the * {@link GeometricSurface}. */ Iterable locationItemsWithin(GeometricSurface surface); /** * Return all the {@link LocationItem}s matching a {@link Predicate}, and within and/or * intersecting some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link LocationItem}s matching the {@link Predicate}, and within and/or * intersecting the surface. */ Iterable locationItemsWithin(GeometricSurface surface, Predicate matcher); /** * @return The meta data for this {@link Atlas}. */ AtlasMetaData metaData(); /** * @param identifier * The {@link Node}'s identifier * @return The {@link Node} that corresponds to the provided identifier */ Node node(long identifier); /** * @return All the {@link Node}s in this Atlas */ Iterable nodes(); /** * A wrapper over {@link #node(long)} for multiple ids. * * @param identifiers * - The node identifiers to fetch. * @return The {@link Node}s that corresponds to the provided identifier. */ default Iterable nodes(final Long... identifiers) { return entitiesMatchingId(identifiers, this::node); } /** * Return all the {@link Node}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Node}s matching the {@link Predicate}. */ Iterable nodes(Predicate matcher); /** * Return all the {@link Node}s at some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link Node}s at the {@link Location}. */ Iterable nodesAt(Location location); /** * Return all the {@link Node}s within and/or intersecting some surface. Note: results may vary, * for an identical boundary, depending on the type, {@link Rectangle} or * {@link GeometricSurface} of the input. This is due to an underlying dependency on the awt * definition of insideness. * * @param surface * The surface to consider * @return All the {@link Node}s within and/or intersecting the surface. */ Iterable nodesWithin(GeometricSurface surface); /** * Return all the {@link Node}s matching a {@link Predicate}, and within and/or intersecting * some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Node}s matching the {@link Predicate}, and within and/or intersecting * the surface. */ Iterable nodesWithin(GeometricSurface surface, Predicate matcher); /** * @return The number of {@link Area}s */ long numberOfAreas(); /** * @return The number of {@link Edge}s */ long numberOfEdges(); /** * @return The number of {@link Line}s */ long numberOfLines(); /** * @return The number of {@link Node}s */ long numberOfNodes(); /** * @return The number of {@link Point}s */ long numberOfPoints(); /** * @return The number of {@link Relation}s */ long numberOfRelations(); /** * @param identifier * The {@link Point}'s identifier * @return The {@link Point} that corresponds to the provided identifier */ Point point(long identifier); /** * @return All the {@link Point}s in this Atlas */ Iterable points(); /** * A wrapper over {@link #point(long)} for multiple ids. * * @param identifiers * - The point identifiers to fetch. * @return The {@link Point}s that corresponds to the provided identifier. */ default Iterable points(final Long... identifiers) { return entitiesMatchingId(identifiers, this::point); } /** * Return all the {@link Point}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Point}s matching the {@link Predicate}. */ Iterable points(Predicate matcher); /** * Return all the {@link Point}s at some {@link Location}. * * @param location * The {@link Location} to consider * @return All the {@link Point}s at the {@link Location}. */ Iterable pointsAt(Location location); /** * Return all the {@link Point}s within some surface. Note: results may vary, for an identical * boundary, depending on the type, {@link Rectangle} or {@link GeometricSurface} of the input. * This is due to an underlying dependency on the awt definition of insideness. * * @param surface * The surface to consider * @return All the {@link Point}s within the surface. */ Iterable pointsWithin(GeometricSurface surface); /** * Return all the {@link Point}s matching a {@link Predicate}. Note: results may vary, for an * identical boundary, depending on the type, {@link Rectangle} or {@link GeometricSurface} of * the input. This is due to an underlying dependency on the awt definition of insideness. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Point}s matching the {@link Predicate}, and within and/or intersecting * the surface. */ Iterable pointsWithin(GeometricSurface surface, Predicate matcher); /** * @param identifier * The {@link Relation}'s identifier * @return The {@link Relation} that corresponds to the provided identifier */ Relation relation(long identifier); /** * @return All the {@link Relation}s in this Atlas */ Iterable relations(); /** * A wrapper over {@link #relation(long)} for multiple ids. * * @param identifiers * - The relation identifiers to fetch. * @return The {@link Relation}s that corresponds to the provided identifier. */ default Iterable relations(final Long... identifiers) { return entitiesMatchingId(identifiers, this::relation); } /** * Return all the {@link Relation}s matching a {@link Predicate}. * * @param matcher * The matcher to consider * @return All the {@link Relation}s matching the {@link Predicate}. */ Iterable relations(Predicate matcher); /** * @return All the {@link Relation}s in this Atlas, with the lower order relations first. This * means at any point, if a relation is returned by this {@link Iterable}, then all the * relations that belong to this relation have already been returned. */ Iterable relationsLowerOrderFirst(); /** * Return all the {@link Relation}s which have at least one feature intersecting some surface. * * @param surface * The surface to consider * @return All the {@link Relation}s which have at least one feature intersecting the surface. */ Iterable relationsWithEntitiesIntersecting(GeometricSurface surface); /** * Return all the {@link Relation}s which have at least one feature intersecting some surface. * * @param matcher * The matcher to consider * @param surface * The surface to consider * @return All the {@link Relation}s matching the {@link Predicate}, and which have at least one * feature intersecting the surface. */ Iterable relationsWithEntitiesIntersecting(GeometricSurface surface, Predicate matcher); /** * Return all the {@link Relation}s which have all features within some surface. * * @param surface * The surface to consider * @return All the {@link Relation}s which have all features within the surface. */ Iterable relationsWithEntitiesWithin(GeometricSurface surface); /** * Serialize this {@link Atlas} to a {@link WritableResource} * * @param writableResource * The resource to write to */ void save(WritableResource writableResource); /** * Save as GeoJSON * * @param resource * The resource to write to */ void saveAsGeoJson(WritableResource resource); /** * Save as GeoJSON with matcher * * @param resource * The resource to write to * @param matcher * The matcher to consider */ void saveAsGeoJson(WritableResource resource, Predicate matcher); /** * Save as line-delimited GeoJSON. This is one feature per line, with no wrapping * FeatureCollection. * * @param resource * The resource to write to * @param jsonMutator * The callback function that will let you change what is in the Feature's JSON. */ void saveAsLineDelimitedGeoJsonFeatures(WritableResource resource, BiConsumer jsonMutator); /** * Save as line-delimited GeoJSON with a matcher. This is one feature per line, with no wrapping * FeatureCollection. * * @param resource * The resource to write to * @param matcher * The matcher to consider * @param jsonMutator * The callback function that will let you change what is in the Feature's JSON. */ void saveAsLineDelimitedGeoJsonFeatures(WritableResource resource, Predicate matcher, BiConsumer jsonMutator); /** * Save as list of items * * @param resource * The resource to write to */ void saveAsList(WritableResource resource); /** * Save as a naive proto file * * @param resource * The resource to write to */ void saveAsProto(WritableResource resource); /** * Save as a text file * * @param resource * The resource to write to */ void saveAsText(WritableResource resource); /** * @return The size for this {@link Atlas}. */ default AtlasSize size() { return new AtlasSize(numberOfEdges(), numberOfNodes(), numberOfAreas(), numberOfLines(), numberOfPoints(), numberOfRelations()); } /** * @param point * A {@link Location} to snap * @param threshold * A {@link Distance} threshold to look for edges around the {@link Location} * @return The best snapped result, or null if there is no valid snap */ SnappedEdge snapped(Location point, Distance threshold); /** * @param point * A {@link Location} to snap * @param threshold * A {@link Distance} threshold to look for edges around the {@link Location} * @return A sorted {@link List} of all the candidate snaps. The list is empty if there are no * candidates. */ List snaps(Location point, Distance threshold); /** * @param point * A {@link Location} to snap * @param threshold * A {@link Distance} threshold to look for edges around the {@link Location} * @return A sorted {@link List} of all the candidate snaps. The list is empty if there are no * candidates. */ List snapsLineItem(Location point, Distance threshold); /** * Return a sub-atlas from this Atlas. * * @param boundary * The boundary within which the sub atlas will be built * @param cutType * The type of cut to perform * @return An optional sub-atlas. The optional will be empty in case there is nothing in the * {@link GeometricSurface} after the cut was applied. Returning an empty atlas is not * allowed. */ Optional subAtlas(GeometricSurface boundary, AtlasCutType cutType); /** * Return a sub-atlas from this Atlas. * * @param matcher * The matcher to consider * @param cutType * The type of cut to perform * @return An optional sub-atlas. The optional will be empty in case the matcher and cut-type * return an empty atlas, which is not allowed. */ Optional subAtlas(Predicate matcher, AtlasCutType cutType); /** * Get a summary of this {@link Atlas}. This string should be relatively compact, for e.g. just * the entity counts. * * @return A summary of this {@link Atlas}. */ String summary(); /** * Get a complete string representation of this {@link Atlas}. This string may include details * on all contained entities. * * @return a complete string representation of this {@link Atlas} */ String toStringDetailed(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/AtlasLoadingCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import com.google.common.collect.Iterables; /** * Any command that wants to load Atlas files based on certain criteria (ISO country codes, etc...) * can subclass this command and receive that behavior for free. Currently we only support the ISO * country codes with the -include-only flag * * @author cstaylor */ public abstract class AtlasLoadingCommand extends Command { /** * Lets us filter both Paths and Strings based on the ISO country names to be included or * excluded * * @author cstaylor */ public static class AcceptableInputFileFilter implements Predicate { /** * These are for extracting the ISO3 country code from the filename */ private static final int START_ISO3_NAME_INDEX = 0; private static final int END_ISO3_NAME_INDEX = 3; private final Set acceptableISOCodes = new HashSet<>(); private final Set excludedISOCodes = new HashSet<>(); public AcceptableInputFileFilter exclude(final Iterable exclude) { if (exclude != null) { Iterables.addAll(this.excludedISOCodes, exclude); } return this; } public AcceptableInputFileFilter include(final Iterable include) { if (include != null) { Iterables.addAll(this.acceptableISOCodes, include); } return this; } @Override public boolean test(final Resource fileName) { final String isoCode = fileName.getName().substring(START_ISO3_NAME_INDEX, END_ISO3_NAME_INDEX); if (this.acceptableISOCodes.size() > 0) { if (!this.acceptableISOCodes.contains(isoCode)) { return false; } } return this.excludedISOCodes.size() == 0 || !this.excludedISOCodes.contains(isoCode); } } protected static final Switch> INCLUDE_ONLY_THESE_COUNTRIES_PARAMETER = new Switch<>( "include-only", "list of comma-delimited ISO country codes that we'll include when searching folders for atlas files", possiblyMultipleISOs -> StringList.split(possiblyMultipleISOs, ",").stream() .collect(Collectors.toSet()), Optionality.OPTIONAL); protected static final Switch INPUT_FOLDER = new Switch<>("inputFolder", "Path of folder which contains Atlas files", File::new, Command.Optionality.OPTIONAL); protected static final Switch INPUT = new Switch<>("input", "Path of Atlas file", File::new, Command.Optionality.OPTIONAL); protected static final Switch> EXCLUDE_THESE_COUNTRIES_PARAMETER = new Switch<>( "exclude", "list of comma-delimited ISO country codes that we'll exclude when searching folders for atlas files", possiblyMultipleISOs -> StringList.split(possiblyMultipleISOs, ",").stream() .collect(Collectors.toSet()), Optionality.OPTIONAL); /** * Helper method for loading a MultiAtlas based on the criteria passed in through the command * line. * * @param commandMap * the list of command line arguments * @return a MultiAtlas containing the data from files who matched our search criteria */ protected Atlas loadAtlas(final CommandMap commandMap) { return loadAtlas(INPUT_FOLDER, commandMap); } /** * Helper method for loading a MultiAtlas based on the criteria passed in through the command * line. * * @param parameter * the command line parameter to use for loading the atlas files * @param commandMap * the list of command line arguments * @return a MultiAtlas containing the data from files who matched our search criteria */ @SuppressWarnings("unchecked") protected Atlas loadAtlas(final Switch parameter, final CommandMap commandMap) { final AcceptableInputFileFilter filter = new AcceptableInputFileFilter() .include((Set) commandMap.get(INCLUDE_ONLY_THESE_COUNTRIES_PARAMETER)) .exclude((Set) commandMap.get(EXCLUDE_THESE_COUNTRIES_PARAMETER)); final File input = (File) commandMap.get(INPUT); if (input != null) { return new AtlasResourceLoader().load(input); } final File inputFolder = (File) commandMap.get(parameter); if (inputFolder == null) { throw new CoreException("Switch missing: input file or input folder"); } return new AtlasResourceLoader().withResourceFilter(filter).loadRecursively(inputFolder); } @Override protected SwitchList switches() { return new SwitchList().with(INPUT, INPUT_FOLDER, INCLUDE_ONLY_THESE_COUNTRIES_PARAMETER, EXCLUDE_THESE_COUNTRIES_PARAMETER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/AtlasMetaData.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.geojson.GeoJsonProperties; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoAtlasMetaDataAdapter; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonObject; /** * Meta data for an {@link Atlas} * * @author matthieun * @author lcram */ public final class AtlasMetaData implements Serializable, Taggable, ProtoSerializable, GeoJsonProperties { public static final String EDGE_CONFIGURATION = "edgeConfiguration"; public static final String AREA_CONFIGURATION = "areaConfiguration"; public static final String WAY_SECTIONING_CONFIGURATION = "waySectioningConfiguration"; public static final String OSM_PBF_WAY_CONFIGURATION = "osmPbfWayConfiguration"; public static final String OSM_PBF_NODE_CONFIGURATION = "osmPbfNodeConfiguration"; public static final String OSM_PBF_RELATION_CONFIGURATION = "osmPbfRelationConfiguration"; /** Set to "true" if -keepAll was passed on the command line */ public static final String KEEP_ALL_CONFIGURATION = "keepAll"; private static final long serialVersionUID = -285346019736489425L; private static final String UNKNOWN_VALUE = "unknown"; private final AtlasSize size; private final boolean original; private final String codeVersion; private final String dataVersion; private final String country; private final String shardName; private final Map tags; public AtlasMetaData() { this(AtlasSize.DEFAULT); } public AtlasMetaData(final AtlasSize size) { this(size, true, UNKNOWN_VALUE, UNKNOWN_VALUE, UNKNOWN_VALUE, UNKNOWN_VALUE, Maps.hashMap()); } public AtlasMetaData(final AtlasSize size, final boolean original, final String codeVersion, final String dataVersion, final String country, final String shardName, final Map tags) { this.size = size; this.original = original; this.codeVersion = codeVersion; this.dataVersion = dataVersion; this.country = country; this.shardName = shardName; this.tags = tags; } public AtlasMetaData copyWithNewOriginal(final boolean original) { return new AtlasMetaData(this.size, original, this.codeVersion, this.dataVersion, this.country, this.shardName, this.tags); } public AtlasMetaData copyWithNewShardName(final String shardName) { return new AtlasMetaData(this.size, this.original, this.codeVersion, this.dataVersion, this.country, shardName, this.tags); } public AtlasMetaData copyWithNewSize(final AtlasSize size) { return new AtlasMetaData(size, this.original, this.codeVersion, this.dataVersion, this.country, this.shardName, this.tags); } /** * Copy this metadata with new tags * * @param tags * The tags to copy * @return The new AtlasMetaData to use */ public AtlasMetaData copyWithNewTags(final Map tags) { return new AtlasMetaData(this.size, this.original, this.codeVersion, this.dataVersion, this.country, this.shardName, tags); } @Override public boolean equals(final Object other) { if (other instanceof AtlasMetaData) { if (this == other) { return true; } final AtlasMetaData that = (AtlasMetaData) other; if (!Objects.equals(this.getSize(), that.getSize())) { return false; } if (this.isOriginal() != that.isOriginal()) { return false; } if (!Objects.equals(this.getCodeVersion(), that.getCodeVersion())) { return false; } if (!Objects.equals(this.getDataVersion(), that.getDataVersion())) { return false; } if (!Objects.equals(this.getCountry(), that.getCountry())) { return false; } if (!Objects.equals(this.getShardName(), that.getShardName())) { return false; } if (!Objects.equals(this.getTags(), that.getTags())) { return false; } return true; } return false; } public Optional getCodeVersion() { return Optional.ofNullable(this.codeVersion); } public Optional getCountry() { return Optional.ofNullable(this.country); } public Optional getDataVersion() { return Optional.ofNullable(this.dataVersion); } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = new JsonObject(); properties.add("size", this.getSize().getGeoJsonProperties()); properties.addProperty("original", this.isOriginal()); this.getCodeVersion() .ifPresent(versionString -> properties.addProperty("Code Version", versionString)); this.getDataVersion() .ifPresent(versionString -> properties.addProperty("Data Version", versionString)); this.getCountry() .ifPresent(countryString -> properties.addProperty("Country", countryString)); this.getShardName() .ifPresent(theShardName -> properties.addProperty("Shard Name", theShardName)); this.getTags().forEach((key, value) -> { if (!properties.has(key)) { properties.addProperty(key, value); } }); return properties; } @Override public ProtoAdapter getProtoAdapter() { return new ProtoAtlasMetaDataAdapter(); } public Optional getShardName() { return Optional.ofNullable(this.shardName); } public AtlasSize getSize() { return this.size; } @Override public Optional getTag(final String key) { return Optional.ofNullable(this.tags.get(key)); } @Override public Map getTags() { if (this.tags == null) { return new HashMap<>(); } return new HashMap<>(this.tags); } @Override public int hashCode() { final int sizeHash = this.getSize().hashCode(); return Objects.hash(Integer.valueOf(sizeHash), Boolean.valueOf(this.original), this.codeVersion, this.dataVersion, this.country, this.shardName, this.tags); } public boolean isOriginal() { return this.original; } public String toReadableString() { final StringBuilder builder = new StringBuilder(); builder.append("Size: "); builder.append("\n\tNodes: "); builder.append(this.size.getNodeNumber()); builder.append("\n\tEdges: "); builder.append(this.size.getEdgeNumber()); builder.append("\n\tAreas: "); builder.append(this.size.getAreaNumber()); builder.append("\n\tLines: "); builder.append(this.size.getLineNumber()); builder.append("\n\tPoints: "); builder.append(this.size.getPointNumber()); builder.append("\n\tRelations: "); builder.append(this.size.getRelationNumber()); builder.append("\n"); builder.append("Original: "); builder.append(this.original); builder.append("\n"); builder.append("Code Version: "); builder.append(this.codeVersion); builder.append("\n"); builder.append("Data Version: "); builder.append(this.dataVersion); builder.append("\n"); builder.append("Country: "); builder.append(this.country); builder.append("\n"); builder.append("Shard: "); builder.append(this.shardName); builder.append("\n"); builder.append("Tags:\n\t"); final SortedSet sortedTags = this.tags.entrySet().stream() .map(entry -> entry.getKey() + " -> " + entry.getValue()) .collect(Collectors.toCollection(TreeSet::new)); builder.append(new StringList(sortedTags).join("\n\t")); builder.append("\n"); return builder.toString(); } @Override public String toString() { return "[AtlasMetaData: size=" + this.size + ", original=" + this.original + ", codeVersion=" + this.codeVersion + ", dataVersion=" + this.dataVersion + ", country=" + this.country + ", shardName=" + this.shardName + ", tags=" + this.tags + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/AtlasResourceLoader.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.AbstractResource; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Load an {@link Atlas} from a {@link Resource} or an {@link Iterable} of {@link Resource}s. Also * supports loading based on a resource name filter. To recursively load all {@link Atlas}es in a * directory, see the {@link AtlasResourceLoader#loadRecursively} method. * * @author lcram */ public class AtlasResourceLoader { public static final Predicate HAS_TEXT_ATLAS_EXTENSION = FileSuffix .resourceFilter(FileSuffix.ATLAS, FileSuffix.TEXT) .or(FileSuffix.resourceFilter(FileSuffix.ATLAS, FileSuffix.TEXT, FileSuffix.GZIP)); public static final Predicate HAS_ATLAS_EXTENSION = FileSuffix .resourceFilter(FileSuffix.ATLAS) .or(FileSuffix.resourceFilter(FileSuffix.ATLAS, FileSuffix.GZIP)); private static final Logger logger = LoggerFactory.getLogger(AtlasResourceLoader.class); private static final Predicate CONTENTS_LOOK_LIKE_TEXT_ATLAS = resource -> { checkFileExistsAndIsNotDirectory(resource); setDecompressorFor(resource); return resource.firstLine().equals(TextAtlasBuilder.getNodesHeader()); }; private Predicate resourceFilter; private Predicate atlasEntityFilter; private String multiAtlasName; private static void checkFileExistsAndIsNotDirectory(final Resource resource) { if (resource instanceof File) { final File fileResource = (File) resource; if (!fileResource.exists()) { throw new CoreException("Resource {} was of type File but it could not be found", resource.getName()); } else if (fileResource.isDirectory()) { throw new CoreException( "Resource {} was of type File but it was a directory. Try loadRecursively instead.", resource.getName()); } } } private static void setDecompressorFor(final Resource resource) { if (FileSuffix.GZIP.matches(resource)) { if (resource instanceof AbstractResource) { ((AbstractResource) resource).setDecompressor(Decompressor.GZIP); } else { throw new CoreException( "Provide resource was of type {} which does not support decompression.", resource.getClass().getName()); } } } public AtlasResourceLoader() { this.resourceFilter = resource -> true; this.atlasEntityFilter = null; } /** * Load an {@link Atlas} from the provided {@link Resource}(s). If more than one * {@link Resource} is provided, the method will utilize the {@link MultiAtlas} to combine them. * This method will fail with an exception if any of the provided {@link Resource}s do not * contain a valid binary or text {@link Atlas}. This method should never return null. * * @param resources * the {@link Resource}(s) from which to load * @return the non-null loaded {@link Atlas} */ public Atlas load(final Resource... resources) { return load(Iterables.from(resources)); } /** * Load an {@link Atlas} from an {@link Iterable} of {@link Resource}s. If more than one * {@link Resource} is provided, the method will utilize the {@link MultiAtlas} to combine them. * This method will fail with an exception if any of the provided {@link Resource}s do not * contain a valid binary or text {@link Atlas}. This method should never return null. * * @param resources * the {@link Iterable} of {@link Resource}s from which to load * @return the non-null loaded {@link Atlas} */ public Atlas load(final Iterable resources) { final List atlasResources = Iterables.stream(resources) .flatMap(this::upcastAndRemoveNullResources).filter(this.resourceFilter) .collectToList(); final Optional resultAtlasOptional; if (atlasResources.isEmpty()) { throw new CoreException("No loadable Resources were found."); } else if (atlasResources.size() == 1) { resultAtlasOptional = loadAtlasResource(atlasResources.get(0)); } else { resultAtlasOptional = loadMultipleAtlasResources(atlasResources); } if (resultAtlasOptional.isEmpty()) { throw new CoreException( "Unable to load atlas from provided Resources. If you are seeing this you likely found a bug with AtlasResourceLoader. Please report it."); } // Apply the filter at the end return applyEntityFilter(resultAtlasOptional.get()); } /** * Load an {@link Atlas} from the provided {@link File} {@link Resource}(s). If any of the * provided {@link File}(s) are directories, the method will recursively descend into the * directory and include every {@link Atlas} it discovers. It identifies {@link Atlas}es by * looking for {@link FileSuffix#ATLAS} file extensions. Like with the * {@link AtlasResourceLoader#load} method, this method will utilize the {@link MultiAtlas} to * combine the {@link Atlas}es. This method should never return null. * * @param resources * the {@link File} {@link Resource}(s) from which to load * @return the non-null loaded {@link Atlas} */ public Atlas loadRecursively(final Resource... resources) { return loadRecursively(Iterables.from(resources)); } /** * Load an {@link Atlas} from an {@link Iterable} of {@link File} {@link Resource}s. If any of * the provided {@link File}(s) are directories, the method will recursively descend into the * directory and include every {@link Atlas} it discovers. It identifies {@link Atlas}es by * looking for {@link FileSuffix#ATLAS} file extensions. Like with the * {@link AtlasResourceLoader#load} method, this method will utilize the {@link MultiAtlas} to * combine the {@link Atlas}es. This method should never return null. * * @param resources * the {@link Iterable} of {@link File} {@link Resource}s from which to load * @return the non-null loaded {@link Atlas} */ public Atlas loadRecursively(final Iterable resources) { final List atlasResources = Iterables.stream(resources).filter(Objects::nonNull) .flatMap(this::expandFileOrDirectoryRecursively) .filter(HAS_ATLAS_EXTENSION.or(HAS_TEXT_ATLAS_EXTENSION)) .filter(this.resourceFilter).collectToList(); final Optional resultAtlasOptional = loadMultipleAtlasResources(atlasResources); if (!resultAtlasOptional.isPresent()) { throw new CoreException( "Unable to load atlas from provided Resources. If you are seeing this you likely found a bug with AtlasResourceLoader. Please report it."); } // Apply the filter at the end return applyEntityFilter(resultAtlasOptional.get()); } /** * This safe load method will never throw an exception. If any if the provided {@link Resource}s * cannot be loaded into an {@link Atlas}, it will simply return an empty {@link Optional}. * * @param resources * the {@link Resource}(s) from which to load * @return an {@link Optional} wrapping the loaded {@link Atlas} if present */ public Optional safeLoad(final Resource... resources) { return safeLoad(Iterables.from(resources)); } /** * This safe load method will never throw an exception. If any if the provided {@link Resource}s * cannot be loaded into an {@link Atlas}, it will simply return an empty {@link Optional}. * * @param resources * the {@link Iterable} of {@link Resource}(s) from which to load * @return an {@link Optional} wrapping the loaded {@link Atlas} if present */ public Optional safeLoad(final Iterable resources) { try { return Optional.of(load(resources)); } catch (final Exception exception) { logger.error("Could not load atlas from supplied resources", exception); return Optional.empty(); } } /** * This safe load method will never throw an exception. If any if the provided {@link Resource}s * cannot be loaded into an {@link Atlas}, it will simply return an empty {@link Optional}. See * the documentation for {@link AtlasResourceLoader#loadRecursively(Resource...)} for details on * how the recursive load works. * * @param resources * the {@link Iterable} of {@link Resource}(s) from which to load * @return an {@link Optional} wrapping the loaded {@link Atlas} if present */ public Optional safeLoadRecursively(final Resource... resources) { return safeLoadRecursively(Iterables.from(resources)); } /** * This safe load method will never throw an exception. If any if the provided {@link Resource}s * cannot be loaded into an {@link Atlas}, it will simply return an empty {@link Optional}. See * the documentation for {@link AtlasResourceLoader#loadRecursively(Resource...)} for details on * how the recursive load works. * * @param resources * the {@link Iterable} of {@link Resource}(s) from which to load * @return an {@link Optional} wrapping the loaded {@link Atlas} if present */ public Optional safeLoadRecursively(final Iterable resources) { try { return Optional.of(loadRecursively(resources)); } catch (final Exception exception) { logger.error("Could not load atlas from supplied resources", exception); return Optional.empty(); } } /** * Optionally add an {@link AtlasEntity} filter * * @param filter * filter which {@link AtlasEntity}s to include/exclude in the {@link Atlas} */ public void setAtlasEntityFilter(final Predicate filter) { this.atlasEntityFilter = filter; } /** * Optionally add a {@link Resource} filter * * @param filter * filter which {@link Resource} to load */ public void setResourceFilter(final Predicate filter) { this.resourceFilter = filter; } /** * Optionally add an {@link AtlasEntity} filter * * @param filter * filter which {@link AtlasEntity}s to include/exclude in the {@link Atlas} * @return fluent interface requires this be returned */ public AtlasResourceLoader withAtlasEntityFilter(final Predicate filter) { setAtlasEntityFilter(filter); return this; } /** * Set the name for the {@link MultiAtlas} that results from the load. * * @param multiAtlasName * the name * @return instance of {@link AtlasResourceLoader} for method chaining */ public AtlasResourceLoader withMultiAtlasName(final String multiAtlasName) { this.multiAtlasName = multiAtlasName; return this; } /** * Optionally add a {@link Resource} filter * * @param filter * filter which {@link Resource} to load * @return instance of {@link AtlasResourceLoader} for method chaining */ public AtlasResourceLoader withResourceFilter(final Predicate filter) { setResourceFilter(filter); return this; } private Atlas applyEntityFilter(final Atlas atlasToFilter) { if (this.atlasEntityFilter != null) { final Optional subAtlas = atlasToFilter.subAtlas(this.atlasEntityFilter, AtlasCutType.SOFT_CUT); return subAtlas.orElseThrow( () -> new CoreException("Entity filter resulted in an empty atlas")); } return atlasToFilter; } private List expandFileOrDirectoryRecursively(final Resource resource) { if (resource == null) { return new ArrayList<>(); } if (!(resource instanceof File)) { throw new CoreException("Resource {} was not a File, instead was {}", resource.getName(), resource.getClass().getName()); } final File file = (File) resource; final List result = new ArrayList<>(); if (file.isDirectory()) { file.listFilesRecursively().forEach(child -> { if (child.isGzipped()) { child.setDecompressor(Decompressor.GZIP); } result.add(child); }); } else { result.add(file); } return result; } private List filterForBinaryAtlasResources(final List atlasResources) { return atlasResources.stream().filter(CONTENTS_LOOK_LIKE_TEXT_ATLAS.negate()) .collect(Collectors.toList()); } private List filterForTextAtlasResources(final List atlasResources) { return atlasResources.stream().filter(CONTENTS_LOOK_LIKE_TEXT_ATLAS) .collect(Collectors.toList()); } private Optional loadAtlasResource(final Resource resource) { final Atlas result; if (resource instanceof File) { checkFileExistsAndIsNotDirectory(resource); } if (resource.length() == 0L) { throw new CoreException("{} {} had zero length!", resource.getClass().getName(), resource.getName()); } if (CONTENTS_LOOK_LIKE_TEXT_ATLAS.test(resource)) { setDecompressorFor(resource); result = new TextAtlasBuilder().read(resource); } else { try { result = PackedAtlas.load(resource); } catch (final Exception exception) { throw new CoreException("Failed to load an atlas from {} with name {}", resource.getClass().getName(), resource.getName(), exception); } } return Optional.ofNullable(result); } private Optional loadMultipleAtlasResources(final List atlasResources) { atlasResources.forEach(resource -> { if (resource instanceof File) { checkFileExistsAndIsNotDirectory(resource); } if (resource.length() == 0L) { throw new CoreException("{} {} had zero length!", resource.getClass().getName(), resource.getName()); } }); final List binaryResources = filterForBinaryAtlasResources(atlasResources); final List textResources = filterForTextAtlasResources(atlasResources); if (binaryResources.isEmpty() && textResources.isEmpty()) { throw new CoreException("No loadable Resources were found."); } /* * There are three scenarios that must be handled. 1) There were only binary atlases. 2) * There was a mix of binary and text atlases. 3) There were only text atlases. */ MultiAtlas resultAtlas = null; if (!binaryResources.isEmpty()) { resultAtlas = MultiAtlas.loadFromPackedAtlas(binaryResources); } if (!textResources.isEmpty()) { final List textAtlases = loadTextAtlases(textResources); if (!textAtlases.isEmpty()) { final MultiAtlas textMultiAtlas = new MultiAtlas(textAtlases); /* * In this case, 'resultAtlas' is not null because there was a mix of binary and * text atlases. */ if (resultAtlas != null) { resultAtlas = new MultiAtlas(resultAtlas, textMultiAtlas); } /* * For this case, there was no previous resultAtlas since no binary atlases were * found. */ else { resultAtlas = textMultiAtlas; } } } if (this.multiAtlasName != null && resultAtlas != null) { resultAtlas.setName(this.multiAtlasName); } return Optional.ofNullable(resultAtlas); } private List loadTextAtlases(final List textAtlasResources) { final List textAtlases = new ArrayList<>(); for (final Resource textResource : textAtlasResources) { setDecompressorFor(textResource); final Atlas atlas = new TextAtlasBuilder().read(textResource); textAtlases.add(atlas); } return textAtlases; } private List upcastAndRemoveNullResources(final Resource resource) { final List result = new ArrayList<>(); if (resource != null) { result.add(resource); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/BareAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.text.NumberFormat; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Snapper; import org.openstreetmap.atlas.geography.Snapper.SnappedLocation; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.atlas.items.SnappedEdge; import org.openstreetmap.atlas.geography.atlas.items.SnappedLineItem; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.atlas.sub.SubAtlasCreator; import org.openstreetmap.atlas.geography.geojson.GeoJsonFeatureCollection; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.proto.builder.ProtoAtlasBuilder; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.gson.JsonObject; /** * @author matthieun * @author tony * @author mgostintsev * @author hallahan */ public abstract class BareAtlas implements Atlas { public static final int MAXIMUM_RELATION_DEPTH = 500; private static final long serialVersionUID = 4733707438968864018L; private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); static { NUMBER_FORMAT.setGroupingUsed(true); } // Transient name private transient String name; private final UUID identifier; protected BareAtlas() { this.identifier = UUID.randomUUID(); } @Override public Iterable areas(final Predicate matcher) { return Iterables.filter(areas(), matcher); } @Override public JsonObject asGeoJson() { return GeoJsonUtils.featureCollection(this); } @Override public JsonObject asGeoJson(final Predicate matcher) { return GeoJsonUtils.featureCollection(new GeoJsonFeatureCollection() { @Override public Iterable getGeoJsonObjects() { return entities(matcher); } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = BareAtlas.this.getGeoJsonProperties(); properties.addProperty("Entity filter used", true); return properties; } }); } @Override public Iterable edges(final Predicate matcher) { return Iterables.filter(edges(), matcher); } @Override public Iterable entities() { return new MultiIterable<>(items(), relations()); } @Override @SuppressWarnings("unchecked") public Iterable entities(final ItemType type, final Class memberClass) { if (type.getMemberClass() != memberClass) { throw new CoreException("ItemType {} and class {} do not match!", type, memberClass.getSimpleName()); } switch (type) { case NODE: return (Iterable) nodes(); case EDGE: return (Iterable) edges(); case AREA: return (Iterable) areas(); case LINE: return (Iterable) lines(); case POINT: return (Iterable) points(); case RELATION: return (Iterable) relations(); default: throw new CoreException("ItemType {} unknown.", type); } } @Override public Iterable entities(final Predicate matcher) { return Iterables.filter(this, matcher); } @Override public Iterable entitiesIntersecting(final GeometricSurface surface) { return new MultiIterable<>(itemsIntersecting(surface), relationsWithEntitiesIntersecting(surface)); } @Override public Iterable entitiesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(entitiesIntersecting(surface), matcher); } @Override public Iterable entitiesWithin(final GeometricSurface surface) { return new MultiIterable<>(itemsWithin(surface), relationsWithEntitiesWithin(surface)); } @Override public Iterable entitiesWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(entitiesWithin(surface), matcher); } @Override public AtlasEntity entity(final long identifier, final ItemType type) { switch (type) { case NODE: return node(identifier); case EDGE: return edge(identifier); case AREA: return area(identifier); case LINE: return line(identifier); case POINT: return point(identifier); case RELATION: return relation(identifier); default: throw new CoreException("Unknown type {}", type); } } @Override public boolean equals(final Object other) { if (other instanceof Atlas) { if (this == other) { // Avoid comparing each item. return true; } final Atlas that = (Atlas) other; for (final AtlasEntity thisEntity : this) { final AtlasEntity thatEntity = that.entity(thisEntity.getIdentifier(), thisEntity.getType()); if (thatEntity == null || !thisEntity.getTags().equals(thatEntity.getTags())) { return false; } if (thisEntity instanceof Area) { final Polygon thisPolygon = ((Area) thisEntity).asPolygon(); final Polygon thatPolygon = ((Area) thatEntity).asPolygon(); if (!thisPolygon.equals(thatPolygon)) { return false; } } else if (thisEntity instanceof LineItem) { final PolyLine thisPolyLine = ((LineItem) thisEntity).asPolyLine(); final PolyLine thatPolyLine = ((LineItem) thatEntity).asPolyLine(); if (!thisPolyLine.equals(thatPolyLine)) { return false; } } else if (thisEntity instanceof LocationItem) { final Location thisLocation = ((LocationItem) thisEntity).getLocation(); final Location thatLocation = ((LocationItem) thatEntity).getLocation(); if (!thisLocation.equals(thatLocation)) { return false; } } else if (thisEntity instanceof Relation) { final RelationMemberList thisMembers = ((Relation) thisEntity).members(); final RelationMemberList thatMembers = ((Relation) thatEntity).members(); if (!thisMembers.equals(thatMembers)) { return false; } } else { throw new CoreException("Unknown type: {}", thisEntity.getClass().getName()); } } return true; } return false; } @Override public Iterable getGeoJsonObjects() { return entities(); } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = this.metaData().getGeoJsonProperties(); properties.addProperty("name", this.getName()); return properties; } @Override public UUID getIdentifier() { return this.identifier; } @Override public String getName() { if (this.name == null) { return String.valueOf(this.getIdentifier()); } else { return this.name; } } @Override public int hashCode() { return Long.hashCode(this.numberOfNodes() + this.numberOfEdges() + this.numberOfAreas() + this.numberOfLines() + this.numberOfPoints() + this.numberOfRelations()); } @Override public Iterable items() { return new MultiIterable<>(nodes(), edges(), areas(), lines(), points()); } @Override public Iterable items(final Predicate matcher) { return Iterables.filter(items(), matcher); } @Override public Iterable itemsContaining(final Location location) { return new MultiIterable<>(edgesContaining(location), nodesAt(location), areasCovering(location), linesContaining(location), pointsAt(location)); } @Override public Iterable itemsContaining(final Location location, final Predicate matcher) { return Iterables.filter(itemsContaining(location), matcher); } @Override public Iterable itemsIntersecting(final GeometricSurface surface) { return new MultiIterable<>(edgesIntersecting(surface), nodesWithin(surface), areasIntersecting(surface), linesIntersecting(surface), pointsWithin(surface)); } @Override public Iterable itemsIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(itemsIntersecting(surface), matcher); } @Override public Iterable itemsWithin(final GeometricSurface surface) { return new MultiIterable<>(locationItemsWithin(surface), lineItemsWithin(surface), areasWithin(surface)); } @Override public Iterator iterator() { return new MultiIterable(nodes(), edges(), areas(), lines(), points(), relations()).iterator(); } @Override public Iterable lineItems() { return new MultiIterable<>(edges(), lines()); } @Override public Iterable lineItems(final Predicate matcher) { return Iterables.filter(lineItems(), matcher); } @Override public Iterable lineItemsContaining(final Location location) { return new MultiIterable<>(edgesContaining(location), linesContaining(location)); } @Override public Iterable lineItemsContaining(final Location location, final Predicate matcher) { return Iterables.filter(lineItemsContaining(location), matcher); } @Override public Iterable lineItemsIntersecting(final GeometricSurface surface) { return new MultiIterable<>(edgesIntersecting(surface), linesIntersecting(surface)); } @Override public Iterable lineItemsIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(lineItemsIntersecting(surface), matcher); } @Override public Iterable lineItemsWithin(final GeometricSurface surface) { return new MultiIterable<>(edgesWithin(surface), linesWithin(surface)); } @Override public Iterable lines(final Predicate matcher) { return Iterables.filter(lines(), matcher); } @Override public Iterable locationItems() { return new MultiIterable<>(nodes(), points()); } @Override public Iterable locationItems(final Predicate matcher) { return Iterables.filter(locationItems(), matcher); } @Override public Iterable locationItemsWithin(final GeometricSurface surface) { return new MultiIterable<>(nodesWithin(surface), pointsWithin(surface)); } @Override public Iterable locationItemsWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(locationItemsWithin(surface), matcher); } @Override public Iterable nodes(final Predicate matcher) { return Iterables.filter(nodes(), matcher); } @Override public Iterable points(final Predicate matcher) { return Iterables.filter(points(), matcher); } @Override public Iterable relations(final Predicate matcher) { return Iterables.filter(relations(), matcher); } @Override public Iterable relationsLowerOrderFirst() { List stagedRelations = new ArrayList<>(); final Set result = new LinkedHashSet<>(); // First pass for (final Relation relation : relations()) { boolean stageable = false; final RelationMemberList members = relation.members(); for (final RelationMember member : members) { if (member.getEntity() instanceof Relation) { stageable = true; } } if (stageable) { stagedRelations.add(relation); } else { result.add(relation); } } // Second pass int depth = 0; while (!stagedRelations.isEmpty() && depth < MAXIMUM_RELATION_DEPTH) { final List newStagedRelations = new ArrayList<>(); for (final Relation relation : stagedRelations) { boolean stageable = false; final RelationMemberList members = relation.members(); for (final RelationMember member : members) { if (member.getEntity() instanceof Relation && !result.contains(member.getEntity())) { stageable = true; } } if (stageable) { newStagedRelations.add(relation); } else { result.add(relation); } } stagedRelations = newStagedRelations; depth++; } return result; } @Override public void saveAsGeoJson(final WritableResource resource) { saveAsGeoJson(resource, item -> true); } @Override public void saveAsGeoJson(final WritableResource resource, final Predicate matcher) { try (JsonWriter writer = new JsonWriter(resource)) { writer.write(this.asGeoJson(matcher)); } } @Override public void saveAsLineDelimitedGeoJsonFeatures(final WritableResource resource, final BiConsumer jsonMutator) { saveAsLineDelimitedGeoJsonFeatures(resource, item -> true, jsonMutator); } @Override public void saveAsLineDelimitedGeoJsonFeatures(final WritableResource resource, final Predicate matcher, final BiConsumer jsonMutator) { try (JsonWriter writer = new JsonWriter(resource)) { entities(matcher).forEach(entity -> { final JsonObject feature = entity.asGeoJson(); jsonMutator.accept(entity, feature); writer.writeLine(feature); }); } } @Override public void saveAsList(final WritableResource resource) { final BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(resource.write(), StandardCharsets.UTF_8)); try { writer.write(this.toString()); Streams.close(writer); } catch (final IOException e) { Streams.close(writer); throw new CoreException("Could not save atlas as list", e); } } @Override public void saveAsProto(final WritableResource resource) { new ProtoAtlasBuilder().write(this, resource); } @Override public void saveAsText(final WritableResource resource) { new TextAtlasBuilder().write(this, resource); } @Override public SnappedEdge snapped(final Location point, final Distance threshold) { SnappedEdge result = null; for (final Edge edge : this.edgesIntersecting(point.boxAround(threshold))) { final SnappedEdge candidate = new SnappedEdge(point.snapTo(edge.asPolyLine()), edge); if (result == null || candidate.getDistance().isLessThan(result.getDistance())) { result = candidate; } } return result; } @Override public List snaps(final Location point, final Distance threshold) { final List snaps = new ArrayList<>(); for (final Edge edge : this.edgesIntersecting(point.boxAround(threshold))) { final SnappedEdge candidate = new SnappedEdge(point.snapTo(edge.asPolyLine()), edge); snaps.add(candidate); } snaps.sort(SnappedLocation::compareTo); return snaps; } @Override public List snapsLineItem(final Location point, final Distance threshold) { final List snaps = new ArrayList<>(); for (final LineItem lineItem : this.lineItemsIntersecting(point.boxAround(threshold))) { final SnappedLineItem candidate = new SnappedLineItem( point.snapTo(lineItem.asPolyLine()), lineItem); snaps.add(candidate); } snaps.sort(Snapper.SnappedLocation::compareTo); return snaps; } @Override public Optional subAtlas(final GeometricSurface boundary, final AtlasCutType cutType) { switch (cutType) { case SILK_CUT: return SubAtlasCreator.silkCut(this, boundary); case SOFT_CUT: return SubAtlasCreator.softCut(this, boundary, false); case HARD_CUT_ALL: return SubAtlasCreator.hardCutAllEntities(this, boundary); case HARD_CUT_RELATIONS_ONLY: return SubAtlasCreator.softCut(this, boundary, true); default: throw new CoreException("Unsupported Atlas cut type: {}", cutType); } } @Override public Optional subAtlas(final Predicate matcher, final AtlasCutType cutType) { switch (cutType) { case SILK_CUT: return SubAtlasCreator.silkCut(this, matcher); case SOFT_CUT: return SubAtlasCreator.softCut(this, matcher); case HARD_CUT_ALL: return SubAtlasCreator.hardCutAllEntities(this, matcher); case HARD_CUT_RELATIONS_ONLY: return SubAtlasCreator.hardCutRelationsOnly(this, matcher); default: throw new CoreException("Unsupported Atlas cut type: {}", cutType); } } @Override public String summary() { final StringBuilder builder = new StringBuilder(); builder.append("["); builder.append(this.getClass().getSimpleName()); builder.append(": Nodes = "); builder.append(NUMBER_FORMAT.format(this.numberOfNodes())); builder.append(", Edges = "); builder.append(NUMBER_FORMAT.format(this.numberOfEdges())); builder.append(", Areas = "); builder.append(NUMBER_FORMAT.format(this.numberOfAreas())); builder.append(", Lines = "); builder.append(NUMBER_FORMAT.format(this.numberOfLines())); builder.append(", Points = "); builder.append(NUMBER_FORMAT.format(this.numberOfPoints())); builder.append(", Relations = "); builder.append(NUMBER_FORMAT.format(this.numberOfRelations())); builder.append("]"); return builder.toString(); } @Override public String toString() { return summary(); } @Override public String toStringDetailed() { final String newLineAfterFeature = ",\n\t\t"; final StringBuilder builder = new StringBuilder(); builder.append("[Atlas <"); builder.append(getName()); builder.append(">: "); final StringList list = new StringList(); list.add(Iterables.toString(this.nodes(), "Nodes", newLineAfterFeature)); list.add(Iterables.toString(this.edges(), "Edges", newLineAfterFeature)); list.add(Iterables.toString(this.areas(), "Areas", newLineAfterFeature)); list.add(Iterables.toString(this.lines(), "Lines", newLineAfterFeature)); list.add(Iterables.toString(this.points(), "Points", newLineAfterFeature)); list.add(Iterables.toString(this.relations(), "Relations", newLineAfterFeature)); builder.append(list.join(",\n\t")); builder.append("]"); return builder.toString(); } protected void setName(final String name) { this.name = name; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/Crawler.java ================================================ package org.openstreetmap.atlas.geography.atlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.slf4j.Logger; /** * Crawl map data, to flag issues for example. * * @author matthieun * @author mgostintsev */ public abstract class Crawler extends AtlasLoadingCommand { private static final Switch OUTPUT_FOLDER = new Switch<>("outputFolder", "Location of the output folder", File::new, Optionality.REQUIRED); private final Logger logger; public Crawler(final Logger logger) { this.logger = logger; } protected void initialize(final CommandMap command) { } @Override protected int onRun(final CommandMap command) { final File inputFolder = (File) command.get(INPUT_FOLDER); final String atlasName = inputFolder.getName(); final File outputFolder = (File) command.get(OUTPUT_FOLDER); initialize(command); if (inputFolder != null) { this.logger.info("Loading Atlas from {}", inputFolder); final Atlas atlas = loadAtlas(command); processAtlas(atlasName, atlas, outputFolder.getPathString()); } return 0; } protected abstract void processAtlas(String atlasName, Atlas atlas, String folder); @Override protected SwitchList switches() { return super.switches().with(OUTPUT_FOLDER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/README.md ================================================ # Using Atlas * Grab one or more Atlas files (from [here](https://www.dropbox.com/sh/54aqfbs12suqd9t/AACGxhjCaJiRcJUBuFh0iLiHa) or after [building them yourself](src/main/java/org/openstreetmap/atlas/geography/atlas/README.md#building-an-atlas-from-an-osmpbf-file)), and open them with [`AtlasResourceLoader`](src/main/java/org/openstreetmap/atlas/geography/atlas/AtlasResourceLoader.java): ``` final File atlasFile = new File("/path/to/source.atlas"); final Atlas atlas = new AtlasResourceLoader().load(atlasFile); ``` Query the data. Examples: * Get all edges within boundaries: ``` Rectangle rectangle = ...; atlas.edgesIntersecting(rectangle).forEach(edge -> ...); ``` * Find all the parks with less than 6 shape points: ``` Predicate filter = area -> { return Validators.isOfType(area, LeisureTag.class, LeisureTag.PARK) && area.asPolygon().size() < 6; } atlas.areas(filter).forEach(area -> ...); ``` or ``` Predicate filter = area -> { return "park".equals(area.getTags().get("leisure")) && area.asPolygon().size() < 6; } atlas.areas(filter).forEach(area -> ...); ``` * Find all buildings with a hole: ``` ComplexBuildingFinder finder = new ComplexBuildingFinder(); Iterables.stream(finder.find(atlas)) .filter(complexBuilding -> !complexBuilding.getOutline().inners().isEmpty()) .forEach(complexBuilding -> ...); ``` * How many `Edge`s are connected to a `Node`: ``` long identifier = 123; int numberOfConnectedEdges = atlas.nodeForIdentifier(identifier).connectedEdges().size(); ``` or ``` long identifier = 123; int numberOfConnectedEdges = atlas.nodeForIdentifier(identifier).absoluteValence(); ``` # Building an `Atlas` from an `.osm.pbf` file Building an `Atlas` from an `.osm.pbf` file involves multiple steps, described below. First create a "Raw Atlas" that is a simple copy of all the items in the PBF file into the Atlas format. Then (optionally) apply country slicing, and finally call the way-sectioning algorithm to create the "navigable network" part of the Atlas. Without country slicing: ``` final File pbfFile; final Atlas rawAtlas = new RawAtlasGenerator(pbfFile).build(); final Atlas atlas = new WaySectionProcessor(rawAtlas, AtlasLoadingOption.createOptionWithAllEnabled()).run(); ``` With country slicing: ``` final File pbfFile; final Set countries; final CountryBoundaryMap boundaries; final Atlas rawAtlas = new RawAtlasGenerator(pbfFile).build(); final Atlas slicedRawAtlas = new RawAtlasCountrySlicer(countries, boundaries).slice(rawAtlas); final Atlas atlas = new WaySectionProcessor(slicedRawAtlas, AtlasLoadingOption.createOptionWithAllEnabled(boundaries)).run(); ``` ## Way Sectioning OSM ways usually span multiple intersections in the case of roads. To make the road network a navigable network, the process of loading an `.osm.pbf` file runs "way sectioning". It will follow a set of rules to break ways at intersections, and create `Atlas` `Edge`s. For that, it pads the OSM feature identifiers with six digits starting from 1 to the number of sections. For example, way 123 would become `Edge`s 123000001, 123000002 and 123000003 if it has to be broken twice. If no sectioning takes place, the edge identifier would end in `000`. ## Country Slicing In case of building an Atlas that is hard-cut along a polygon (usually a country boundary with boundary=administrative and admin_level=2), the process of loading an `.osm.pbf` file runs "country slicing". All the features that span outside of the boundary will be cut at the boundary and excluded. All the features that are inside will be assigned a country code tag if a country code is given with the AtlasLoadingOption. The `AtlasLoadingOption` contains all the country boundaries, along with the country codes in a `CountryBoundaryMap`. All feature identifiers will be padded and the first 3 digits of the 6 digit padding (described above) will be a country counter. If a `Line` 123 spans 2 countries, gets out and comes back for example, it will ship with 123001000 and 123001002 within the first country, and 123002001 in the country where it spans out (in a separate `Atlas`). ## Configuration Way-sectioning logic, edge definition and which pbf entities (Way, Node, Relation) are brought into an `Atlas` are all configurable. The default configurations can be found in the main resources directory as json files (see atlas-way-section.json for an example of the way section configuration). These configurations are initialized and can be set in `AtlasLoadingOption`. # Building an `Atlas` from scratch The `PackedAtlasBuilder` is here for that. It ensures that all the data that makes its way to an Atlas is consistent (for example making sure that if an `Edge` says its start `Node` is 123, then `Node` 123 really exists) and that an Atlas is final and cannot be modified once it has been accessed once. * First add all the `Node`s * Then add all the `Edge`s, `Area`s, `Line`s and `Point`s in any order. * Finally add all the `Relation`s from the lowest order (no other `Relation` is within its members) to the higher order (other `Relation`s are within its members). The `PackedAtlasBuilder` will throw an exception if a `Relation` is added and any of the listed members have not already been added. # Saving an `Atlas` The `Atlas` API offers a `save(WritableResource)` method, that is implemented by `PackedAtlas`. Trying to save a `MultiAtlas` will result in an exception suggesting to copy the `Atlas` to a `PackedAtlas` first. # Copying an `Atlas` From any `Atlas`, a `PackedAtlas` can be created and saved to a `WritableResource` (A File for example). This is done with the `PackedAtlasCloner`: ``` final Atlas atlas1; final Atlas atlas2; new PackedAtlasCloner().cloneFrom(new MultiAtlas(atlas1, atlas2)).save(new File("/path/to/file.atlas")); ``` # Filtering an `Atlas` Atlas objects can be soft-filtered based on a `Predicate` or a `Polygon`. ``` final Atlas atlas; final Predicate predicate; final Atlas predicateAtlas = atlas.subAtlas(predicate, AtlasCutType.SOFT_CUT); final Polygon polygon; final Atlas polygonAtlas = atlas.subAtlas(polygon, AtlasCutType.SOFT_CUT); ``` ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/ShardFileOverlapsPolygon.java ================================================ package org.openstreetmap.atlas.geography.atlas; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.sharding.DynamicTileSharding; import org.openstreetmap.atlas.streaming.resource.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Predicate that uses a sharding tree to determine whether a given atlas shard file overlaps a * given Polygon. By default it depends on shard files following the naming convention of * [name]_[zoom]-[x]-[y].atlas.gz, where the .gz extension is optional. For example, * XYZ_9-272-162.atlas.gz and XYZ_9-272-162.atlas are valid name formats. The shard filename pattern * can be overridden to work with other naming conventions as long as the [zoom]-[x]-[y] portion of * the name still exists as the first group in the pattern. * * @author rmegraw */ public class ShardFileOverlapsPolygon implements Predicate { private static final Logger logger = LoggerFactory.getLogger(ShardFileOverlapsPolygon.class); /** * Matches shard filenames such as XYZ_9-272-162.atlas.gz and XYZ_9-272-162.atlas */ public static final String DEFAULT_SHARD_FILE_REGEX = "^.+_(\\d{1,2}-\\d+-\\d+)\\.atlas(\\.gz)?$"; private final Pattern shardFilePattern; private final Set shardsOverlappingPolygon; /** * @param shardingTree * Sharding tree * @param bounds * Polygon over which shard file overlap is tested */ public ShardFileOverlapsPolygon(final DynamicTileSharding shardingTree, final Polygon bounds) { this(shardingTree, bounds, DEFAULT_SHARD_FILE_REGEX); } /** * @param shardingTree * Sharding tree * @param bounds * Polygon over which shard file overlap is tested * @param shardFileRegex * Regex which must extract [zoom]-[x]-[y] portion of shard filename as the first * group (see default regex for example) */ public ShardFileOverlapsPolygon(final DynamicTileSharding shardingTree, final Polygon bounds, final String shardFileRegex) { this.shardFilePattern = Pattern.compile(shardFileRegex); this.shardsOverlappingPolygon = new HashSet<>(); shardingTree.shards(bounds) .forEach(shard -> this.shardsOverlappingPolygon.add(shard.getName())); } @Override public boolean test(final Resource resource) { boolean result = false; final String resourceName = resource.getName(); if (resourceName != null) { final Matcher matcher = this.shardFilePattern.matcher(resourceName); if (matcher.find()) { final String shardName = matcher.group(1); if (this.shardsOverlappingPolygon.contains(shardName)) { logger.debug("Resource {} overlaps polygon.", resourceName); result = true; } else { logger.debug("Resource {} does not overlap polygon.", resourceName); } } else { logger.debug("Resource {} does not match shard filename pattern.", resourceName); } } else { logger.debug("Resource {} name is null.", resource.toString()); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/AtlasBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Build an {@link Atlas} from {@link Node} and {@link Edge} data. * * @author matthieun */ public interface AtlasBuilder { /** * Add a {@link Area} to the {@link Atlas}. * * @param identifier * The {@link Area}'s identifier. * @param geometry * The geometry of the {@link Area} * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Area} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addArea(long identifier, Polygon geometry, Map tags); /** * Add an {@link Edge} to the {@link Atlas}. Its start and end {@link Node} should have been * added already when this is called. * * @param identifier * The {@link Edge}'s identifier. * @param geometry * The geometry of the {@link Edge} * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Edge} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addEdge(long identifier, PolyLine geometry, Map tags); /** * Add a {@link Line} to the {@link Atlas}. * * @param identifier * The {@link Line}'s identifier. * @param geometry * The geometry of the {@link Line} * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Line} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addLine(long identifier, PolyLine geometry, Map tags); /** * Add a {@link Node} to the {@link Atlas} * * @param identifier * The {@link Node}'s identifier * @param geometry * The {@link Node}'s {@link Location} * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Node} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addNode(long identifier, Location geometry, Map tags); /** * Add a {@link Point} to the {@link Atlas}. * * @param identifier * The {@link Point}'s identifier. * @param geometry * The geometry of the {@link Point} * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Point} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addPoint(long identifier, Location geometry, Map tags); /** * Add a {@link Relation} to the {@link Atlas}. * * @param identifier * The {@link Relation}'s identifier. * @param osmIdentifier * The {@link Relation}'s OSM identifier for split relations. The same identifier * otherwise. * @param structure * The structure of the {@link Relation}. This cannot be empty! * @param tags * An arbitrary set of OSM key-value pairs that are attached to this {@link Relation} * @throws IllegalAccessError * In case the {@link Atlas} has already been generated and this builder is locked. */ void addRelation(long identifier, long osmIdentifier, RelationBean structure, Map tags); /** * @return The {@link Atlas} comprising all the {@link Edge}s, {@link Node}s, {@link Area}s, * {@link Line}s, and {@link Point}s added using this builder. Once this is called, the * addEdge or addNode methods should throw an exception. */ Atlas get(); /** * Give the meta data of the {@link Atlas} to be created. * * @param metaData * The meta data */ void setMetaData(AtlasMetaData metaData); /** * Give an estimate of the size of the Atlas. * * @param estimates * The estimates of the size of the Atlas */ void setSizeEstimates(AtlasSize estimates); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/AtlasSize.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder; import java.io.Serializable; import java.util.Iterator; import java.util.Objects; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.geojson.GeoJsonProperties; import com.google.gson.JsonObject; /** * Size estimates for an {@link AtlasBuilder} * * @author matthieun */ public class AtlasSize implements Serializable, GeoJsonProperties { /** * A simple builder class for creating {@link AtlasSize} objects with custom sizes. * * @author lcram */ public static class AtlasSizeBuilder { private long edgeEstimate; private long nodeEstimate; private long areaEstimate; private long lineEstimate; private long pointEstimate; private long relationEstimate; public AtlasSizeBuilder() { this.edgeEstimate = DEFAULT_ESTIMATE; this.nodeEstimate = DEFAULT_ESTIMATE; this.areaEstimate = DEFAULT_ESTIMATE; this.lineEstimate = DEFAULT_ESTIMATE; this.pointEstimate = DEFAULT_ESTIMATE; this.relationEstimate = DEFAULT_ESTIMATE; } /** * Builds an {@link AtlasSize}. By default it uses {@link AtlasSize#DEFAULT_ESTIMATE} for * the size estimates. * * @return A new {@link AtlasSize} */ public AtlasSize build() { return new AtlasSize(this.edgeEstimate, this.nodeEstimate, this.areaEstimate, this.lineEstimate, this.pointEstimate, this.relationEstimate); } public AtlasSizeBuilder withAreaEstimate(final long areaNumber) { this.areaEstimate = areaNumber; return this; } public AtlasSizeBuilder withEdgeEstimate(final long edgeNumber) { this.edgeEstimate = edgeNumber; return this; } public AtlasSizeBuilder withLineEstimate(final long lineNumber) { this.lineEstimate = lineNumber; return this; } public AtlasSizeBuilder withNodeEstimate(final long nodeNumber) { this.nodeEstimate = nodeNumber; return this; } public AtlasSizeBuilder withPointEstimate(final long pointNumber) { this.pointEstimate = pointNumber; return this; } public AtlasSizeBuilder withRelationEstimate(final long relationNumber) { this.relationEstimate = relationNumber; return this; } } private static final long serialVersionUID = -4365680097735345765L; private static final long DEFAULT_ESTIMATE = 1024L; public static final AtlasSize DEFAULT = new AtlasSize(DEFAULT_ESTIMATE, DEFAULT_ESTIMATE, DEFAULT_ESTIMATE, DEFAULT_ESTIMATE, DEFAULT_ESTIMATE, DEFAULT_ESTIMATE); private final long edgeNumber; private final long nodeNumber; private final long areaNumber; private final long lineNumber; private final long pointNumber; private final long relationNumber; /** * Constructor that calculates the number of occurrences for each {@link AtlasEntity}. * * @param entities * The {@link AtlasEntity}s to use for generating an {@link AtlasSize} */ public AtlasSize(final Iterable entities) { long nodeNumber = 0L; long edgeNumber = 0L; long areaNumber = 0L; long lineNumber = 0L; long pointNumber = 0L; long relationNumber = 0L; final Iterator entityIterator = entities.iterator(); while (entityIterator.hasNext()) { final AtlasEntity entity = entityIterator.next(); final ItemType type = entity.getType(); switch (type) { case NODE: nodeNumber++; break; case EDGE: edgeNumber++; break; case AREA: areaNumber++; break; case LINE: lineNumber++; break; case POINT: pointNumber++; break; case RELATION: relationNumber++; break; default: throw new CoreException("Invalid Item Type {}", type); } } this.edgeNumber = edgeNumber; this.nodeNumber = nodeNumber; this.areaNumber = areaNumber; this.lineNumber = lineNumber; this.pointNumber = pointNumber; this.relationNumber = relationNumber; } /** * Default constructor that takes explicit number of occurrences of each {@link AtlasEntity}. * * @param edgeNumber * Number of {@link Edge}s * @param nodeNumber * Number of {@link Node}s * @param areaNumber * Number of {@link Area}s * @param lineNumber * Number of {@link Line}s * @param pointNumber * Number of {@link Point}s * @param relationNumber * Number of {@link Relation}s */ public AtlasSize(final long edgeNumber, final long nodeNumber, final long areaNumber, final long lineNumber, final long pointNumber, final long relationNumber) { this.edgeNumber = edgeNumber; this.nodeNumber = nodeNumber; this.areaNumber = areaNumber; this.lineNumber = lineNumber; this.pointNumber = pointNumber; this.relationNumber = relationNumber; } @Override public boolean equals(final Object other) { if (other instanceof AtlasSize) { if (this == other) { return true; } final AtlasSize that = (AtlasSize) other; if (this.getEdgeNumber() != that.getEdgeNumber()) { return false; } if (this.getNodeNumber() != that.getNodeNumber()) { return false; } if (this.getAreaNumber() != that.getAreaNumber()) { return false; } if (this.getLineNumber() != that.getLineNumber()) { return false; } if (this.getPointNumber() != that.getPointNumber()) { return false; } if (this.getRelationNumber() != that.getRelationNumber()) { return false; } return true; } return false; } public long getAreaNumber() { return this.areaNumber; } public long getEdgeNumber() { return this.edgeNumber; } public long getEntityNumber() { return this.nodeNumber + this.edgeNumber + this.pointNumber + this.lineNumber + this.areaNumber + this.relationNumber; } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = new JsonObject(); properties.addProperty("Number of Edges", this.getEdgeNumber()); properties.addProperty("Number of Nodes", this.getNodeNumber()); properties.addProperty("Number of Areas", this.getAreaNumber()); properties.addProperty("Number of Lines", this.getLineNumber()); properties.addProperty("Number of Points", this.getPointNumber()); properties.addProperty("Number of Relations", this.getRelationNumber()); return properties; } public long getLineNumber() { return this.lineNumber; } public long getNodeNumber() { return this.nodeNumber; } public long getNonRelationEntityNumber() { return this.nodeNumber + this.edgeNumber + this.pointNumber + this.lineNumber + this.areaNumber; } public long getPointNumber() { return this.pointNumber; } public long getRelationNumber() { return this.relationNumber; } @Override public int hashCode() { return Objects.hash(Long.valueOf(this.edgeNumber), Long.valueOf(this.nodeNumber), Long.valueOf(this.areaNumber), Long.valueOf(this.lineNumber), Long.valueOf(this.pointNumber), Long.valueOf(this.relationNumber)); } @Override public String toString() { return "[AtlasSize: edgeNumber=" + this.edgeNumber + ", nodeNumber=" + this.nodeNumber + ", areaNumber=" + this.areaNumber + ", lineNumber=" + this.lineNumber + ", pointNumber=" + this.pointNumber + ", relationNumber=" + this.relationNumber + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/GeoJsonAtlasBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.streaming.readers.GeoJsonReader; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.tags.oneway.OneWayTag; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonElement; /** * Create an Atlas from an non-way-sectioned overpass-turbo GeoJson resource. * * @author matthieun */ public class GeoJsonAtlasBuilder { /** * @author matthieun */ private static class GeoJsonEdge { private final long identifier; private final Map tags; private final PolyLine polyLine; GeoJsonEdge(final long identifier, final Map tags, final PolyLine polyLine) { this.identifier = identifier; this.tags = tags; this.polyLine = polyLine; } public long getIdentifier() { return this.identifier; } public PolyLine getPolyLine() { return this.polyLine; } public Map getTags() { return this.tags; } } public Atlas create(final Resource geoJson) { GeoJsonReader reader = new GeoJsonReader(geoJson); final AtlasBuilder builder = new PackedAtlasBuilder(); final List edges = new ArrayList<>(); long nodeIdentifier = 0L; reader.forEachRemaining(item -> { if (item.getItem() instanceof PolyLine && !(item.getItem() instanceof Polygon)) { // We have an edge Long identifier = null; final Set> jsonTags = item.getProperties() .entrySet(); final Map tags = new HashMap<>(); for (final Map.Entry entry : jsonTags) { final String key = entry.getKey(); final String value = entry.getValue().getAsString(); if (key.contains("@id")) { identifier = Long.valueOf(StringList.split(value, "/").get(1)); } else { tags.put(key, value); } } if (!tags.containsKey("highway")) { // it was not an edge after all return; } edges.add(new GeoJsonEdge(identifier, tags, (PolyLine) item.getItem())); } }); final Set locations = new HashSet<>(); for (final GeoJsonEdge edge : edges) { locations.add(edge.getPolyLine().first()); locations.add(edge.getPolyLine().last()); } for (final Location location : locations) { // Node builder.addNode(nodeIdentifier++, location, new HashMap()); } for (final GeoJsonEdge edge : edges) { // Edge if (edge.getTags().containsKey(OneWayTag.KEY) && !OneWayTag.NO.name().equalsIgnoreCase(edge.getTags().get(OneWayTag.KEY))) { final String onewayTag = edge.getTags().get(OneWayTag.KEY); if (OneWayTag.YES.name().equalsIgnoreCase(onewayTag) || "1".equals(onewayTag)) { builder.addEdge(edge.getIdentifier(), edge.getPolyLine(), edge.getTags()); } else if ("-1".equals(onewayTag)) { builder.addEdge(edge.getIdentifier(), edge.getPolyLine().reversed(), edge.getTags()); } } else { builder.addEdge(edge.getIdentifier(), edge.getPolyLine(), edge.getTags()); builder.addEdge(-edge.getIdentifier(), edge.getPolyLine().reversed(), edge.getTags()); } } reader = new GeoJsonReader(geoJson); reader.forEachRemaining(item -> { if (item.getItem() instanceof Polygon) { // Area Long identifier = null; final Set> jsonTags = item.getProperties() .entrySet(); final Map tags = new HashMap<>(); for (final Map.Entry entry : jsonTags) { final String key = entry.getKey(); final String value = entry.getValue().getAsString(); if (key.contains("@id")) { identifier = Long.valueOf(StringList.split(value, "/").get(1)); } else { tags.put(key, value); } } builder.addArea(identifier, (Polygon) item.getItem(), tags); } if (item.getItem() instanceof PolyLine && !(item.getItem() instanceof Polygon)) { // Line Long identifier = null; final Set> jsonTags = item.getProperties() .entrySet(); final Map tags = new HashMap<>(); for (final Map.Entry entry : jsonTags) { final String key = entry.getKey(); final String value = entry.getValue().getAsString(); if (key.contains("@id")) { identifier = Long.valueOf(StringList.split(value, "/").get(1)); } else { tags.put(key, value); } } if (tags.containsKey("highway")) { // it was an edge after all return; } builder.addLine(identifier, (PolyLine) item.getItem(), tags); } if (item.getItem() instanceof Location) { // Area Long identifier = null; final Set> jsonTags = item.getProperties() .entrySet(); final Map tags = new HashMap<>(); for (final Map.Entry entry : jsonTags) { final String key = entry.getKey(); final String value = entry.getValue().getAsString(); if (key.contains("@id")) { identifier = Long.valueOf(StringList.split(value, "/").get(1)); } else { tags.put(key, value); } } try { builder.addPoint(identifier, (Location) item.getItem(), tags); } catch (final CoreException e) { if (!tags.isEmpty()) { throw e; } // ignore. It is a duplicated node in GeoJson without any tags } } }); return builder.get(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/RelationBean.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder; import java.io.Serializable; import java.util.AbstractCollection; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedRelation; /** * @author matthieun */ public class RelationBean extends AbstractCollection implements Serializable { /** * @author matthieun */ public static class RelationBeanItem implements Serializable, Comparable { private static final long serialVersionUID = 441160361936498695L; private final Long identifier; private final String role; private final ItemType type; public RelationBeanItem(final Long identifier, final String role, final ItemType type) { this.identifier = identifier; this.role = role; this.type = type; } public RelationBeanItem(final RelationBeanItem item) { this.identifier = item.identifier; this.role = item.role; this.type = item.type; } @Override public int compareTo(final RelationBeanItem other) { int result = this.getType().compareTo(other.getType()); if (result == 0) { result = Long.compare(this.getIdentifier(), other.getIdentifier()); } if (result == 0) { result = this.getRole().compareTo(other.getRole()); } return result; } @Override public boolean equals(final Object other) { if (other instanceof RelationBeanItem) { final RelationBeanItem that = (RelationBeanItem) other; return this.getIdentifier().equals(that.getIdentifier()) && this.getRole().equals(that.getRole()) && this.getType() == that.getType(); } return false; } public Long getIdentifier() { return this.identifier; } public String getRole() { return this.role; } public ItemType getType() { return this.type; } @Override public int hashCode() { return Objects.hash(this.identifier, this.role, this.type); } @Override public String toString() { return "[" + this.type + ", " + this.identifier + ", " + this.role + "]"; } } private static final long serialVersionUID = 8511830231633569713L; private List beanItems; /** * This set has no concept of how many {@link RelationBeanItem}s of a given value have been * removed. Technically, OSM allows for duplicate {@link RelationBeanItem}s in a given relation. * However, these duplicates are disallowed by {@link PackedAtlas#relationMembers} and by * extension {@link PackedRelation#members}. As a result, we need not worry about that edge case * here. */ private final Set explicitlyExcluded; public static RelationBean fromSet(final Set set) { final RelationBean bean = new RelationBean(); for (final RelationBeanItem item : set) { bean.addItem(item); } return bean; } public static RelationBean mergeBeans(final RelationBean left, final RelationBean right) { final RelationBean result = new RelationBean(); for (final RelationBeanItem leftItem : left) { if (!right.isExplicitlyExcluded(leftItem)) { result.addItem(leftItem); } } for (final RelationBeanItem rightItem : right) { final Optional existingLeftItem = left .getItemFor(rightItem.getIdentifier(), rightItem.getType()); if (existingLeftItem.isPresent() && existingLeftItem.get().getRole().equals(rightItem.getRole())) { // Role already exists, continue. continue; } if (!left.isExplicitlyExcluded(rightItem)) { result.addItem(rightItem); } } left.explicitlyExcluded.forEach(result::addItemExplicitlyExcluded); right.explicitlyExcluded.forEach(result::addItemExplicitlyExcluded); return result; } public RelationBean() { this.beanItems = new ArrayList<>(); this.explicitlyExcluded = new HashSet<>(); } @Override public boolean add(final RelationBeanItem item) { this.addItem(item); return true; } public void addItem(final Long identifier, final String role, final ItemType itemType) { this.beanItems.add(new RelationBeanItem(identifier, role, itemType)); } public void addItem(final RelationBeanItem item) { addItem(item.getIdentifier(), item.getRole(), item.getType()); } public void addItemExplicitlyExcluded(final Long identifier, final String role, final ItemType itemType) { addItemExplicitlyExcluded(new RelationBeanItem(identifier, role, itemType)); } public void addItemExplicitlyExcluded(final RelationBeanItem item) { this.explicitlyExcluded.add(item); } /** * Get this {@link RelationBean} as a {@link List} of its {@link RelationBeanItem}s. * * @return the item list representing this bean */ public List asList() { return new ArrayList<>(this.beanItems); } /** * Get this {@link RelationBean} as a {@link Map} from its constituent {@link RelationBeanItem}s * to their counts (Here, counts refers to the number of times a given {@link RelationBeanItem} * appears in the bean. While abnormal, duplicate bean items are technically allowed by OSM). * This method is useful for comparing the equality of two {@link RelationBean}s, since the map * representation intrinsically ignores the internal ordering of the constituent * {@link RelationBeanItem}s (this ordering is irrelevant as far as equality is concerned). * * @return the item map representing this bean */ public Map asMap() { final Map result = new HashMap<>(); for (final RelationBeanItem beanItem : this) { if (result.containsKey(beanItem)) { int count = result.get(beanItem); count += 1; result.put(beanItem, count); } else { result.put(beanItem, 1); } } return result; } /** * Get this {@link RelationBean} as a {@link Set} from its constituent * {@link RelationBeanItem}s. This will ignore the fact that a {@link RelationBean} can * technically contain duplicate items. * * @return the set representing this bean */ public Set asSet() { return new HashSet<>(this.asMap().keySet()); } /** * Get this {@link RelationBean} as a sorted {@link List} of its {@link RelationBeanItem}s. * * @return the item list representing this bean */ public List asSortedList() { final List result = asList(); Collections.sort(result); return result; } /** * Get this {@link RelationBean} as a {@link SortedSet} from its constituent * {@link RelationBeanItem}s. This will ignore the fact that a {@link RelationBean} can * technically contain duplicate items. * * @return the set representing this bean */ public SortedSet asSortedSet() { return new TreeSet<>(this.asMap().keySet()); } /** * Check if the two beans are the same, without looking at the bean order. * * @param other * The other object * @return True if the other object is a {@link RelationBean} and is equal regardless of order. */ @Override public boolean equals(final Object other) { if (other instanceof RelationBean) { final RelationBean that = (RelationBean) other; return this.asMap().equals(that.asMap()); } return false; } /** * Check if the two beans are the same, without looking at the bean order. Also, ensure that * their explicitlyExcluded sets match. * * @param other * The other object * @return True if the other object satisfies {@link RelationBean#equals(Object)} AND has a * matching explicitlyExcluded set. */ public boolean equalsIncludingExplicitlyExcluded(final Object other) { if (other instanceof RelationBean) { return other instanceof RelationBean && this.equals(other) && this.explicitlyExcluded.equals(((RelationBean) other).explicitlyExcluded); } return false; } public Set getExplicitlyExcluded() { return this.explicitlyExcluded; } public Optional getItemFor(final long identifier, final ItemType type) { for (int index = 0; index < this.beanItems.size(); index++) { if (this.beanItems.get(index).getIdentifier() == identifier && this.beanItems.get(index).getType() == type) { return Optional.of(getItemFor(index)); } } return Optional.empty(); } public Optional getItemFor(final long identifier, final String role, final ItemType type) { for (int index = 0; index < this.beanItems.size(); index++) { if (this.beanItems.get(index).getIdentifier() == identifier && role.equals(this.beanItems.get(index).getRole()) && this.beanItems.get(index).getType() == type) { return Optional.of(getItemFor(index)); } } return Optional.empty(); } public List getMemberIdentifiers() { return this.beanItems.stream().map(RelationBeanItem::getIdentifier) .collect(Collectors.toList()); } public List getMemberRoles() { return this.beanItems.stream().map(RelationBeanItem::getRole).collect(Collectors.toList()); } public List getMemberTypes() { return this.beanItems.stream().map(RelationBeanItem::getType).collect(Collectors.toList()); } @Override public int hashCode() { return Objects.hash(this.getMemberIdentifiers(), this.getMemberRoles(), this.getMemberTypes()); } /** * @return True if this bean has no members */ @Override public boolean isEmpty() { return this.beanItems.isEmpty(); } @Override public Iterator iterator() { return this.beanItems.iterator(); } public RelationBean merge(final RelationBean other) { return RelationBean.mergeBeans(this, other); } /** * Remove all matching item from this {@link RelationBean} with a given identifier and * {@link ItemType}. If item(s) are actually removed, this method will return a {@link List} of * their roles. If nothing was removed, then the {@link List} will be empty. * * @param identifier * the id to remove * @param itemType * the type to remove * @return a {@link List} containing the roles of the removed items */ public List removeAllMatchingItems(final Long identifier, final ItemType itemType) { final List newBeanItems = new ArrayList<>(); final List removedRoles = new ArrayList<>(); for (final RelationBeanItem item : this.beanItems) { if (item.getIdentifier().equals(identifier) && item.getType().equals(itemType)) { removedRoles.add(item.getRole()); } else { newBeanItems.add( new RelationBeanItem(item.getIdentifier(), item.getRole(), item.getType())); } } if (!removedRoles.isEmpty()) { this.beanItems = newBeanItems; } return removedRoles; } public boolean removeItem(final Long identifier, final String role, final ItemType itemType) { return removeItem(new RelationBeanItem(identifier, role, itemType)); } public boolean removeItem(final RelationBeanItem item) { return this.beanItems.remove(item); } /** * @return The number of members in this {@link RelationBean} */ @Override public int size() { return this.beanItems.size(); } @Override public String toString() { return "RelationBean [" + this.beanItems + "]"; } private RelationBeanItem getItemFor(final int index) { if (index < 0 || index >= size()) { throw new CoreException("Invalid index {}", index); } return new RelationBeanItem(this.beanItems.get(index)); } private boolean isExplicitlyExcluded(final RelationBeanItem relationBeanItem) { return this.explicitlyExcluded.contains(relationBeanItem); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Map; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.Area; /** * A primitive object for {@link Area} * * @author tony */ public class AtlasPrimitiveArea extends AtlasPrimitiveEntity { private static final long serialVersionUID = -8890808695358609272L; private final Polygon polygon; public AtlasPrimitiveArea(final long identifier, final Polygon polygon, final Map tags) { super(identifier, tags); this.polygon = polygon; } @Override public Rectangle bounds() { return this.polygon.bounds(); } public Polygon getPolygon() { return this.polygon; } @Override public String toString() { return "AtlasPrimitiveArea [polygon=" + this.polygon + ", getIdentifier()=" + getIdentifier() + ", getTags()=" + getTags() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveBigNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode.Path; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode.Type; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.RestrictedPath; /** * A serializable Big Node thats useful in spark jobs * * @author Sid */ public class AtlasPrimitiveBigNode extends AtlasPrimitiveEntity { private static final long serialVersionUID = -1722511104597663348L; // All the nodes defining this BigNode private final Set nodes; // Set of connected edges private final Set edges; // All the possible paths in then out of this BigNode private final Set paths; // All the restricted paths of this BigNode private final Set restrictedPaths; // Type of Big Node private final Type type; public static AtlasPrimitiveBigNode from(final BigNode bigNode) { return from(bigNode, false, Path.SHORTEST); } public static AtlasPrimitiveBigNode from(final BigNode bigNode, final boolean storeRestrictedPaths, final Path pathType) { final Set nodes = bigNode.nodes().stream() .map(node -> AtlasPrimitiveLocationItem.from(node)).collect(Collectors.toSet()); final Set edges = bigNode.edges().stream() .map(edge -> AtlasPrimitiveEdge.from(edge)).collect(Collectors.toSet()); // All the paths in then out of this BigNode - the type (shortest/all/etc.) of path is // configurable final Set paths; switch (pathType) { case SHORTEST: paths = bigNode.shortestPaths().stream().map(AtlasPrimitiveRoute::from) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toSet()); break; case ALL: paths = bigNode.allPaths().stream().map(AtlasPrimitiveRoute::from) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toSet()); break; default: throw new CoreException("Invalid Path Type: {}", pathType); } // All the restricted paths of this BigNode - will use all possible paths under the covers // to capture as many restrictions as possible. final Set restrictedPaths = storeRestrictedPaths ? bigNode .turnRestrictions().stream().map(RestrictedPath::getRoute) .map(AtlasPrimitiveRoute::from).filter(Optional::isPresent).map(Optional::get) .map(AtlasPrimitiveRouteIdentifier::from).collect(Collectors.toSet()) : Collections.emptySet(); return new AtlasPrimitiveBigNode(bigNode.getIdentifier(), nodes, edges, paths, restrictedPaths, bigNode.getTags(), bigNode.getType()); } public AtlasPrimitiveBigNode(final long identifier, final Set nodes, final Set edges, final Set paths, final Set restrictedPaths, final Map tags, final Type type) { super(identifier, tags); this.nodes = nodes; this.edges = edges; this.paths = paths; this.type = type; this.restrictedPaths = restrictedPaths; } @Override public Rectangle bounds() { return Rectangle.forLocated(this.nodes); } public Set edges() { return this.edges; } public Set getRestrictedPaths() { return this.restrictedPaths; } public Set inEdges() { return edges().stream().filter(edge -> !nodesContain(edge.start())) .collect(Collectors.toSet()); } public Set junctionEdges() { return edges().stream() .filter(edge -> nodesContain(edge.start()) && nodesContain(edge.end())) .collect(Collectors.toSet()); } public Set nodeLocations() { return this.nodes().stream().map(AtlasPrimitiveLocationItem::getLocation) .collect(Collectors.toSet()); } public Set nodes() { return this.nodes; } public boolean nodesContain(final Location location) { return this.nodes.stream().filter(node -> node.getLocation().equals(location)).count() > 0; } public Set outEdges() { return edges().stream().filter(edge -> !this.nodeLocations().contains(edge.end())) .collect(Collectors.toSet()); } public Set paths() { return this.paths; } @Override public String toString() { return "[AtlasPrimitiveBigNode: nodes=" + nodes().stream().map(node -> node.getIdentifier()).collect(Collectors.toSet()) + "]"; } public Type type() { return this.type; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; /** * @author Sid */ public class AtlasPrimitiveEdge extends AtlasPrimitiveLineItem { private static final long serialVersionUID = 4693146600590040648L; public static AtlasPrimitiveEdge from(final Edge edge) { return new AtlasPrimitiveEdge(edge.getIdentifier(), edge.asPolyLine(), edge.getTags()); } public AtlasPrimitiveEdge(final long identifier, final PolyLine polyLine, final Map tags) { super(identifier, polyLine, tags); } public Location end() { return this.getPolyLine().last(); } public boolean isReversedEdge(final AtlasPrimitiveEdge reverseEdgeCandidate) { return this.getIdentifier() == -reverseEdgeCandidate.getIdentifier(); } public Location start() { return this.getPolyLine().first(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[AtlasPrimitiveEdge : "); builder.append("id = " + getIdentifier()); builder.append(" : polyLine = " + getPolyLine()); builder.append(" ]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveEdgeIdentifier.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.io.Serializable; /** * @author Sid */ public class AtlasPrimitiveEdgeIdentifier implements Serializable { private static final long serialVersionUID = 6082277761179021192L; private long identifier; public static AtlasPrimitiveEdgeIdentifier from(final AtlasPrimitiveEdge atlasPrimitiveEdge) { return new AtlasPrimitiveEdgeIdentifier(atlasPrimitiveEdge.getIdentifier()); } public AtlasPrimitiveEdgeIdentifier() { } public AtlasPrimitiveEdgeIdentifier(final long identifier) { this.identifier = identifier; } @Override public boolean equals(final Object other) { return other instanceof AtlasPrimitiveEdgeIdentifier && this.getIdentifier() == ((AtlasPrimitiveEdgeIdentifier) other).getIdentifier(); } public long getIdentifier() { return this.identifier; } @Override public int hashCode() { return Long.hashCode(this.identifier); } public void setIdentifier(final long identifier) { this.identifier = identifier; } @Override public String toString() { return "AtlasPrimitiveEdgeIdentifier [identifier=" + this.identifier + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.io.Serializable; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.tags.Taggable; /** * A primitive object for {@link AtlasEntity} * * @author tony * @author Sid */ public abstract class AtlasPrimitiveEntity implements Serializable, Taggable, Located { private static final long serialVersionUID = -4372740269485938585L; private final long identifier; private final Map tags; public AtlasPrimitiveEntity(final long identifier, final Map tags) { this.identifier = identifier; this.tags = tags; } @Override public boolean equals(final Object other) { return other != null && other instanceof AtlasPrimitiveEntity && this.getIdentifier() == ((AtlasPrimitiveEntity) other).getIdentifier(); } public long getIdentifier() { return this.identifier; } @Override public Optional getTag(final String key) { return Optional.ofNullable(getTags().get(key)); } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return Long.hashCode(this.identifier); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveLineItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Map; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.LineItem; /** * A primitive object for {@link LineItem} * * @author tony */ public class AtlasPrimitiveLineItem extends AtlasPrimitiveEntity { private static final long serialVersionUID = 4435750184254868724L; private final PolyLine polyLine; public AtlasPrimitiveLineItem(final long identifier, final PolyLine polyLine, final Map tags) { super(identifier, tags); this.polyLine = polyLine; } @Override public Rectangle bounds() { return this.polyLine.bounds(); } public PolyLine getPolyLine() { return this.polyLine; } @Override public String toString() { return "AtlasPrimitiveLineItem [polyLine=" + this.polyLine + ", getIdentifier()=" + getIdentifier() + ", getTags()=" + getTags() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveLocationItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; /** * A primitive object for {@link LocationItem} * * @author tony */ public class AtlasPrimitiveLocationItem extends AtlasPrimitiveEntity { private static final long serialVersionUID = -6767702793907654973L; private final Location location; public static AtlasPrimitiveLocationItem from(final Node node) { final AtlasPrimitiveLocationItem locationItem = new AtlasPrimitiveLocationItem( node.getIdentifier(), node.getLocation(), node.getTags()); return locationItem; } public AtlasPrimitiveLocationItem(final long identifier, final Location location, final Map tags) { super(identifier, tags); this.location = location; } @Override public Rectangle bounds() { return this.location.bounds(); } public Location getLocation() { return this.location; } @Override public String toString() { return "AtlasPrimitiveLocationItem [location=" + this.location + ", getIdentifier()=" + getIdentifier() + ", getTags()=" + getTags() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveObjectStore.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.utilities.collections.JoinedCollection; import org.openstreetmap.atlas.utilities.collections.ParallelIterable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Store all primitive entities which can then be used to create @{link Atlas} * * @author tony */ public class AtlasPrimitiveObjectStore { private static final Logger logger = LoggerFactory.getLogger(AtlasPrimitiveObjectStore.class); private final Map nodes = new HashMap<>(); private final Map points = new HashMap<>(); private final Map edges = new HashMap<>(); private final Map lines = new HashMap<>(); private final Map areas = new HashMap<>(); private final Map relations = new HashMap<>(); public void addArea(final AtlasPrimitiveArea area) { this.areas.put(area.getIdentifier(), area); } public void addEdge(final AtlasPrimitiveLineItem edge) { this.edges.put(edge.getIdentifier(), edge); } public void addLine(final AtlasPrimitiveLineItem line) { this.lines.put(line.getIdentifier(), line); } public void addNode(final AtlasPrimitiveLocationItem node) { this.nodes.put(node.getIdentifier(), node); } public void addPoint(final AtlasPrimitiveLocationItem point) { this.points.put(point.getIdentifier(), point); } public void addRelation(final AtlasPrimitiveRelation relation) { this.relations.put(relation.getIdentifier(), relation); } /** * @return an {@link Atlas} object based on this object store */ public Atlas build() { // There should be no missing identifiers if the data is complete final Optional missingIdentifiers = checkDataIntegrity(); missingIdentifiers.ifPresent(missingObjects -> { throw new CoreException("Data is not complete, still missing {} ", missingObjects.toDebugString()); }); logger.debug("Data is complete, starts to build atlas"); // Data is complete, run atlas builder final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(new AtlasSize( this.getEdges().size(), this.getNodes().size(), this.getAreas().size(), this.getLines().size(), this.getPoints().size(), this.getRelations().size())); logger.debug("Building atlas nodes..."); getNodes().values().forEach(node -> { builder.addNode(node.getIdentifier(), node.getLocation(), node.getTags()); }); logger.debug("Building atlas points..."); getPoints().values().forEach(point -> { builder.addPoint(point.getIdentifier(), point.getLocation(), point.getTags()); }); logger.debug("Building atlas edges..."); getEdges().values().forEach(edge -> { builder.addEdge(edge.getIdentifier(), edge.getPolyLine(), edge.getTags()); }); logger.debug("Building atlas lines..."); getLines().values().forEach(line -> { builder.addLine(line.getIdentifier(), line.getPolyLine(), line.getTags()); }); logger.debug("Building atlas areas..."); getAreas().values().forEach(area -> { builder.addArea(area.getIdentifier(), area.getPolygon(), area.getTags()); }); logger.debug("Building atlas relations..."); getRelations().values().forEach(relation -> { builder.addRelation(relation.getIdentifier(), relation.getOsmIdentifier(), relation.getRelationBean(), relation.getTags()); }); return builder.get(); } /** * This method checks the data integrity and will be run at the beginning of build() method. * Also different readers (PBF/other) can call this method to check if the current data set is * complete. If not the reader needs to make data set complete * * @return A {@link TemporaryObjectStore} if the data in current store is not complete */ public Optional checkDataIntegrity() { logger.debug("Checking object store data integrity"); final TemporaryObjectStore missingObjects = new TemporaryObjectStore(); // Check relations getRelations().values().forEach(relation -> { final RelationBean bean = relation.getRelationBean(); final ParallelIterable parallel = new ParallelIterable(bean.getMemberIdentifiers(), bean.getMemberTypes()); final Iterator iterator = parallel.iterator(); while (iterator.hasNext()) { final JoinedCollection relationMember = iterator.next(); final long identifier = relationMember.get(0); final ItemType type = (ItemType) relationMember.get(1); switch (type) { case NODE: if (!this.nodes.containsKey(identifier)) { missingObjects.addNode(identifier); } break; case EDGE: if (!this.edges.containsKey(identifier)) { missingObjects.addEdge(identifier); } break; case AREA: if (!this.areas.containsKey(identifier)) { missingObjects.addArea(identifier); } break; case LINE: if (!this.lines.containsKey(identifier)) { missingObjects.addLine(identifier); } break; case POINT: if (!this.points.containsKey(identifier)) { missingObjects.addPoint(identifier); } break; case RELATION: if (!this.relations.containsKey(identifier)) { missingObjects.addRelation(identifier); } break; default: throw new CoreException("Unknown type {}", type); } } }); // Put location of all nodes into a set final Set locationOfNodes = new HashSet<>(this.nodes.size(), 1); getNodes().values().forEach(node -> locationOfNodes.add(node.getLocation())); // Check Edge getEdges().values().forEach(edge -> { final PolyLine shape = edge.getPolyLine(); final Location first = shape.first(); final Location last = shape.last(); if (!locationOfNodes.contains(first)) { missingObjects.addLocation(first); } if (!locationOfNodes.contains(last)) { missingObjects.addLocation(last); } }); return missingObjects.isEmpty() ? Optional.empty() : Optional.of(missingObjects); } public Map getAreas() { return this.areas; } public Map getEdges() { return this.edges; } public Map getLines() { return this.lines; } public Map getNodes() { return this.nodes; } public Map getPoints() { return this.points; } public Map getRelations() { return this.relations; } public String summary() { return "The store has " + this.nodes.size() + " nodes, " + this.points.size() + " points, " + this.edges.size() + " edges, " + this.lines.size() + " lines, " + this.areas.size() + " areas, " + this.relations.size() + " relations"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.Map; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; /** * A primitive object for {@link RelationBean} * * @author tony */ public class AtlasPrimitiveRelation extends AtlasPrimitiveEntity { private static final long serialVersionUID = 4189537752256202439L; private final long osmIdentifier; private final RelationBean relationBean; private final Rectangle bounds; public AtlasPrimitiveRelation(final long identifier, final long osmIdentifier, final RelationBean relationBean, final Map tags, final Rectangle bounds) { super(identifier, tags); this.osmIdentifier = osmIdentifier; this.relationBean = relationBean; this.bounds = bounds; } @Override public Rectangle bounds() { return this.bounds; } public long getOsmIdentifier() { return this.osmIdentifier; } public RelationBean getRelationBean() { return this.relationBean; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveRoute.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.utilities.collections.Iterables; import com.google.common.collect.ImmutableList; /** * @author Sid */ public final class AtlasPrimitiveRoute implements Iterable, Serializable { private static final long serialVersionUID = 804872809334744220L; /* * This comparator can be used to sort AtlasPrimitiveRoutes in descending order of size. */ public static final Comparator ROUTE_SIZE_COMPARATOR = ( final AtlasPrimitiveRoute route1, final AtlasPrimitiveRoute route2) -> new CompareToBuilder() .append(route2.size(), route1.size()) // The hashcode needs to be deterministic to ensure deterministic order .append(route1.hashCode(), route2.hashCode()).toComparison(); private final List primitiveRoute; public static AtlasPrimitiveRoute from(final AtlasPrimitiveEdge... primitiveEdges) { return new AtlasPrimitiveRoute(Arrays.asList(primitiveEdges)); } public static AtlasPrimitiveRoute from( final AtlasPrimitiveRouteIdentifier atlasPrimitiveRouteIdentifier, final Atlas atlas) { final List fromEdges = Iterables.stream(atlasPrimitiveRouteIdentifier) .map(edgeIdentifier -> atlas.edge(edgeIdentifier.getIdentifier())).collectToList(); return AtlasPrimitiveRoute.from(fromEdges); } public static AtlasPrimitiveRoute from(final List edges) { return new AtlasPrimitiveRoute( edges.stream().map(AtlasPrimitiveEdge::from).collect(Collectors.toList())); } public static Optional from(final Route route) { if (route != null && route.size() > 0) { return Optional.of( new AtlasPrimitiveRoute(Iterables.translate(route, AtlasPrimitiveEdge::from))); } return Optional.empty(); } public AtlasPrimitiveRoute(final Iterable primitiveEdges) { this.primitiveRoute = ImmutableList.copyOf(primitiveEdges); } public PolyLine asPolyLine() { final List locations = new ArrayList<>(); Location lastLocation = null; for (final AtlasPrimitiveEdge edge : this.primitiveRoute) { final PolyLine polyLine = edge.getPolyLine(); locations.addAll(polyLine); lastLocation = polyLine.last(); locations.remove(lastLocation); } locations.add(lastLocation); return new PolyLine(locations); } public AtlasPrimitiveEdge end() { if (this.primitiveRoute.size() > 0) { return this.primitiveRoute.get(this.primitiveRoute.size() - 1); } throw new CoreException("Illegal State : Empty route"); } @Override public boolean equals(final Object other) { if (other instanceof AtlasPrimitiveRoute) { final AtlasPrimitiveRoute that = (AtlasPrimitiveRoute) other; if (this.size() == that.size()) { return new EqualsBuilder().append(this.start().start(), that.start().start()) .append(this.end().end(), that.end().end()) .append(this.primitiveRoute, that.primitiveRoute).isEquals(); } } return false; } /** * This method uses the provided {@link Atlas} to return the {@link Route} corresponding to this * {@link AtlasPrimitiveRoute} * * @param atlas * {@link Atlas} containing the {@link Route} {@link Edge}s * @return {@link Route} corresponding to this {@link AtlasPrimitiveRoute} */ public Optional getRoute(final Atlas atlas) { final List edges = new ArrayList<>(); for (final AtlasPrimitiveEdge primitiveEdge : this.primitiveRoute) { final Edge edge = atlas.edge(primitiveEdge.getIdentifier()); if (edge == null) { return Optional.empty(); } edges.add(edge); } return Optional.of(Route.forEdges(edges)); } /** * Note: The start and end {@link Node}s are part of the hash code to reduce the probability of * a collision. There are other candidates to add here, like distance between start/end, but * start/end by themselves are the least computationally intensive to derive. For best practice, * this is consistent with {@link Route}'s hash code. */ @Override public int hashCode() { return new HashCodeBuilder().append(this.start().start()).append(this.end().end()) .append(Iterables.asList(this)).hashCode(); } public int indexOf(final AtlasPrimitiveEdge primitiveEdge) { return this.primitiveRoute.indexOf(primitiveEdge); } public boolean isOverlappedBy(final AtlasPrimitiveRoute primitiveRoute) { return AtlasPrimitiveRouteIdentifier.from(this) .isOverlappedBy(AtlasPrimitiveRouteIdentifier.from(primitiveRoute)); } public boolean isOverlappedBy(final Route route) { final Optional atlasPrimitiveRoute = AtlasPrimitiveRoute.from(route); return atlasPrimitiveRoute.isPresent() && isOverlappedBy(atlasPrimitiveRoute.get()); } @Override public Iterator iterator() { return this.primitiveRoute.iterator(); } /** * Counts the number of times a subRoute overlaps a route. Currently this doesn't handle cases * with loops within subRoute E.g [1,2,3,1,2,3,1,4,5], subRoute is [1,2,3,1,4] * * @param subRoute * Smaller subsequence of the route that can overlap with route * @return overlapCount */ public int overlapCount(final AtlasPrimitiveRoute subRoute) { int overlapCount = 0; if (this.primitiveRoute == null || subRoute == null) { return overlapCount; } Iterator subRouteIterator = subRoute.iterator(); AtlasPrimitiveEdge subRouteEdge = subRouteIterator.hasNext() ? subRouteIterator.next() : null; for (final AtlasPrimitiveEdge edge : this.primitiveRoute) { if (subRouteEdge == null) { break; } if (edge.equals(subRouteEdge)) { if (!subRouteIterator.hasNext()) { overlapCount++; subRouteIterator = subRoute.iterator(); } } else { subRouteIterator = subRoute.iterator(); if (edge.equals(subRoute.start())) { subRouteEdge = subRouteIterator.hasNext() ? subRouteIterator.next() : null; } } subRouteEdge = subRouteIterator.hasNext() ? subRouteIterator.next() : null; } return overlapCount; } public int size() { return this.primitiveRoute.size(); } public AtlasPrimitiveEdge start() { if (this.primitiveRoute.size() > 0) { return this.primitiveRoute.get(0); } throw new CoreException("Illegal State : Empty route"); } public List subRoute(final int fromIndex, final int toIndex) { return this.primitiveRoute.subList(fromIndex, toIndex); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[AtlasPrimitiveRoute: "); builder.append(this.primitiveRoute.stream().map(edge -> edge.toString()) .collect(Collectors.joining(", "))); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/AtlasPrimitiveRouteIdentifier.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.io.Serializable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import com.google.common.collect.ImmutableList; /** * @author Sid */ public class AtlasPrimitiveRouteIdentifier implements Iterable, Serializable { private static final long serialVersionUID = 2321636844479248974L; private List primitiveRouteIdentifier; public static AtlasPrimitiveRouteIdentifier from(final AtlasPrimitiveRoute atlasPrimitiveRoute) { final List atlasPrimitiveEdgeIds = new ArrayList<>(); atlasPrimitiveRoute.forEach( edge -> atlasPrimitiveEdgeIds.add(AtlasPrimitiveEdgeIdentifier.from(edge))); return new AtlasPrimitiveRouteIdentifier(atlasPrimitiveEdgeIds); } public AtlasPrimitiveRouteIdentifier() { } public AtlasPrimitiveRouteIdentifier(final AtlasPrimitiveEdgeIdentifier... primitiveEdgeIds) { this.primitiveRouteIdentifier = ImmutableList.copyOf(primitiveEdgeIds); } public AtlasPrimitiveRouteIdentifier( final Iterable primitiveEdgeIds) { this.primitiveRouteIdentifier = ImmutableList.copyOf(primitiveEdgeIds); } public AtlasPrimitiveEdgeIdentifier end() { if (this.primitiveRouteIdentifier.size() > 0) { return this.primitiveRouteIdentifier.get(this.primitiveRouteIdentifier.size() - 1); } throw new CoreException("Illegal State : Empty route"); } @Override public boolean equals(final Object other) { if (other instanceof AtlasPrimitiveRouteIdentifier) { final AtlasPrimitiveRouteIdentifier that = (AtlasPrimitiveRouteIdentifier) other; if (this.primitiveRouteIdentifier.size() == that.primitiveRouteIdentifier.size()) { return new EqualsBuilder() .append(this.primitiveRouteIdentifier, that.primitiveRouteIdentifier) .isEquals(); } } return false; } @Override public int hashCode() { return new HashCodeBuilder().append(this.primitiveRouteIdentifier).hashCode(); } /* * Similar to {@link Route#overlapIndex} */ public boolean isOverlappedBy(final AtlasPrimitiveRouteIdentifier primitiveRouteIdentifier) { if (primitiveRouteIdentifier == null) { return false; } // Keep track of the last index at which the last Edge was overlapping this route, to avoid // returning false positives in case of routes making a loop. int lastOverlapIndex = -1; for (final AtlasPrimitiveEdgeIdentifier primitiveEdgeIdentifier : primitiveRouteIdentifier) { final int index = this.primitiveRouteIdentifier.indexOf(primitiveEdgeIdentifier); if (index <= lastOverlapIndex) { // The edge does not overlap, or it does but at a smaller index which would indicate // a loop. return false; } lastOverlapIndex = index; } return true; } @Override public Iterator iterator() { return this.primitiveRouteIdentifier.iterator(); } /** * Counts the number of times a subRoute overlaps a route. * * @param subRouteIdentifier * - Smaller subsequence of the route that can overlap with route * @return overlapCount */ public int overlapCount(final AtlasPrimitiveRouteIdentifier subRouteIdentifier) { int overlapCount = 0; if (this.primitiveRouteIdentifier == null || subRouteIdentifier == null) { return overlapCount; } Iterator subRouteIdentifierIterator = subRouteIdentifier .iterator(); AtlasPrimitiveEdgeIdentifier subRouteEdgeIdentifier = subRouteIdentifierIterator.hasNext() ? subRouteIdentifierIterator.next() : null; for (final AtlasPrimitiveEdgeIdentifier edge : this.primitiveRouteIdentifier) { if (subRouteEdgeIdentifier == null) { break; } if (edge.equals(subRouteEdgeIdentifier)) { if (!subRouteIdentifierIterator.hasNext()) { overlapCount++; subRouteIdentifierIterator = subRouteIdentifier.iterator(); } } else { subRouteIdentifierIterator = subRouteIdentifier.iterator(); if (edge.equals(subRouteIdentifier.start())) { subRouteEdgeIdentifier = subRouteIdentifierIterator.hasNext() ? subRouteIdentifierIterator.next() : null; } } subRouteEdgeIdentifier = subRouteIdentifierIterator.hasNext() ? subRouteIdentifierIterator.next() : null; } return overlapCount; } public void setPrimitiveRouteIdentifier( final List primitiveRouteIdentifier) { this.primitiveRouteIdentifier = primitiveRouteIdentifier; } public int size() { return this.primitiveRouteIdentifier.size(); } public AtlasPrimitiveEdgeIdentifier start() { if (this.primitiveRouteIdentifier.size() > 0) { return this.primitiveRouteIdentifier.get(0); } throw new CoreException("Illegal State : Empty route"); } public List subRoute(final int fromIndex, final int toIndex) { return this.primitiveRouteIdentifier.subList(fromIndex, toIndex); } @Override public String toString() { return "AtlasPrimitiveRouteIdentifier [identifier=" + this.primitiveRouteIdentifier + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/store/TemporaryObjectStore.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.store; import java.util.HashSet; import java.util.Set; import org.openstreetmap.atlas.geography.Location; /** * A temporary object store to hold missing objects when checking data integrity of * {@link AtlasPrimitiveObjectStore} * * @author tony */ public class TemporaryObjectStore { private final Set nodes = new HashSet<>(); private final Set points = new HashSet<>(); private final Set edges = new HashSet<>(); private final Set lines = new HashSet<>(); private final Set areas = new HashSet<>(); private final Set relations = new HashSet<>(); private final Set locations = new HashSet<>(); public void addArea(final long identifier) { this.areas.add(identifier); } public void addEdge(final long identifier) { this.edges.add(identifier); } public void addLine(final long identifier) { this.lines.add(identifier); } public void addLocation(final Location location) { this.locations.add(location); } public void addNode(final long identifier) { this.nodes.add(identifier); } public void addPoint(final long identifier) { this.points.add(identifier); } public void addRelation(final long identifier) { this.relations.add(identifier); } public Set getAreas() { return this.areas; } public Set getEdges() { return this.edges; } public Set getLines() { return this.lines; } public Set getLocations() { return this.locations; } public Set getNodes() { return this.nodes; } public Set getPoints() { return this.points; } public Set getRelations() { return this.relations; } public boolean isEmpty() { return this.nodes.isEmpty() && this.points.isEmpty() && this.edges.isEmpty() && this.lines.isEmpty() && this.areas.isEmpty() && this.relations.isEmpty() && this.locations.isEmpty(); } public int size() { return this.nodes.size() + this.points.size() + this.edges.size() + this.lines.size() + this.areas.size() + this.relations.size() + this.locations.size(); } public String toDebugString() { return "TemporaryIdentifierStore has " + this.nodes.size() + " nodes, " + this.points.size() + " points, " + this.edges.size() + " edges, " + this.lines.size() + " lines, " + this.areas.size() + " areas, " + this.relations.size() + " relations, " + this.locations.size() + " locations"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/builder/text/TextAtlasBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.builder.text; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.converters.PolyLineStringConverter; import org.openstreetmap.atlas.geography.converters.PolygonStringConverter; import org.openstreetmap.atlas.streaming.resource.LineWriter; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Build some Atlas from a simple text file * * @author matthieun */ public class TextAtlasBuilder { /** * @author matthieun */ private enum WriteMode { NODE(NODES_HEADER), EDGE(EDGES_HEADER), AREA(AREAS_HEADER), LINE(LINES_HEADER), POINT(POINTS_HEADER), RELATION(RELATIONS_HEADER); private final String header; protected static WriteMode forHeader(final String header) { switch (header) { case NODES_HEADER: return WriteMode.NODE; case EDGES_HEADER: return WriteMode.EDGE; case AREAS_HEADER: return WriteMode.AREA; case LINES_HEADER: return WriteMode.LINE; case POINTS_HEADER: return WriteMode.POINT; case RELATIONS_HEADER: return WriteMode.RELATION; default: throw new CoreException("Invalid Header {}", header); } } WriteMode(final String header) { this.header = header; } protected String getHeader() { return this.header; } } private static final Logger logger = LoggerFactory.getLogger(TextAtlasBuilder.class); private static final String NODES_HEADER = "# Nodes"; private static final String EDGES_HEADER = "# Edges"; private static final String AREAS_HEADER = "# Areas"; private static final String LINES_HEADER = "# Lines"; private static final String POINTS_HEADER = "# Points"; private static final String RELATIONS_HEADER = "# Relations"; private static final String SEPARATOR = " && "; private static final String SECONDARY_SEPARATOR = " || "; private static final String TERTIARY_SEPARATOR = " -> "; private static final String SEPARATOR_REPLACEMENT = " "; public static String getNodesHeader() { return NODES_HEADER; } public PackedAtlas read(final Resource resource) { WriteMode mode = null; long numberOfNodes = 0L; long numberOfEdges = 0L; long numberOfAreas = 0L; long numberOfLines = 0L; long numberOfPoints = 0L; long numberOfRelations = 0L; for (final String line : resource.lines()) { if (line.startsWith("#")) { mode = WriteMode.forHeader(line); } else { if (mode == null) { throw new CoreException( "Failed reading {}. Is that a text Atlas? Is it compressed?", resource); } switch (mode) { case NODE: numberOfNodes++; break; case EDGE: numberOfEdges++; break; case AREA: numberOfAreas++; break; case LINE: numberOfLines++; break; case POINT: numberOfPoints++; break; case RELATION: numberOfRelations++; break; default: break; } } } final AtlasSize size = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, numberOfLines, numberOfPoints, numberOfRelations); if (size.getEntityNumber() == 0) { throw new CoreException("Invalid text Atlas, it appears to be empty!"); } if (size.getNonRelationEntityNumber() == 0 && size.getRelationNumber() > 0) { throw new CoreException("Invalid text Atlas, it only contained Relations!"); } final AtlasMetaData metaData = new AtlasMetaData(size, true, "unknown", "TextAtlas", "unknown", "unknown", Maps.hashMap()); final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(size) .withMetaData(metaData).withName(resource.getName()).withEnhancedRelationGeometry(); for (final String line : resource.lines()) { if (line.startsWith("#")) { mode = WriteMode.forHeader(line); } else { switch (mode) { case NODE: parseNode(builder, line); break; case EDGE: parseEdge(builder, line); break; case AREA: parseArea(builder, line); break; case LINE: parseLine(builder, line); break; case POINT: parsePoint(builder, line); break; case RELATION: parseRelation(builder, line); break; default: break; } } } final PackedAtlas atlas = (PackedAtlas) builder.get(); if (atlas == null) { throw new CoreException("Atlas resulting from PackedAtlasBuilder was null."); } return atlas; } public void write(final Atlas atlas, final WritableResource resource) { try (LineWriter writer = new LineWriter(resource)) { for (final WriteMode mode : WriteMode.values()) { writer.writeLine(mode.getHeader()); switch (mode) { case NODE: atlas.nodes().forEach(item -> writer.writeLine(convertNode(item))); break; case EDGE: atlas.edges().forEach(item -> writer.writeLine(convertEdge(item))); break; case AREA: atlas.areas().forEach(item -> writer.writeLine(convertArea(item))); break; case LINE: atlas.lines().forEach(item -> writer.writeLine(convertLine(item))); break; case POINT: atlas.points().forEach(item -> writer.writeLine(convertPoint(item))); break; case RELATION: atlas.relations().forEach(item -> writer.writeLine(convertRelation(item))); break; default: break; } } } catch (final Exception e) { throw new CoreException("Unable to write Atlas to {}", resource, e); } } private String cleanupTags(final String value) { if (value != null) { String result = value; if (value.contains(SEPARATOR)) { logger.warn("Tag {} contains {}. Replacing it with \"{}\"", value, SEPARATOR, SEPARATOR_REPLACEMENT); result = result.replace(SEPARATOR, SEPARATOR_REPLACEMENT); } if (value.contains(SECONDARY_SEPARATOR)) { logger.warn("Tag {} contains {}. Replacing it with \"{}\"", value, SECONDARY_SEPARATOR, SEPARATOR_REPLACEMENT); result = result.replace(SECONDARY_SEPARATOR, SEPARATOR_REPLACEMENT); } if (value.contains(TERTIARY_SEPARATOR)) { logger.warn("Tag {} contains {}. Replacing it with \"{}\"", value, TERTIARY_SEPARATOR, SEPARATOR_REPLACEMENT); result = result.replace(TERTIARY_SEPARATOR, SEPARATOR_REPLACEMENT); } if (value.contains(System.lineSeparator())) { logger.warn("Tag {} contains a new line. Removing it.", value); result = result.replace(System.lineSeparator(), ""); } return result; } return value; } private String convertArea(final Area item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(item.asPolygon().toCompactString()); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertEdge(final Edge item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(item.asPolyLine().toCompactString()); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertLine(final Line item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(item.asPolyLine().toCompactString()); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertNode(final Node item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(item.getLocation().toCompactString()); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertPoint(final Point item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(item.getLocation().toCompactString()); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertRelation(final Relation item) { final StringList list = new StringList(); list.add(item.getIdentifier()); list.add(convertRelationBean(item)); list.add(convertTags(item)); return list.join(SEPARATOR); } private String convertRelationBean(final Relation relation) { final StringList bean = new StringList(); for (final RelationMember member : relation.members()) { final StringList list = new StringList(); list.add(member.getEntity().getIdentifier()); list.add(member.getRole()); final ItemType type = ItemType.forEntity(member.getEntity()); list.add(type.toShortString()); bean.add(list.join(TERTIARY_SEPARATOR)); } return bean.join(SECONDARY_SEPARATOR); } private String convertTags(final Taggable taggable) { final StringList tags = new StringList(); for (final Entry entry : taggable.getTags().entrySet()) { final StringBuilder builder = new StringBuilder(); builder.append(cleanupTags(entry.getKey())); builder.append(TERTIARY_SEPARATOR); builder.append(cleanupTags(entry.getValue())); tags.add(builder.toString()); } return tags.join(SECONDARY_SEPARATOR); } private void parseArea(final PackedAtlasBuilder builder, final String line) { final StringList split = StringList.split(line, SEPARATOR); final long identifier = Long.parseLong(split.get(0)); final Polygon geometry = new PolygonStringConverter().convert(split.get(1)); final Map tags = new HashMap<>(); if (split.size() > 2) { tags.putAll(parseTags(split.get(2))); } builder.addArea(identifier, geometry, tags); } private void parseEdge(final PackedAtlasBuilder builder, final String line) { final StringList split = StringList.split(line, SEPARATOR); final long identifier = Long.parseLong(split.get(0)); final PolyLine geometry = new PolyLineStringConverter().convert(split.get(1)); final Map tags = new HashMap<>(); if (split.size() > 2) { tags.putAll(parseTags(split.get(2))); } builder.addEdge(identifier, geometry, tags); } private void parseLine(final PackedAtlasBuilder builder, final String line) { final StringList split = StringList.split(line, SEPARATOR); final long identifier = Long.parseLong(split.get(0)); final PolyLine geometry = new PolyLineStringConverter().convert(split.get(1)); final Map tags = new HashMap<>(); if (split.size() > 2) { tags.putAll(parseTags(split.get(2))); } builder.addLine(identifier, geometry, tags); } private void parseNode(final PackedAtlasBuilder builder, final String line) { final StringList split = StringList.split(line, SEPARATOR); final long identifier = Long.parseLong(split.get(0)); final Location geometry = Location.forString(split.get(1)); final Map tags = new HashMap<>(); if (split.size() > 2) { tags.putAll(parseTags(split.get(2))); } builder.addNode(identifier, geometry, tags); } private void parsePoint(final PackedAtlasBuilder builder, final String line) { final StringList split = StringList.split(line, SEPARATOR); final long identifier = Long.parseLong(split.get(0)); final Location geometry = Location.forString(split.get(1)); final Map tags = new HashMap<>(); if (split.size() > 2) { tags.putAll(parseTags(split.get(2))); } builder.addPoint(identifier, geometry, tags); } private void parseRelation(final PackedAtlasBuilder builder, final String line) { final WKTReader reader = new WKTReader(); final StringList split = StringList.split(line, SEPARATOR); final Iterator itr = split.iterator(); final long identifier = Long.parseLong(itr.next()); final RelationBean structure = parseRelationBean(itr.next()); final Map tags = new HashMap<>(); if (itr.hasNext()) { tags.putAll(parseTags(itr.next())); } try { if (itr.hasNext()) { builder.addRelation(identifier, identifier, structure, tags, (MultiPolygon) reader.read(itr.next())); } else { builder.addRelation(identifier, identifier, structure, tags); } } catch (final ParseException e) { throw new CoreException("Bad relation data for relation {}", identifier); } } private RelationBean parseRelationBean(final String value) { final RelationBean bean = new RelationBean(); final StringList split = StringList.split(value, SECONDARY_SEPARATOR); for (final String beanValue : split) { final StringList valueSplit = StringList.split(beanValue, TERTIARY_SEPARATOR); final long identifier = Long.parseLong(valueSplit.get(0)); final String role = valueSplit.get(1); final ItemType itemType = ItemType.shortValueOf(valueSplit.get(2)); bean.addItem(identifier, role, itemType); } return bean; } private Map parseTags(final String value) { try { final Map result = Maps.hashMap(); final StringList split = StringList.split(value, SECONDARY_SEPARATOR); for (final String tag : split) { final StringList tagSplit = StringList.split(tag, TERTIARY_SEPARATOR); if (tagSplit.size() == 2) { result.put(tagSplit.get(0), tagSplit.get(1)); } if (tagSplit.size() == 1) { result.put(tagSplit.get(0), ""); } } return result; } catch (final Throwable error) { throw new CoreException("Unable to parse tags from \"{}\"", value, error); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/AtlasChangeGenerator.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.io.Serializable; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * Something that takes an {@link Atlas} and produces a set of {@link FeatureChange} that should be * apply-able back to the initial {@link Atlas} * * @author matthieun */ public interface AtlasChangeGenerator extends Converter>, Serializable { @Override default Set convert(final Atlas atlas) { return generate(atlas); } /** * Generate a set of changes that make sense out of the gate. * * @param atlas * The Atlas to generate the changes from. * @return The validated set of {@link FeatureChange}s */ default Set generate(final Atlas atlas) { final Set result = new FeatureChangeBoundsExpander( generateWithoutValidation(atlas), atlas).apply(); result.stream().forEach(featureChange -> featureChange.withAtlasContext(atlas)); if (result.isEmpty()) { return result; } final ChangeBuilder builder = new ChangeBuilder(); result.forEach(builder::add); final Change change = builder.get(); // Validate validate(atlas, change); // Return the already merged changes return change.changes().collect(Collectors.toSet()); } /** * Generate a set of changes. * * @param atlas * The Atlas to generate the changes from. * @return The un-validated set of {@link FeatureChange}s */ Set generateWithoutValidation(Atlas atlas); default String getName() { return this.getClass().getSimpleName(); } default void validate(final Atlas source, final Change change) { new ChangeAtlas(source, change).validate(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/AtlasEntityKey.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.complete.CompleteItemType; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Caters to use cases where {@link AtlasEntity}-ies need to be grouped by the * {@link AtlasEntity#getIdentifier()}. {@link AtlasEntity#getIdentifier()} could repeat across * different entity types, and hence combined with {@link ItemType}. This class extends * {@link Tuple} and adds some functionality to reduce code verbosity. * * @author Yazad Khambata */ public class AtlasEntityKey extends Tuple { private static final long serialVersionUID = -3670403373644942819L; public static AtlasEntityKey from(final ItemType itemType, final Long identifier) { return new AtlasEntityKey(itemType, identifier); } public static AtlasEntityKey from(final FeatureChange featureChange) { return from(featureChange.getItemType(), featureChange.getIdentifier()); } protected AtlasEntityKey(final ItemType itemType, final Long identifier) { super(itemType, identifier); } public AtlasEntity getAtlasEntity(final Atlas atlas) { return getItemType().entityForIdentifier(atlas, getIdentifier()); } public CompleteItemType getCompleteItemType() { return CompleteItemType.from(getItemType()); } public Long getIdentifier() { return getSecond(); } public ItemType getItemType() { return getFirst(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/Change.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescription; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptorName; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.TagChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.serializer.ChangeGeoJsonSerializer; import org.openstreetmap.atlas.geography.atlas.change.serializer.FeatureChangeGeoJsonSerializer; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.complete.PrettifyStringFormat; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.tuples.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A change that can be applied to an {@link Atlas} to generate a {@link ChangeAtlas}. *

* It contains a collection of {@link FeatureChange} objects, which describe the changes. * * @author matthieun * @author Yazad Khambata */ public class Change implements Located, Serializable { private static final long serialVersionUID = 1048481626851547987L; private static final Logger logger = LoggerFactory.getLogger(Change.class); private static final AtomicInteger CHANGE_IDENTIFIER_FACTORY = new AtomicInteger(); private final List featureChanges; private final Map identifierToIndex; private final int identifier; private Rectangle bounds; private String name; private transient Map allChangesMappedByAtlasEntityKey; /** * Merge {@link FeatureChange}s inside {@link Change} objects and create a * {@link Change#newInstance()} with the merged {@link FeatureChange}s. The * {@link #merge(Change...)} is guided by groupings based on {@link FeatureChangeMergeGroup}. * * @param changeInstances * - the {@link Change} instances to merge. * @return - A {@link #newInstance()} of Change with {@link FeatureChange}s * {@link FeatureChange#merge(FeatureChange)}-ed. */ public static Change merge(final Change... changeInstances) { final FeatureChange[] mergedFeatureChanges = Arrays.stream(changeInstances) .flatMap(Change::changes) .collect(Collectors.groupingBy(FeatureChangeMergeGroup::from, LinkedHashMap::new, Collectors.reducing(FeatureChange::merge))) .values().stream().map(Optional::get).toArray(FeatureChange[]::new); return Change.newInstance().withName("Merged Change").addAll(mergedFeatureChanges); } public static Change newInstance() { return new Change(); } protected Change() { this.featureChanges = new ArrayList<>(); this.identifierToIndex = new HashMap<>(); this.identifier = CHANGE_IDENTIFIER_FACTORY.getAndIncrement(); } public Map allChangesMappedByAtlasEntityKey() { if (this.allChangesMappedByAtlasEntityKey == null) { this.allChangesMappedByAtlasEntityKey = changes() .map(featureChange -> Tuple.createTuple(AtlasEntityKey.from(featureChange), featureChange)) .collect(Collectors.toMap(Tuple::getFirst, Tuple::getSecond)); } return this.allChangesMappedByAtlasEntityKey; } @Override public Rectangle bounds() { return this.bounds; } public int changeCount() { return this.featureChanges.size(); } public Optional changeFor(final ItemType itemType, final Long identifier) { final AtlasEntityKey key = AtlasEntityKey.from(itemType, identifier); if (!this.identifierToIndex.containsKey(key)) { return Optional.empty(); } return Optional.ofNullable(this.featureChanges.get(this.identifierToIndex.get(key))); } public Stream changes() { return this.featureChanges.stream(); } public Stream changesFor(final ItemType itemType) { return this.identifierToIndex.keySet().stream() .filter(tuple -> tuple.getFirst() == itemType) .map(tuple -> this.featureChanges.get(this.identifierToIndex.get(tuple))); } /** * An Object{@link #equals(Object)} implementation based on {@link #featureChanges} in the * {@link Change} objects being compared. * * @param other * - the object to compare. * @return boolean - true if the objects are equal * @see Objects#equals(Object, Object) * @see Object#equals(Object) */ @Override public boolean equals(final Object other) { // self check if (this == other) { return true; } // null check if (other == null) { return false; } // type check and cast if (getClass() != other.getClass()) { return false; } final Change that = (Change) other; return Objects.equals(this.featureChanges, that.featureChanges); } public List getFeatureChanges() { return this.featureChanges; } public int getIdentifier() { return this.identifier; } public String getName() { return Optional.ofNullable(this.name).orElse(String.valueOf(this.getIdentifier())); } /** * An Object{@link #hashCode()} implementation based on {@link #featureChanges} in the * {@link Change}. * * @return - the hash code. * @see Objects#hashCode(Object) * @see Object#hashCode() */ @Override public int hashCode() { return Objects.hashCode(this.featureChanges); } /** * Transform this {@link Change} into a pretty string. This will use the pretty strings for * {@link CompleteEntity} classes that make up this {@link Change}'s constituent * {@link FeatureChange}s. * * @param featureChangeFormat * the format type for the the constituent {@link FeatureChange}s * @param completeEntityFormat * the format type for the constituent {@link CompleteEntity}s * @return the pretty string */ public String prettify(final PrettifyStringFormat featureChangeFormat, final PrettifyStringFormat completeEntityFormat) { return this.prettify(featureChangeFormat, completeEntityFormat, true); } /** * Transform this {@link Change} into a pretty string. This will use the pretty strings for * {@link CompleteEntity} classes that make up this {@link Change}'s constituent * {@link FeatureChange}s. * * @param featureChangeFormat * the format type for the the constituent {@link FeatureChange}s * @param completeEntityFormat * the format type for the constituent {@link CompleteEntity}s * @param truncate * whether or not to truncate long fields * @return the pretty string */ public String prettify(final PrettifyStringFormat featureChangeFormat, final PrettifyStringFormat completeEntityFormat, final boolean truncate) { final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " ["); builder.append("\n"); for (final FeatureChange featureChange : this.featureChanges) { builder.append( featureChange.prettify(featureChangeFormat, completeEntityFormat, truncate)); builder.append("\n"); } builder.append("]"); return builder.toString(); } /** * Save a JSON representation of that feature change. * * @param resource * The {@link WritableResource} to save the JSON to. */ public void save(final WritableResource resource) { new ChangeGeoJsonSerializer().accept(this, resource); } /** * Save a JSON representation of that change. * * @param resource * The {@link WritableResource} to save the JSON to. * @param showDescription * whether or not to show the {@link ChangeDescription} for each component * {@link FeatureChange} */ public void save(final WritableResource resource, final boolean showDescription) { new ChangeGeoJsonSerializer(true, showDescription).accept(this, resource); } public String summaryString() { final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " ["); builder.append("\n"); for (final ItemType itemType : ItemType.values()) { for (final ChangeDescriptorType changeType : ChangeDescriptorType.values()) { builder.append(itemType); builder.append(" had "); builder.append(changeCountFor(itemType, changeType)); builder.append(" "); builder.append(changeType); builder.append(" changes\n"); } } builder.append("]"); return builder.toString(); } public Map>> tagCountMap() { final Map>> tagMap = new EnumMap<>( ItemType.class); for (final ItemType itemType : ItemType.values()) { final Map> descriptorMap = new EnumMap<>( ChangeDescriptorType.class); for (final ChangeDescriptorType type : ChangeDescriptorType.values()) { descriptorMap.put(type, new HashMap<>()); } tagMap.put(itemType, descriptorMap); // The first stream here gets all update changes with Tag changes; the second // iterates over those changes and places them in the map based on their tag key // and update type this.featureChanges.stream().filter(change -> change.getItemType().equals(itemType) && change.explain().getChangeDescriptorType() .equals(ChangeDescriptorType.UPDATE) && change.explain().getChangeDescriptors().stream().anyMatch( descriptor -> descriptor.getName().equals(ChangeDescriptorName.TAG))) .forEach( change -> change.explain().getChangeDescriptors().stream() .filter(changeDescriptor -> changeDescriptor.getName() .equals(ChangeDescriptorName.TAG)) .forEach(changeDescriptor -> { final TagChangeDescriptor tagChangeDescriptor = (TagChangeDescriptor) changeDescriptor; if (tagMap.get(itemType) .get(tagChangeDescriptor.getChangeDescriptorType()) .containsKey(tagChangeDescriptor.getKey())) { tagMap.get(itemType) .get(tagChangeDescriptor .getChangeDescriptorType()) .get(tagChangeDescriptor.getKey()) .incrementAndGet(); } else { tagMap.get(itemType) .get(tagChangeDescriptor .getChangeDescriptorType()) .put(tagChangeDescriptor.getKey(), new AtomicLong(1)); } })); } return tagMap; } public String toJson() { return new ChangeGeoJsonSerializer().convert(this); } public String toJson(final boolean showDescription) { return new ChangeGeoJsonSerializer(true, showDescription).convert(this); } public String toLineDelimitedFeatureChanges(final boolean sorted) { final StringBuilder builder = new StringBuilder(); final FeatureChangeGeoJsonSerializer serializer = new FeatureChangeGeoJsonSerializer(false); final List sortedFeatureChanges = new ArrayList<>(this.getFeatureChanges()); if (sorted) { Collections.sort(sortedFeatureChanges); } for (final FeatureChange featureChange : sortedFeatureChanges) { builder.append(serializer.apply(featureChange) + "\n"); } return builder.toString(); } public String toLineDelimitedFeatureChanges() { return toLineDelimitedFeatureChanges(false); } @Override public String toString() { final StringList split = new StringList(); final StringBuilder builder = new StringBuilder(); for (int index = 0; index < this.featureChanges.size(); index++) { split.add(index + " - " + this.featureChanges.get(index)); } builder.append("[Change:"); builder.append(System.lineSeparator()); builder.append(split.join(System.lineSeparator())); builder.append(System.lineSeparator()); builder.append("]"); return builder.toString(); } public Change withName(final String name) { this.name = name; return this; } protected Change add(final FeatureChange featureChange) { final int currentIndex = this.featureChanges.size(); final AtlasEntityKey key = AtlasEntityKey.from(featureChange.getItemType(), featureChange.getIdentifier()); FeatureChange chosen = featureChange; if (!this.identifierToIndex.containsKey(key)) { this.identifierToIndex.put(key, currentIndex); this.featureChanges.add(featureChange); } else { final int existingIndex = this.identifierToIndex.get(key); final FeatureChange existing = this.featureChanges.get(existingIndex); logger.trace( "Change already has a similar feature change. Triggered a merge attempt! Existing: {}; New: {}", existing, featureChange); chosen = existing.merge(featureChange); this.featureChanges.set(existingIndex, chosen); } final Rectangle featureBounds = chosen.bounds(); if (this.bounds != null) { this.bounds = featureBounds != null ? this.bounds.combine(featureBounds) : this.bounds; } else { this.bounds = featureBounds; } return this; } protected Change addAll(final FeatureChange... featureChanges) { Arrays.stream(featureChanges).forEach(this::add); return this; } private long changeCountFor(final ItemType itemType, final ChangeDescriptorType changeType) { return this.featureChanges.stream().filter(change -> change.getItemType().equals(itemType) && change.explain().getChangeDescriptorType().equals(changeType)).count(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Area} that references a {@link ChangeAtlas}. That {@link Area} makes sure that all the * parent {@link Relation}s are {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangeArea extends Area // NOSONAR { private static final long serialVersionUID = -5658471275390043045L; // At most one of those two can be null. Not using Optional here as it is not Serializable. private final Area source; private final Area override; // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); protected ChangeArea(final ChangeAtlas atlas, final Area source, final Area override) { super(atlas); this.source = source; this.override = override; } @Override public Polygon asPolygon() { return attribute(Area::asPolygon, "polygon"); } @Override public long getIdentifier() { return attribute(Area::getIdentifier, "identifier"); } @Override public Map getTags() { return attribute(Area::getTags, "tags"); } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.LongFunction; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.AbstractAtlas; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.validators.AtlasValidator; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * Shallow atlas view that applies a set of change objects and presents the result without updating * the whole dataset. * * @author matthieun */ public class ChangeAtlas extends AbstractAtlas // NOSONAR { private static final long serialVersionUID = -5741815439928958165L; private static final ChangeRelation NULL_PLACEHOLDER_RELATION = new ChangeRelation(null, null, null); private static final ChangeNode NULL_PLACEHOLDER_NODE = new ChangeNode(null, null, null); private static final ChangeEdge NULL_PLACEHOLDER_EDGE = new ChangeEdge(null, null, null); private static final ChangeArea NULL_PLACEHOLDER_AREA = new ChangeArea(null, null, null); private static final ChangeLine NULL_PLACEHOLDER_LINE = new ChangeLine(null, null, null); private static final ChangePoint NULL_PLACEHOLDER_POINT = new ChangePoint(null, null, null); private final Change change; private final Atlas source; private String name; private boolean validated = false; private transient Rectangle bounds; private transient AtlasMetaData metaData; private transient Long numberOfNodes; private transient Long numberOfEdges; private transient Long numberOfAreas; private transient Long numberOfLines; private transient Long numberOfPoints; private transient Long numberOfRelations; // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map relationsCache; private transient Object relationsCacheLock = new Object(); // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map nodesCache; private transient Object nodesCacheLock = new Object(); // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map edgesCache; private transient Object edgesCacheLock = new Object(); // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map areasCache; private transient Object areasCacheLock = new Object(); // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map linesCache; private transient Object linesCacheLock = new Object(); // Computing relations in ChangeAtlas is very expensive, so we cache them here. private transient Map pointsCache; private transient Object pointsCacheLock = new Object(); private static void checkChanges(final Change... changes) { if (changes == null) { throw new CoreException("Change cannot be null in a ChangeAtlas."); } if (changes.length < 1) { throw new CoreException("ChangeAtlas has to have at least one Change."); } } private static void checkSource(final Atlas source) { if (source == null) { throw new CoreException("Source Atlas cannot be null in a ChangeAtlas."); } } public ChangeAtlas(final Atlas source, final Change... changes) { this(source, "", changes); } public ChangeAtlas(final Atlas source, final String name, final Change... changes) { checkSource(source); checkChanges(changes); this.change = Change.merge(changes); this.source = source; this.name = name == null || name.isEmpty() ? source.getName() : name; this.validate(); } public ChangeAtlas(final Change... changes) { this("", changes); } public ChangeAtlas(final String name, final Change... changes) { checkChanges(changes); final Change changeInternal = Change.merge(changes); boolean valid = false; Atlas sourceInternal = null; FeatureChange dummy = null; for (final FeatureChange featureChange : changeInternal.getFeatureChanges()) { if (featureChange.getChangeType() == ChangeType.ADD) { if (!featureChange.afterViewIsFull()) { throw new CoreException( "ChangeAtlas needs all ADD featureChanges to be full (no partial after view) to exist with no source Atlas."); } if (sourceInternal == null) { final PackedAtlasBuilder builder = new PackedAtlasBuilder(); builder.addPoint(-1L, Location.CENTER, Maps.hashMap()); sourceInternal = builder.get(); dummy = FeatureChange .remove(CompletePoint.shallowFrom(sourceInternal.point(-1L))); } valid = true; } } if (valid) { final ChangeBuilder changeBuilder = new ChangeBuilder(); changeBuilder.addAll(changeInternal.changes()); changeBuilder.add(dummy); this.change = changeBuilder.get(); this.source = sourceInternal; this.name = name == null || name.isEmpty() ? sourceInternal.getName() : name; new AtlasValidator(this).validate(); } else { throw new CoreException( "ChangeAtlas needs at least a full ADD featureChange to exist with no source Atlas."); } } @Override public Area area(final long identifier) { final Supplier creator = () -> entityFor(identifier, ItemType.AREA, () -> this.source.area(identifier), (sourceEntity, overrideEntity) -> new ChangeArea(this, (Area) sourceEntity, (Area) overrideEntity)); return getFromCacheOrCreate(this.areasCache, cache -> this.areasCache = cache, this.areasCacheLock, NULL_PLACEHOLDER_AREA, identifier, creator); } @Override public Iterable areas() { return entitiesFor(ItemType.AREA, this::area, this.source.areas()); } @Override public synchronized Rectangle bounds() { if (this.bounds == null) { // Stream it to make sure the "Iterable" signature is used here (vs. Located, which // would stack overflow). this.bounds = Rectangle.forLocated(Iterables.stream(this)); } return this.bounds; } @Override public Edge edge(final long identifier) { /* * If the edge was not found in this atlas, return null. Additionally, we then check to see * if this edge is missing a start or end node (which may have been removed by a * FeatureChange). In this case, we also want to "remove" the edge by returning null. */ final Predicate nullableEdge = edge -> edge.start() == null || edge.end() == null; final Supplier creator = () -> entityFor(identifier, ItemType.EDGE, () -> this.source.edge(identifier), (sourceEntity, overrideEntity) -> new ChangeEdge(this, (Edge) sourceEntity, (Edge) overrideEntity)); return getFromCacheOrCreate(this.edgesCache, cache -> this.edgesCache = cache, this.edgesCacheLock, NULL_PLACEHOLDER_EDGE, identifier, creator, Optional.of(nullableEdge)); } @Override public Iterable edges() { return entitiesFor(ItemType.EDGE, this::edge, this.source.edges()); } @Override public String getName() { if (this.name == null) { return super.getName(); } return this.name; } @Override public Line line(final long identifier) { final Supplier creator = () -> entityFor(identifier, ItemType.LINE, () -> this.source.line(identifier), (sourceEntity, overrideEntity) -> new ChangeLine(this, (Line) sourceEntity, (Line) overrideEntity)); return getFromCacheOrCreate(this.linesCache, cache -> this.linesCache = cache, this.linesCacheLock, NULL_PLACEHOLDER_LINE, identifier, creator); } @Override public Iterable lines() { return entitiesFor(ItemType.LINE, this::line, this.source.lines()); } @Override public synchronized AtlasMetaData metaData() { if (this.metaData == null) { AtlasMetaData sourceMetaData = this.source.metaData(); if (sourceMetaData == null) { sourceMetaData = new AtlasMetaData(); } final AtlasSize size = new AtlasSize(this); this.metaData = sourceMetaData.copyWithNewSize(size).copyWithNewOriginal(false); } return this.metaData; } @Override public Node node(final long identifier) { final Supplier creator = () -> entityFor(identifier, ItemType.NODE, () -> this.source.node(identifier), (sourceEntity, overrideEntity) -> new ChangeNode(this, (Node) sourceEntity, (Node) overrideEntity)); return getFromCacheOrCreate(this.nodesCache, cache -> this.nodesCache = cache, this.nodesCacheLock, NULL_PLACEHOLDER_NODE, identifier, creator); } @Override public Iterable nodes() { return entitiesFor(ItemType.NODE, this::node, this.source.nodes()); } @Override public synchronized long numberOfAreas() { if (this.numberOfAreas == null) { this.numberOfAreas = Iterables.size(areas()); } return this.numberOfAreas; } @Override public synchronized long numberOfEdges() { if (this.numberOfEdges == null) { this.numberOfEdges = Iterables.size(edges()); } return this.numberOfEdges; } @Override public synchronized long numberOfLines() { if (this.numberOfLines == null) { this.numberOfLines = Iterables.size(lines()); } return this.numberOfLines; } @Override public synchronized long numberOfNodes() { if (this.numberOfNodes == null) { this.numberOfNodes = Iterables.size(nodes()); } return this.numberOfNodes; } @Override public synchronized long numberOfPoints() { if (this.numberOfPoints == null) { this.numberOfPoints = Iterables.size(points()); } return this.numberOfPoints; } @Override public synchronized long numberOfRelations() { if (this.numberOfRelations == null) { this.numberOfRelations = Iterables.size(relations()); } return this.numberOfRelations; } @Override public Point point(final long identifier) { final Supplier creator = () -> entityFor(identifier, ItemType.POINT, () -> this.source.point(identifier), (sourceEntity, overrideEntity) -> new ChangePoint(this, (Point) sourceEntity, (Point) overrideEntity)); return getFromCacheOrCreate(this.pointsCache, cache -> this.pointsCache = cache, this.pointsCacheLock, NULL_PLACEHOLDER_POINT, identifier, creator); } @Override public Iterable points() { return entitiesFor(ItemType.POINT, this::point, this.source.points()); } @Override public Relation relation(final long identifier) { /* * If the relation was not found in this atlas, return null. Additionally, we check to see * if the relation has no members. If so, it is considered empty and is dropped from the * atlas. This logic, combined with the logic in ChangeRelation.membersFor, will * automatically handle removing non-empty but shallow relations as well. */ final Predicate nullableRelation = relationCandidate -> relationCandidate .members().isEmpty(); final Supplier creator = () -> entityFor(identifier, ItemType.RELATION, () -> this.source.relation(identifier), (sourceEntity, overrideEntity) -> new ChangeRelation(this, (Relation) sourceEntity, (Relation) overrideEntity)); return getFromCacheOrCreate(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, NULL_PLACEHOLDER_RELATION, identifier, creator, Optional.of(nullableRelation)); } @Override public Iterable relations() { return entitiesFor(ItemType.RELATION, this::relation, this.source.relations()); } public void validate() { if (!this.validated) { new AtlasValidator(this).validate(); this.validated = true; } } public ChangeAtlas withName(final String name) { this.name = name; return this; } /** * Get the {@link Iterable} of entities corresponding to the right type. This takes care of * surfacing only the ones not deleted, or if added or modified, the new ones. * * @param * The {@link AtlasEntity} subclass. * @param itemType * The type of entity * @param entityForIdentifier * A function that creates a new object from its identifier. * @param sourceEntities * All the corresponding entities from the source atlas. * @return All the corresponding entities in this atlas. */ private Iterable entitiesFor(final ItemType itemType, final LongFunction entityForIdentifier, final Iterable sourceEntities) { return new MultiIterable<>( this.change.getFeatureChanges().stream() .filter(featureChange -> featureChange.getItemType() == itemType && featureChange.getChangeType() == ChangeType.ADD) .map(featureChange -> entityForIdentifier .apply(featureChange.getIdentifier())) .filter(Objects::nonNull).collect(Collectors.toList()), Iterables.stream(sourceEntities) .filter(entity -> !this.change.changeFor(itemType, entity.getIdentifier()) .isPresent()) .map(entity -> entityForIdentifier.apply(entity.getIdentifier())) .filter(Objects::nonNull).collect()); } /** * Build a "Change" feature for this {@link ChangeAtlas} by querying the change object for * matching features. Use the source atlas otherwise. * * @param * The type of the feature to be built. Has to extend {@link AtlasEntity}. * @param identifier * The feature identifier * @param itemType * The feature type * @param sourceSupplier * A supplier function that creates the entity from the source. Can return null if * the source atlas does not contain that feature. * @param entityConstructorFromSource * A function that takes the updated feature from the {@link Change} object, and * constructs a new ChangeItem from it, that attaches to this Atlas. * @return The ChangeItem that corresponds to that feature. Can be a ChangeNode, ChangeEdge, * etc. It links back to this Atlas. */ private M entityFor(final long identifier, final ItemType itemType, final Supplier sourceSupplier, final BiFunction entityConstructorFromSource) { final Optional itemChangeOption = this.change.changeFor(itemType, identifier); final AtlasEntity sourceItem = sourceSupplier.get(); if (itemChangeOption.isPresent()) { // That Entity is affected by a change final FeatureChange itemChange = itemChangeOption.get(); if (ChangeType.REMOVE == itemChange.getChangeType()) { return null; } else { // Create the ChangeItem from the change object (the override). The source item // might be null (In case of an ADD which is a create and not a modify) return entityConstructorFromSource.apply(sourceItem, itemChange.getAfterView()); } } else { if (sourceItem != null) { // Create the ChangeItem from the untouched source; the override is null return entityConstructorFromSource.apply(sourceItem, null); } } return null; } private E getFromCacheOrCreate(final Map cache, final Consumer> cacheSetter, final Object lock, final E nullPlaceholder, final Long identifier, final Supplier creator) { return getFromCacheOrCreate(cache, cacheSetter, lock, nullPlaceholder, identifier, creator, Optional.empty()); } /** * @param * The type of the entity returned. Intended to be a {@link ChangeArea}, * {@link ChangeNode}, etc. * @param cache * The cache to use to retrieve the entity * @param cacheSetter * A function that will set the cache not null in case it was null. * @param lock * The synchronization lock used for that specific type * @param nullPlaceholder * What placeholder in the cache specifies a null object at some identifier * @param identifier * The identifier to return * @param creator * A {@link Supplier} that provides the correct object for the specified identifier * above * @param entityNullable * A predicate that decides if a non null object should still return null. Example a * relation with no members. * @return */ private E getFromCacheOrCreate(final Map cache, final Consumer> cacheSetter, final Object lock, final E nullPlaceholder, final Long identifier, final Supplier creator, final Optional> entityNullable) { // Get or create the cache (in case it was null) final Map cacheIn = ChangeEntity.getOrCreateCache(cache, cacheSetter, lock, ConcurrentHashMap::new); E result; if (cacheIn.containsKey(identifier)) { // Retrieve an existing object result = cacheIn.get(identifier); result = result == nullPlaceholder ? null : result; } else { // Create a new object result = creator.get(); if (result == null || entityNullable.isPresent() && entityNullable.get().test(result)) { // If the created object is null, or nullable, use the null placeholder cacheIn.put(identifier, nullPlaceholder); result = null; } else { cacheIn.put(identifier, result); } } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Arrays; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.change.validators.ChangeValidator; /** * Construct a {@link Change}. This is a gatekeeper that ensures validation. * * @author matthieun * @author Yazad Khambata */ public class ChangeBuilder { private final Change change; private boolean open; /** * A factory-method to construct a new {@link ChangeBuilder}. Constructs a {@code new} * {@link ChangeBuilder} with default values. * * @return - a new ChangeBuilder. */ public static ChangeBuilder newInstance() { return new ChangeBuilder(); } public ChangeBuilder() { this.change = new Change(); this.open = true; } /** * @param featureChange * - the {@link FeatureChange} to add to the builder. * @return ChangeBuilder - returns itself to allow fluency in calls. */ public synchronized ChangeBuilder add(final FeatureChange featureChange) { if (!this.open) { throw new CoreException( "Cannot append to a Change object that has already been validated"); } this.change.add(featureChange); return this; } /** * @see #addAll(Stream) * @param featureChanges * - The featureChanges to add. * @return ChangeBuilder - returns itself to allow fluency in calls. */ public synchronized ChangeBuilder addAll(final FeatureChange... featureChanges) { return addAll(Arrays.stream(featureChanges)); } /** * @see #addAll(Stream) * @param featureChanges * - The featureChanges to add. * @return ChangeBuilder - returns itself to allow fluency in calls. */ public synchronized ChangeBuilder addAll(final Iterable featureChanges) { return addAll(StreamSupport.stream(featureChanges.spliterator(), false)); } /** * Iteratively {@link #add(FeatureChange)} all the FeatureChanges. * * @param featureChanges * - The featureChanges to add. * @return ChangeBuilder - returns itself to allow fluency in calls. */ public synchronized ChangeBuilder addAll(final Stream featureChanges) { featureChanges.forEach(this::add); return this; } public synchronized Change get() { new ChangeValidator(this.change).validate(); this.open = false; return this.change; } public synchronized int peekNumberOfChanges() { return this.change.changeCount(); } /** * Assign a name to the change being constructed. * * @param name * - a name for the change. * @return ChangeBuilder - returns itself to allow fluency in calls. */ public ChangeBuilder withName(final String name) { this.change.withName(name); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Edge} that references a {@link ChangeAtlas}. That {@link Edge} makes sure that all the * connected {@link Node}s are {@link ChangeNode}s, and that all the parent {@link Relation}s are * {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangeEdge extends Edge // NOSONAR { private static final long serialVersionUID = -5658471275390043045L; // At most one of those two can be null. Not using Optional here as it is not Serializable. private final Edge source; private final Edge override; // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); // Computing Start Node is very expensive, so we cache it here. private transient Node startNodeCache; private transient Object startNodeCacheLock = new Object(); // Computing End Node is very expensive, so we cache it here. private transient Node endNodeCache; private transient Object endNodeCacheLock = new Object(); protected ChangeEdge(final ChangeAtlas atlas, final Edge source, final Edge override) { super(atlas); this.source = source; this.override = override; } @Override public PolyLine asPolyLine() { return attribute(Edge::asPolyLine, "polyLine"); } @Override public Node end() { final Supplier creator = () -> getChangeAtlas().node(endNodeIdentifier()); return ChangeEntity.getOrCreateCache(this.endNodeCache, cache -> this.endNodeCache = cache, this.endNodeCacheLock, creator); } public long endNodeIdentifier() { return attribute(Edge::end, "end node").getIdentifier(); } @Override public long getIdentifier() { return attribute(Edge::getIdentifier, "identifier"); } @Override public Map getTags() { return attribute(Edge::getTags, "tags"); } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } @Override public Node start() { final Supplier creator = () -> getChangeAtlas().node(startNodeIdentifier()); return ChangeEntity.getOrCreateCache(this.startNodeCache, cache -> this.startNodeCache = cache, this.startNodeCacheLock, creator); } public long startNodeIdentifier() { return attribute(Edge::start, "start node").getIdentifier(); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Utility class for Change entities: ChangeNode, ChangeEdge, etc. * * @author matthieun */ public final class ChangeEntity { /** * Filter parent relations that are mentioned in a ChangeEntity to only those that are not null. * * @param listed * The relation set to filter * @param parent * The parent {@link ChangeAtlas} * @return the set of {@link ChangeRelation} that are not null. */ static Set filterRelations(final Set listed, final ChangeAtlas parent) { return listed.stream().map(relation -> parent.relation(relation.getIdentifier())) .filter(Objects::nonNull).collect(Collectors.toSet()); } /** * Get either the attribute asked from the source entity * * @param source * The source entity * @param memberExtractor * Extract the member attribute from that entity * @param * The entity type that some object will be extracted from * @param * The object type that will be extracted and returned * @return The corresponding attribute */ static T getAttribute(final M source, final Function memberExtractor) { T result = null; if (source != null) { result = memberExtractor.apply(source); } return result; } /** * Get all the available attributes asked from the change entity (override), and/or from the * backup entity. * * @param name * The name of the extraction operation * @param source * The source entity * @param override * The change entity (override) * @param memberExtractor * Extract the member attribute from that entity * @param * The entity type to extract information from * @param * The type that will be extracted from the entity * @return The corresponding attribute list. Will not be empty. */ static List getAttributeAndOptionallyBackup(final M source, final M override, final Function memberExtractor, final String name) { final List result = new ArrayList<>(); if (override != null) { final T member = memberExtractor.apply(override); if (member != null) { result.add(member); } } if (source != null) { final T member = memberExtractor.apply(source); if (member != null) { result.add(member); } } if (result.isEmpty()) { throw new CoreException( "Could not retrieve attribute \"{}\" from override nor source!\noverride: {}\nsource:{}", name, override, source); } return result; } /** * Get either the attribute asked from the change entity (override), or from the backup entity * if unavailable. * * @param name * The name of the extraction operation * @param source * The source entity * @param override * The change entity (override) * @param memberExtractor * Extract the member attribute from that entity * @param * The atlas entity type that will be extracted from * @param * The expected return type * @return The corresponding attribute */ static T getAttributeOrBackup(final M source, final M override, final Function memberExtractor, final String name) { T result = null; if (override != null) { result = memberExtractor.apply(override); } if (result == null && source != null) { result = memberExtractor.apply(source); } if (result == null) { throw new CoreException( "Could not retrieve attribute \"{}\" from override nor source!\noverride: {}\nsource:{}", name, override, source); } return result; } /** * @param * The cached value type * @param fieldCache * The cache * @param cacheSetter * A function that will set the cache not null in case it was null. * @param lock * The synchronization lock to access the cache * @param creator * The original creator of the type if the cache does not contain it. * @return Either the cached value or the freshly created one. */ static V getOrCreateCache(final V fieldCache, final Consumer cacheSetter, final Object lock, final Supplier creator) { V localRelationCache = fieldCache; if (localRelationCache == null) { synchronized (lock) // NOSONAR { localRelationCache = fieldCache; // NOSONAR if (localRelationCache == null) // NOSONAR { localRelationCache = creator.get(); cacheSetter.accept(localRelationCache); } } } return localRelationCache; } private ChangeEntity() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Line} that references a {@link ChangeAtlas}. That {@link Line} makes sure that all the * parent {@link Relation}s are {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangeLine extends Line // NOSONAR { private static final long serialVersionUID = -5658471275390043045L; // At most one of those two can be null. Not using Optional here as it is not Serializable. private final Line source; private final Line override; // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); protected ChangeLine(final ChangeAtlas atlas, final Line source, final Line override) { super(atlas); this.source = source; this.override = override; } @Override public PolyLine asPolyLine() { return attribute(Line::asPolyLine, "polyLine"); } @Override public long getIdentifier() { return attribute(Line::getIdentifier, "identifier"); } @Override public Map getTags() { return attribute(Line::getTags, "tags"); } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link Node} that references a {@link ChangeAtlas}. That {@link Node} makes sure that all the * connected {@link Edge}s are {@link ChangeEdge}s, and that all the parent {@link Relation}s are * {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangeNode extends Node // NOSONAR { private static final long serialVersionUID = 4353679260691518275L; private static final Logger logger = LoggerFactory.getLogger(ChangeNode.class); private final Node source; private final Node override; // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); // Computing In Edges is very expensive, so we cache it here. private transient SortedSet inEdgesCache; private transient Object inEdgesCacheLock = new Object(); // Computing Out Edges is very expensive, so we cache it here. private transient SortedSet outEdgesCache; private transient Object outEdgesCacheLock = new Object(); protected ChangeNode(final ChangeAtlas atlas, final Node source, final Node override) { super(atlas); this.source = source; this.override = override; } @Override public long getIdentifier() { return attribute(Node::getIdentifier, "identifier"); } @Override public Location getLocation() { return attribute(Node::getLocation, "location"); } @Override public Map getTags() { return attribute(Node::getTags, "tags"); } public SortedSet inEdgeIdentifiers() { final List> inEdgeSetList = allAvailableAttributes(Node::inEdges, "in edges"); Set mergedIdentifiers = inEdgeSetList.stream().flatMap(Set::stream) .map(Edge::getIdentifier) .filter(edgeIdentifier -> getChangeAtlas().edge(edgeIdentifier) != null) .collect(Collectors.toSet()); if (this.override != null) { /* * We need to filter any identifiers that were marked as explicitly excluded in the * override. It is possible that the atlas view used to generate a FeatureChange context * will differ from the atlas on which the FeatureChange will be applied. In that case, * we must distinguish between two kinds of missing edge identifiers: 1) those that are * missing because a shard simply couldn't see them and 2) those that are missing * because a FeatureChange explicitly removed them. */ final CompleteNode completeNodeOverride = (CompleteNode) this.override; mergedIdentifiers = mergedIdentifiers.stream() .filter(edgeIdentifier -> !completeNodeOverride .explicitlyExcludedInEdgeIdentifiers().contains(edgeIdentifier)) .collect(Collectors.toSet()); } return new TreeSet<>(mergedIdentifiers); } @Override public SortedSet inEdges() { final Supplier> creator = () -> inEdgeIdentifiers().stream() .map(edgeIdentifier -> getChangeAtlas().edge(edgeIdentifier)) .collect(Collectors.toCollection(TreeSet::new)); return ChangeEntity.getOrCreateCache(this.inEdgesCache, cache -> this.inEdgesCache = cache, this.inEdgesCacheLock, creator); } public SortedSet outEdgeIdentifiers() { final List> outEdgeSetList = allAvailableAttributes(Node::outEdges, "out edges"); Set mergedIdentifiers = outEdgeSetList.stream().flatMap(Set::stream) .map(Edge::getIdentifier) .filter(edgeIdentifier -> getChangeAtlas().edge(edgeIdentifier) != null) .collect(Collectors.toSet()); if (this.override != null) { /* * We need to filter any identifiers that were marked as explicitly excluded in the * override. It is possible that the atlas view used to generate a FeatureChange context * will differ from the atlas on which the FeatureChange will be applied. In that case, * we must distinguish between two kinds of missing edge identifiers: 1) those that are * missing because a shard simply couldn't see them and 2) those that are missing * because a FeatureChange explicitly removed them. */ final CompleteNode completeNodeOverride = (CompleteNode) this.override; mergedIdentifiers = mergedIdentifiers.stream() .filter(edgeIdentifier -> !completeNodeOverride .explicitlyExcludedOutEdgeIdentifiers().contains(edgeIdentifier)) .collect(Collectors.toSet()); } return new TreeSet<>(mergedIdentifiers); } @Override public SortedSet outEdges() { final Supplier> creator = () -> outEdgeIdentifiers().stream() .map(edgeIdentifier -> getChangeAtlas().edge(edgeIdentifier)) .collect(Collectors.toCollection(TreeSet::new)); return ChangeEntity.getOrCreateCache(this.outEdgesCache, cache -> this.outEdgesCache = cache, this.outEdgesCacheLock, creator); } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } private List allAvailableAttributes( final Function memberExtractor, final String name) { return ChangeEntity.getAttributeAndOptionallyBackup(this.source, this.override, memberExtractor, name); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangePoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Point} that references a {@link ChangeAtlas}. That {@link Point} makes sure that all the * parent {@link Relation}s are {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangePoint extends Point // NOSONAR { private static final long serialVersionUID = 4353679260691518275L; private final Point source; private final Point override; // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); protected ChangePoint(final ChangeAtlas atlas, final Point source, final Point override) { super(atlas); this.source = source; this.override = override; } @Override public long getIdentifier() { return attribute(Point::getIdentifier, "identifier"); } @Override public Location getLocation() { return attribute(Point::getLocation, "location"); } @Override public Map getTags() { return attribute(Point::getTags, "tags"); } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.util.GeometryFixer; import org.locationtech.jts.operation.overlayng.OverlayNG; import org.locationtech.jts.operation.polygonize.Polygonizer; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiLineStringConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPrecisionManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link Relation} that references a {@link ChangeAtlas}. That {@link Relation} makes sure that all * the member entitiess are "Change" types, and that all the parent {@link Relation}s are * {@link ChangeRelation}s. *

* NOSONAR here to avoid "Subclasses that add fields should override "equals" (squid:S2160)". Here * the equals from the parent works. * * @author matthieun */ public class ChangeRelation extends Relation // NOSONAR { private static final long serialVersionUID = 4353679260691518275L; private static final Logger logger = LoggerFactory.getLogger(ChangeRelation.class); private static final JtsMultiPolygonToMultiLineStringConverter converter = new JtsMultiPolygonToMultiLineStringConverter(); private final Relation source; private final Relation override; // Computing ChangeRelation members is very expensive, so we cache it here. private transient RelationMemberList membersCache; private transient Object membersCacheLock = new Object(); private transient Optional geometryCache; private transient Object geometryCacheLock = new Object(); // Computing Parent Relations is very expensive, so we cache it here. private transient Set relationsCache; private transient Object relationsCacheLock = new Object(); protected ChangeRelation(final ChangeAtlas atlas, final Relation source, final Relation override) { super(atlas); this.source = source; this.override = override; } @Override public RelationMemberList allKnownOsmMembers() { return membersFor( attribute(Relation::allKnownOsmMembers, "all known osm members").asBean()); } @Override public List allRelationsWithSameOsmIdentifier() { return attribute(Relation::allRelationsWithSameOsmIdentifier, "all relations with same osm identifier").stream() .map(relation -> getChangeAtlas().relation(relation.getIdentifier())) .collect(Collectors.toList()); } @Override public Optional asMultiPolygon() { if (!this.isGeometric()) { return Optional.empty(); } final Supplier> creator = () -> { if (this.override != null && ((CompleteRelation) this.override).isOverrideGeometry()) { return this.override.asMultiPolygon(); } else if (this.source != null) { final Optional sourceJtsGeometry = ChangeEntity .getAttribute(this.source, Relation::asMultiPolygon); if (sourceJtsGeometry.isPresent()) { // don't do anything to invalid geom if (!sourceJtsGeometry.get().isValid()) { return sourceJtsGeometry; } final org.locationtech.jts.geom.MultiPolygon sourceGeom; sourceGeom = sourceJtsGeometry.get(); final Set removed = removedMembers(); final Set added = addedMembers(); // if nothing was changed, return the original geometry if (removed.isEmpty() && added.isEmpty()) { return sourceJtsGeometry; } // get the constituent linework and remove the old geometry and add in the new // geometry Geometry updatedGeometry = converter.convert(sourceGeom); for (final Geometry memberGeometry : removed) { updatedGeometry = OverlayNG.overlay(updatedGeometry, memberGeometry, OverlayNG.DIFFERENCE); } for (final Geometry memberGeometry : added) { updatedGeometry = OverlayNG.overlay(updatedGeometry, memberGeometry, OverlayNG.UNION); } // turn it into a multipolygon, fixing if necessary final Polygonizer update = new Polygonizer(true); update.add(updatedGeometry); MultiPolygon built = converter.backwardConvert(new GeometryCollection( (Geometry[]) update.getPolygons() .toArray(new Polygon[update.getPolygons().size()]), JtsPrecisionManager.getGeometryFactory())); if (!built.isValid()) { final Geometry fixed = GeometryFixer.fix(built); if (fixed instanceof Polygon) { built = new MultiPolygon(new Polygon[] { (Polygon) fixed }, JtsPrecisionManager.getGeometryFactory()); } else if (fixed instanceof MultiPolygon) { built = (MultiPolygon) fixed; } else { throw new CoreException( "Fixed geometry {} included unexpected type! {}", fixed.toText(), fixed.getGeometryType()); } logger.error("Had to fix geometry for relation {}", this.getIdentifier()); } return Optional.ofNullable(built); } } else if (this.override != null && !((CompleteRelation) this.override).asMultiPolygon().isPresent()) { // new ChangeRelation that never had geometry-- reconstruct it ((CompleteRelation) this.override).updateGeometry(); } return attribute(Relation::asMultiPolygon, "geometry"); }; return ChangeEntity.getOrCreateCache(this.geometryCache, cache -> this.geometryCache = cache, this.geometryCacheLock, creator); } @Override public long getIdentifier() { return attribute(Relation::getIdentifier, "identifier"); } @Override public Map getTags() { return attribute(Relation::getTags, "tags"); } @Override public RelationMemberList members() { final Supplier creator = () -> { final List availableMemberLists = allAvailableAttributes( Relation::members, "members"); final RelationBean mergedMembersBean = availableMemberLists.stream() .map(RelationMemberList::asBean) .reduce(new RelationBean(), RelationBean::merge); final RelationBean filteredAndMergedMembersBean = new RelationBean(); mergedMembersBean.forEach(relationBeanItem -> { if (getChangeAtlas().entity(relationBeanItem.getIdentifier(), relationBeanItem.getType()) != null) { filteredAndMergedMembersBean.addItem(relationBeanItem); } }); return membersFor(filteredAndMergedMembersBean); }; return ChangeEntity.getOrCreateCache(this.membersCache, cache -> this.membersCache = cache, this.membersCacheLock, creator); } @Override public Long osmRelationIdentifier() { return attribute(Relation::osmRelationIdentifier, "osm relation identifier"); } public boolean preservedValidGeometry() { if (this.source != null && (!addedMembers().isEmpty() || !removedMembers().isEmpty())) { final Optional sourceGeom = this.source.asMultiPolygon(); if (sourceGeom.isPresent() && !sourceGeom.get().isEmpty() && sourceGeom.get().isValid()) { final Optional geom = this.asMultiPolygon(); return geom.isPresent() && !geom.get().isEmpty() && geom.get().isValid(); } } return true; } @Override public Set relations() { final Supplier> creator = () -> ChangeEntity .filterRelations(attribute(AtlasEntity::relations, "relations"), getChangeAtlas()); return ChangeEntity.getOrCreateCache(this.relationsCache, cache -> this.relationsCache = cache, this.relationsCacheLock, creator); } private Set addedMembers() { if (this.override == null) { return new HashSet<>(); } return ((CompleteRelation) this.override).getAddedGeometry(); } private List allAvailableAttributes( final Function memberExtractor, final String name) { return ChangeEntity.getAttributeAndOptionallyBackup(this.source, this.override, memberExtractor, name); } private T attribute(final Function memberExtractor, final String name) { return ChangeEntity.getAttributeOrBackup(this.source, this.override, memberExtractor, name); } private ChangeAtlas getChangeAtlas() { return (ChangeAtlas) getAtlas(); } private RelationMemberList membersFor(final RelationBean bean) { if (bean == null) { return null; } final List memberList = new ArrayList<>(); for (final RelationBeanItem item : bean) { final AtlasEntity memberChangeEntity = getChangeAtlas().entity(item.getIdentifier(), item.getType()); if (memberChangeEntity != null) { memberList.add( new RelationMember(item.getRole(), memberChangeEntity, getIdentifier())); } } return new RelationMemberList(memberList); } private Set removedMembers() { if (this.override == null) { return new HashSet<>(); } return ((CompleteRelation) this.override).getRemovedGeometry(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/ChangeType.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; /** * Types of changes. Here, MODIFY is omitted, as it can be translated to an ADD, for simplicity. * * @author matthieun */ public enum ChangeType { ADD, REMOVE } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/FeatureChange.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.exception.change.FeatureChangeMergeException; import org.openstreetmap.atlas.exception.change.MergeFailureType; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescription; import org.openstreetmap.atlas.geography.atlas.change.serializer.FeatureChangeGeoJsonSerializer; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLineItem; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLocationItem; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.complete.PrettifyStringFormat; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasObject; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.lightweight.LightEntity; import org.openstreetmap.atlas.geography.atlas.lightweight.LightPoint; import org.openstreetmap.atlas.geography.atlas.walker.OsmWayWalker; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Single feature change, does not include any consistency checks. *

* To add a new, non existing feature: {@link ChangeType} is ADD, and the included reference needs * to contain all the information related to that new feature. *

* To modify an existing feature: {@link ChangeType} is ADD, and the included reference needs to * contain the only the changed information related to that changed feature. *

* To remove an existing feature: {@link ChangeType} is REMOVE. The reference entity need only * contain the identifier of the feature to remove. *

* For all {@link FeatureChange}s, you may indirectly include a reference to the before view of the * entity using the {@link FeatureChange#add(AtlasEntity, Atlas)} and * {@link FeatureChange#remove(AtlasEntity, Atlas)} methods. Providing the atlas context allows * {@link FeatureChange} to perform more sophisticated merge logic. * * @author matthieun * @author lcram * @author Yazad Khambata */ public class FeatureChange implements Located, Taggable, Serializable, Comparable { /** * Options to use for the feature change */ public enum Options { /** This performs expensive calculations when {@link #withAtlasContext(Atlas)} is called */ OSC_IF_POSSIBLE } private static final long serialVersionUID = 9172045162819925515L; private final String featureChangeIdentifier = UUID.randomUUID().toString(); private final ChangeType changeType; private AtlasEntity beforeView; private final AtlasEntity afterView; private final Map metaData; /** * The collection will be empty, have one item, or have multiple items. */ private Collection nodes; private Map originalTags; private String osc; /** The options for this FeatureChange */ private final EnumSet options = EnumSet.noneOf(Options.class); /** * Create a new {@link ChangeType#ADD} {@link FeatureChange} with a given afterView. The * afterView should be a {@link CompleteEntity} that specifies how the newly added or modified * feature should look. For the modified case, the afterView {@link CompleteEntity} need only * contain the fields that were modified. For ADDs that are adding a brand new feature, it * should be fully populated. * * @param afterView * the after view {@link CompleteEntity} * @return the created {@link FeatureChange} */ public static FeatureChange add(final AtlasEntity afterView) { return new FeatureChange(ChangeType.ADD, afterView); } /** * Create a new {@link ChangeType#ADD} {@link FeatureChange} with a given after view. The * afterView should be a {@link CompleteEntity} that specifies how the newly added or modified * feature should look. For the modified case, the afterView {@link CompleteEntity} need only * contain the fields that were modified. For ADDs that are adding a brand new feature, it * should be fully populated. The atlasContext parameter creates a richer {@link FeatureChange} * that contains information on how the entity looked before the update. This allows for more * sophisticated merge logic. * * @param afterView * the after view {@link CompleteEntity} * @param atlasContext * the atlas context * @return the created {@link FeatureChange} */ public static FeatureChange add(final AtlasEntity afterView, final Atlas atlasContext) { return add(afterView, atlasContext, (Options) null); } /** * Create a new {@link ChangeType#ADD} {@link FeatureChange} with a given after view. The * afterView should be a {@link CompleteEntity} that specifies how the newly added or modified * feature should look. For the modified case, the afterView {@link CompleteEntity} need only * contain the fields that were modified. For ADDs that are adding a brand new feature, it * should be fully populated. The atlasContext parameter creates a richer {@link FeatureChange} * that contains information on how the entity looked before the update. This allows for more * sophisticated merge logic. * * @param afterView * the after view {@link CompleteEntity} * @param atlasContext * the atlas context * @param options * The options for this {@link FeatureChange} * @return the created {@link FeatureChange} */ public static FeatureChange add(final AtlasEntity afterView, final Atlas atlasContext, final Options... options) { return new FeatureChange(ChangeType.ADD, afterView).setOptions(options) .withAtlasContext(atlasContext); } /** * Create a new {@link ChangeType#REMOVE} {@link FeatureChange} using a given reference. The * reference can be a shallow {@link CompleteEntity}, i.e. containing only the identifier of the * feature to be removed. * * @param reference * the {@link CompleteEntity} to remove * @return the created {@link FeatureChange} */ public static FeatureChange remove(final AtlasEntity reference) { return new FeatureChange(ChangeType.REMOVE, reference); } /** * Create a new {@link ChangeType#REMOVE} {@link FeatureChange} using a given reference. The * reference can be a shallow {@link CompleteEntity}, i.e. containing only the identifier of the * feature to be removed. The atlasContext parameter creates a richer {@link FeatureChange} that * contains information on how the entity looked before the update. This allows for more * sophisticated merge logic. * * @param reference * the {@link CompleteEntity} to remove * @param atlasContext * the atlas context * @return the created {@link FeatureChange} */ public static FeatureChange remove(final AtlasEntity reference, final Atlas atlasContext) { return remove(reference, atlasContext, (Options) null); } /** * Create a new {@link ChangeType#REMOVE} {@link FeatureChange} using a given reference. The * reference can be a shallow {@link CompleteEntity}, i.e. containing only the identifier of the * feature to be removed. The atlasContext parameter creates a richer {@link FeatureChange} that * contains information on how the entity looked before the update. This allows for more * sophisticated merge logic. * * @param reference * the {@link CompleteEntity} to remove * @param atlasContext * the atlas context * @param options * The options for this {@link FeatureChange} * @return the created {@link FeatureChange} */ public static FeatureChange remove(final AtlasEntity reference, final Atlas atlasContext, final Options... options) { return new FeatureChange(ChangeType.REMOVE, reference).setOptions(options) .withAtlasContext(atlasContext); } /** * Get the polyline for a view * * @param atlas * The atlas to use (used if the view parameter is an Edge) * @param view * The view to use * @return A PolyLine */ @Nullable static PolyLine getPolyline(final Atlas atlas, final AtlasEntity view) { if (view instanceof Line) { return ((Line) view).asPolyLine(); } else if (view instanceof Area) { return ((Area) view).asPolygon(); } else if (view instanceof Edge && ((Edge) view).asPolyLine() != null) { // Edges are special. We kind of need to get all the edges for that way to make the full // polyline return new PolyLine(new OsmWayWalker(atlas.edge(((Edge) view).getMainEdgeIdentifier())) .collectEdges().stream().map(Edge::asPolyLine).flatMap(PolyLine::stream) .collect(Collectors.toList())); } // Despite `PolyLine` implementing Collection, we cannot return an empty PolyLine here (the // polyline fails to be created). return null; } /** * Get the tags from an entity * * @param entity * The entity to get tags from * @return The tags */ @Nonnull private static Map getTags(@Nullable final AtlasEntity entity) { if (entity != null) { return Optional.ofNullable(entity.getTags()).orElseGet(Collections::emptyMap); } return Collections.emptyMap(); } /** * Create a new {@link FeatureChange} with a given type and after view. * * @param changeType * the type, either ADD or REMOVE. * @param afterView * the after view of the changed entity */ public FeatureChange(final ChangeType changeType, final AtlasEntity afterView) { this(changeType, afterView, null); } /** * Create a new {@link FeatureChange} with a given type, after view, and before view. This * constructor is provided for exact control over the before view of a change. It is kept * package private, and is used for testing purposes only. * * @param changeType * the change type * @param afterView * the updated entity * @param beforeView * the before entity */ public FeatureChange(final ChangeType changeType, final AtlasEntity afterView, final AtlasEntity beforeView) { if (afterView == null) { throw new CoreException("After view cannot be null."); } if (!(afterView instanceof CompleteEntity)) { throw new CoreException( "FeatureChange afterView requires CompleteEntity, found reference of type {}", afterView.getClass().getName()); } if (beforeView != null && !(beforeView instanceof CompleteEntity)) { throw new CoreException( "FeatureChange beforeView requires CompleteEntity, found reference of type {}", beforeView.getClass().getName()); } if (changeType == null) { throw new CoreException("changeType cannot be null."); } this.changeType = changeType; this.afterView = afterView; this.beforeView = beforeView; if (this.afterView.bounds() == null) { throw new CoreException("afterView {} bounds was null for {}", this.afterView, this.toString()); } if (this.beforeView != null && this.beforeView.bounds() == null) { throw new CoreException("beforeView {} bounds was null for {}", this.beforeView, this.toString()); } this.validateNotShallow(); this.metaData = new HashMap<>(); } /** * Add a new key value pair to this FeatureChange's meta-data * * @param key * The key * @param value * The value */ public void addMetaData(final String key, final String value) { if (key == null) { throw new CoreException("Meta-Data key (value={}) cannot be null!", value); } if (value == null) { throw new CoreException("Meta-Data value (key={}) cannot be null!", key); } this.metaData.put(key, value); } /** * Check if this {@link FeatureChange}'s afterView is full. A full afterView is a * {@link CompleteEntity} that has all its fields set to non-null values. * * @return if this {@link FeatureChange} has a full afterView */ public boolean afterViewIsFull() { if (this.getAfterView().getTags() == null || this.getAfterView().relations() == null) { return false; } switch (this.getItemType()) { case NODE: final Node nodeReference = (Node) this.getAfterView(); if (nodeReference.inEdges() == null || nodeReference.outEdges() == null || nodeReference.getLocation() == null) { return false; } break; case EDGE: final Edge edgeReference = (Edge) this.getAfterView(); if (edgeReference.start() == null || edgeReference.end() == null || edgeReference.asPolyLine() == null) { return false; } break; case AREA: final Area areaReference = (Area) this.getAfterView(); if (areaReference.asPolygon() == null) { return false; } break; case LINE: final Line lineReference = (Line) this.getAfterView(); if (lineReference.asPolyLine() == null) { return false; } break; case POINT: final Point pointReference = (Point) this.getAfterView(); if (pointReference.getLocation() == null) { return false; } break; case RELATION: final Relation relationReference = (Relation) this.getAfterView(); if (relationReference.members() == null || relationReference.allKnownOsmMembers() == null || relationReference.allRelationsWithSameOsmIdentifier() == null) { return false; } break; default: throw new CoreException("Unknown Item Type {}", this.getItemType()); } return true; } @Override public Rectangle bounds() { final Rectangle updatedBounds = this.afterView.bounds(); if (updatedBounds == null) { throw new CoreException("Corrupted FeatureChange: afterView bounds were null"); } if (this.beforeView == null) { return updatedBounds; } if (this.beforeView.bounds() == null) { throw new CoreException("Corrupted FeatureChange: beforeView bounds were null"); } return Rectangle.forLocated(this.beforeView.bounds(), updatedBounds); } @Override public int compareTo(final FeatureChange otherFeatureChange) { return Comparator.comparing(FeatureChange::getChangeType) .thenComparing(FeatureChange::getItemType) .thenComparing(FeatureChange::getIdentifier).compare(this, otherFeatureChange); } @Override public boolean equals(final Object other) { if (other instanceof FeatureChange) { final FeatureChange that = (FeatureChange) other; return this.getChangeType() == that.getChangeType() && this.getAfterView().equals(that.getAfterView()); } return false; } /** * Return a {@link ChangeDescription} object that explains the differences represented by this * {@link FeatureChange}. * * @return the {@link ChangeDescription} representing this {@link FeatureChange} */ public ChangeDescription explain() { if (this.afterView == null) { throw new CoreException("Cannot explain a FeatureChange with a null afterView!"); } final var changeDescription = new ChangeDescription(this.getIdentifier(), this.getItemType(), this.beforeView, this.afterView, this.changeType, this.originalTags, this.nodes); if (this.osc != null) { changeDescription.setOsc(this.osc); } return changeDescription; } public AtlasEntity getAfterView() { return this.afterView; } public AtlasEntity getBeforeView() { return this.beforeView; } public ChangeType getChangeType() { return this.changeType; } public String getFeatureChangeIdentifier() { return this.featureChangeIdentifier; } public long getIdentifier() { return getAfterView().getIdentifier(); } public ItemType getItemType() { return getAfterView().getType(); } public Map getMetaData() { return new HashMap<>(this.metaData); } /** * Get a tag based on a key, taking the changes into account. * * @param key * - The tag key to look for. * @return - the changed value of the tag, if available. */ @Override public Optional getTag(final String key) { return this.getAfterView().getTag(key); } /** * Get the changed tags. * * @return Map - the changed tags. */ @Override public Map getTags() { return this.getAfterView().getTags(); } @Override public int hashCode() { if (this.afterView instanceof Relation) { return Objects.hash(this.changeType, this.afterView, ((Relation) this.afterView).members()); } if (this.afterView instanceof Node) { return Objects.hash(this.changeType, this.afterView, ((Node) this.afterView).inEdges(), ((Node) this.afterView).outEdges()); } else { return Objects.hash(this.changeType, this.afterView, this.afterView.getTags()); } } /** * Merge two feature changes together. If it cannot succeed, this method will throw a * {@link CoreException} explaining why. * * @param other * The other to merge into this one. * @return The merged {@link FeatureChange} */ public FeatureChange merge(final FeatureChange other) { /* * FeatureChanges are mergeable under certain pre-conditions. If those pre-conditions are * satisfied, then we can proceed with attempting to merge the FeatureChanges. */ // Pre-conditions: // 1) The left and right FeatureChanges must be operating on the same entity identifier and // ItemType. Additionally, the ChangeType (i.e. ADD or REMOVE) must match. If these // conditions do not hold, there is no logical way to merge the FeatureChanges. // // 2) Either both FeatureChanges must provide a beforeView, or neither should provide one. // Attempting to merge two FeatureChanges where one has a beforeView and one does not // will always fail. We enforce this assumption in order to make the ADD/REMOVE merge logic // simpler. /* * Once basic mergeability is established, the merge logic proceeds: */ // Merging two REMOVE changes: // There is no need to merge the afterViews (since they are shallow), but we must ensure // that the beforeViews are properly merged. There are 3 possibilities, // outlined below. // // 1) Both FeatureChanges had fully populated, equivalent beforeViews, which are computed // automatically when a REMOVE FeatureChange is created (except possibly in the case of Node // and Relation, see 3) below) // // 2) Neither FeatureChange had a beforeView, in which case no merge is required. // // 3) In cases where the REMOVE is acting on a Relation, we first need to check if // there are inconsistencies in the beforeViews of members and allKnownOsmMembers. If the // REMOVE is acting on a Node, we need to check if there are inconsistencies in the // beforeViews of the in/out Edge identifier sets. Any inconsistencies must be merged. We // allow for inconsistencies in these specific cases, since it is possible that // FeatureChanges generated in different shards will have slightly different views of the // same Feature (since RelationMemberLists and in/out edge sets can be inconsistent across // shards). // // Merging two ADD changes: // In this case, we need to perform additional checks to ensure that the FeatureChanges can // indeed properly merge. We also must ensure that the potentially differing beforeViews can // merge. For more information on this, see // FeatureChangeMergingHelpers#mergeADDFeatureChangePair. FeatureChange result = this; try { // Pre-condition 1) if (this.getIdentifier() != other.getIdentifier() || this.getItemType() != other.getItemType()) { throw new FeatureChangeMergeException( MergeFailureType.FEATURE_CHANGE_INVALID_PROPERTIES_MERGE, "Cannot merge FeatureChanges with mismatching properties: [{}, {}, {}] vs [{}, {}, {}]", this.getIdentifier(), this.getItemType(), this.getChangeType(), other.getIdentifier(), other.getItemType(), other.getChangeType()); } // Pre-condition 1A) (we separate this one to provide a better exception) if (this.getIdentifier() == other.getIdentifier() && this.getItemType() == other.getItemType() && this.getChangeType() != other.getChangeType()) { throw new FeatureChangeMergeException( MergeFailureType.FEATURE_CHANGE_INVALID_ADD_REMOVE_MERGE, "Cannot merge FeatureChanges for [{}, {}], one is ADD and one is REMOVE", this.getIdentifier(), this.getItemType()); } // Pre-condition 2) if (this.getBeforeView() == null && other.getBeforeView() != null || this.getBeforeView() != null && other.getBeforeView() == null) { throw new FeatureChangeMergeException( MergeFailureType.FEATURE_CHANGE_IMBALANCED_BEFORE_VIEW, "One of the FeatureChanges was missing a beforeView - " + "cannot merge two FeatureChanges unless both either explicitly provide or explicitly exclude a beforeView, {} and {}", this.toString(), other.toString()); } // Actually merge the changes if (this.getChangeType() == ChangeType.REMOVE) { /* * Pre-condition 2 implies that if one beforeView is null, both are null so it is * safe to arbitrarily pick from the left or right side of the merge. */ if (this.getBeforeView() != null) { result = FeatureChangeMergingHelpers.mergeREMOVEFeatureChangePair(this, other); } } else if (this.getChangeType() == ChangeType.ADD) { result = FeatureChangeMergingHelpers.mergeADDFeatureChangePair(this, other); } else { // If we get here, something very unexpected happened. throw new CoreException("Unexpected merge failure for {} and {}", this.prettify(), other.prettify()); } } catch (final FeatureChangeMergeException exception) { final List newFailureTrace = exception .withNewTopLevelFailure(MergeFailureType.HIGHEST_LEVEL_MERGE_FAILURE); throw new FeatureChangeMergeException(newFailureTrace, "Cannot merge two feature changes:\n{}\nAND\n{}\nFailureTrace: {}", this.prettify(), other.prettify(), newFailureTrace, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException(MergeFailureType.HIGHEST_LEVEL_MERGE_FAILURE, "Cannot merge two feature changes:\n{}\nAND\n{}", this.prettify(), other.prettify(), exception); } FeatureChangeMergingHelpers.mergeMetaData(this, other).forEach(result::addMetaData); return result; } /** * Transform this {@link FeatureChange} into a pretty string. This will use the pretty strings * for {@link CompleteEntity} classes. By default, this method will use * {@link PrettifyStringFormat#MINIMAL_MULTI_LINE} for the {@link FeatureChange} itself, but * will use {@link PrettifyStringFormat#MINIMAL_SINGLE_LINE} for the constituent * {@link CompleteEntity}s. * * @return the pretty string */ public String prettify() { return this.prettify(PrettifyStringFormat.MINIMAL_MULTI_LINE, PrettifyStringFormat.MINIMAL_SINGLE_LINE); } /** * Transform this {@link FeatureChange} into a pretty string. This will use the pretty strings * for {@link CompleteEntity} classes. If you are unsure about which * {@link PrettifyStringFormat}s to use, try {@link FeatureChange#prettify()} which has some * sane defaults. * * @param format * the format type for the this {@link FeatureChange} * @param completeEntityFormat * the format type for the constituent {@link CompleteEntity}s * @return the pretty string */ public String prettify(final PrettifyStringFormat format, final PrettifyStringFormat completeEntityFormat) { return this.prettify(PrettifyStringFormat.MINIMAL_MULTI_LINE, PrettifyStringFormat.MINIMAL_SINGLE_LINE, true); } /** * Transform this {@link FeatureChange} into a pretty string. This will use the pretty strings * for {@link CompleteEntity} classes. If you are unsure about which * {@link PrettifyStringFormat}s to use, try {@link FeatureChange#prettify()} which has some * sane defaults. * * @param format * the format type for the this {@link FeatureChange} * @param completeEntityFormat * the format type for the constituent {@link CompleteEntity}s * @param truncate * whether or not to truncate long fields * @return the pretty string */ public String prettify(final PrettifyStringFormat format, final PrettifyStringFormat completeEntityFormat, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName()).append(" ").append("[").append(separator) .append("changeType: ").append(this.getChangeType()).append(", ").append(separator) .append("itemType: ").append(this.getItemType()).append(", ").append(separator) .append("identifier: ").append(this.getIdentifier()).append(", ").append(separator) .append("bounds: ").append(this.bounds()).append(", ").append(separator); if (this.beforeView != null) { builder.append("bfView: ").append( ((CompleteEntity) this.beforeView).prettify(completeEntityFormat, truncate)) .append(", ").append(separator); } builder.append("afView: ") .append(((CompleteEntity) this.afterView).prettify(completeEntityFormat, truncate)) .append(", ").append(separator).append("metadata: ").append(this.metaData) .append(separator).append(this.explain()).append(separator).append("]"); return builder.toString(); } /** * Save a GeoJSON representation of that feature change. * * @param resource * The {@link WritableResource} to save the GeoJSON to. */ public void save(final WritableResource resource) { new FeatureChangeGeoJsonSerializer(true).accept(this, resource); } /** * Save a GeoJSON representation of that feature change. * * @param resource * The {@link WritableResource} to save the GeoJSON to. * @param showDescription * whether or not to show the {@link ChangeDescription} */ public void save(final WritableResource resource, final boolean showDescription) { new FeatureChangeGeoJsonSerializer(true, showDescription).accept(this, resource); } /** * Set the options for this FeatureChange. This should be called as soon as possible, and always * before any method that the {@link FeatureChange.Options} specifies. * * @param options * the options to set. {@code null} clears the options. * @return {@code this}, for easy chaining */ public FeatureChange setOptions(final Options... options) { this.options.clear(); if (options != null) { Stream.of(options).filter(Objects::nonNull).forEach(this.options::add); } return this; } public String toGeoJson() { return new FeatureChangeGeoJsonSerializer(false).convert(this); } public String toGeoJson(final boolean showDescription) { return new FeatureChangeGeoJsonSerializer(false, showDescription).convert(this); } public String toPrettyGeoJson() { return new FeatureChangeGeoJsonSerializer(true, true).convert(this); } public String toPrettyGeoJson(final boolean showDescription) { return new FeatureChangeGeoJsonSerializer(true, showDescription).convert(this); } @Override public String toString() { return "FeatureChange [changeType: " + this.changeType + ", reference: {" + this.afterView.getType() + "," + this.afterView.getIdentifier() + "}, tags: " + getTags() + ", bounds: " + bounds() + "]"; } /** * Specify the Atlas on which this {@link FeatureChange} is based. {@link FeatureChange} objects * with a contextual Atlas are able to calculate their before view, and so are able to leverage * richer and more robust merging mechanics. * * @param atlas * the contextual atlas * @return the updated {@link FeatureChange} */ public FeatureChange withAtlasContext(final Atlas atlas) { this.computeBeforeViewUsingAtlasContext(atlas, this.changeType); if (this.options.contains(Options.OSC_IF_POSSIBLE)) { final long identifier = this.afterView.getIdentifier(); // Don't keep the original object, as this keeps the atlas alive if (this.afterView instanceof Line) { this.originalTags = getTags(atlas.line(identifier)); } else if (this.afterView instanceof Edge) { final var edge = atlas.edge(identifier); this.originalTags = edge == null ? null : getTags(edge.getMainEdge()); } else if (this.afterView instanceof Point) { this.originalTags = getTags(atlas.point(identifier)); } else if (this.afterView instanceof Node) { this.originalTags = getTags(atlas.node(identifier)); } else if (this.afterView instanceof Area) { this.originalTags = getTags(atlas.area(identifier)); } else if (this.afterView instanceof Relation) { this.originalTags = getTags(atlas.relation(identifier)); } this.computeRequiredOpenStreetMapChangeInformation(atlas, this.changeType); } return this; } /** * Use the OSC information for OpenStreetMap diffs. Used by deserialization. * * @param osc * The OSC to use * @return this, for easy chaining */ public FeatureChange withOsc(final String osc) { this.osc = osc; return this; } /** * Build the nodes needed for this feature change * * @param atlas * The atlas with the required nodes * @param locationsToFind * The locations to map to nodes in the atlas */ private void buildNodes(final Atlas atlas, final Collection locationsToFind) { this.nodes = new ArrayList<>(locationsToFind.size()); long currentNewId = -1; for (final Location point : locationsToFind) { final List localNodes = Iterables.asList(atlas.nodesAt(point)); final List nodePoints = Iterables.asList(atlas.pointsAt(point)); final List locationItems = Stream .concat(localNodes.stream(), nodePoints.stream()) .filter(LocationItem.class::isInstance).map(LocationItem.class::cast) .collect(Collectors.toList()); final long possibleNodes = locationItems.stream() .mapToLong(AtlasObject::getOsmIdentifier).distinct().count(); if (possibleNodes == 1) { // CompletePoint and CompleteNode both extend Point and Node respectively this.nodes.add((LocationItem) LightEntity.from(locationItems.get(0))); } else if (possibleNodes == 0) { // OK. New node. this.nodes.add(new LightPoint(currentNewId, point, Collections.emptySet())); currentNewId--; } else { // We cannot determine the nodes of the way. This will have to be manually edited. localNodes.clear(); break; } } } /** * Compute the beforeView using a given afterView and Atlas context. The beforeView is always a * CompleteEntity. For ChangeType.ADD, the beforeView will only contain references to fields * that were updated in the afterView. For ChangeType.REMOVE, the beforeView will be fully * populated. This will facilitate better debug printouts. * * @param atlas * the atlas context * @param changeType * the change type */ private void computeBeforeViewUsingAtlasContext(final Atlas atlas, final ChangeType changeType) { if (atlas == null) { throw new CoreException("Atlas context cannot be null for {}", this.toString()); } final AtlasEntity beforeViewUpdatesOnly; final AtlasEntity beforeViewFromAtlas = atlas.entity(this.afterView.getIdentifier(), this.afterView.getType()); /* * Check that the beforeViewFromAtlas is non-null. In case of REMOVE, this must be the case. * In case of ADD, it is possible the beforeViewFromAtlas is null when adding a brand new * feature. */ if (beforeViewFromAtlas == null && changeType != ChangeType.ADD) { throw new CoreException( "Could not find {} with ID {} in atlas context, ChangeType was {}", this.afterView.getType(), this.afterView.getIdentifier(), changeType); } /* * For the REMOVE case, we fully populate the beforeView and return. */ if (changeType == ChangeType.REMOVE) { if (beforeViewFromAtlas instanceof Edge && this.options.contains(Options.OSC_IF_POSSIBLE)) { // Edges are sectioned on a per-intersection basis, along with some other special // cases. // With OSC, we need to know the original way geometry. final List locations = new OsmWayWalker((Edge) beforeViewFromAtlas) .collectEdges().stream().map(Edge::asPolyLine).flatMap(PolyLine::stream) .collect(Collectors.toList()); this.beforeView = (AtlasEntity) CompleteEdge .from(((Edge) beforeViewFromAtlas).getMainEdge()).withGeometry(locations); } else { this.beforeView = CompleteEntity.from(beforeViewFromAtlas); } return; } /* * Otherwise, we continue with the ADD case. */ if (changeType != ChangeType.ADD) { throw new CoreException("Unknown ChangeType {}", changeType); } /* * If the beforeViewFromAtlas is null, then this is a brand new ADD. We just set the * beforeView to null and return. */ if (beforeViewFromAtlas == null) { this.beforeView = null; return; } /* * Make type specific updates first. */ if (this.afterView instanceof Area) { /* * Area specific updates. The only Area-specific field is the polygon. */ final Area afterAreaView = (Area) this.afterView; final Area beforeAreaViewFromAtlas = (Area) beforeViewFromAtlas; beforeViewUpdatesOnly = CompleteArea.shallowFrom(beforeAreaViewFromAtlas); if (afterAreaView.asPolygon() != null) { ((CompleteArea) beforeViewUpdatesOnly) .withPolygon(beforeAreaViewFromAtlas.asPolygon()); } } else if (this.afterView instanceof LineItem) { /* * LineItem specific updates. The LineItem-specific fields are the polyline, and the * start/end nodes in case of an Edge LineItem. */ final LineItem afterLineItemView = (LineItem) this.afterView; final LineItem beforeLineItemViewFromAtlas = (LineItem) beforeViewFromAtlas; beforeViewUpdatesOnly = CompleteEntity.shallowFrom(beforeLineItemViewFromAtlas); if (afterLineItemView.asPolyLine() != null) { ((CompleteLineItem) beforeViewUpdatesOnly) .withPolyLine(beforeLineItemViewFromAtlas.asPolyLine()); } if (this.afterView instanceof Edge) { final Edge afterEdgeView = (Edge) afterLineItemView; final Edge beforeEdgeViewFromAtlas = (Edge) beforeViewFromAtlas; if (afterEdgeView.start() != null) { ((CompleteEdge) beforeViewUpdatesOnly).withStartNodeIdentifier( beforeEdgeViewFromAtlas.start().getIdentifier()); } if (afterEdgeView.end() != null) { ((CompleteEdge) beforeViewUpdatesOnly) .withEndNodeIdentifier(beforeEdgeViewFromAtlas.end().getIdentifier()); } } } else if (this.afterView instanceof LocationItem) { /* * LocationItem specific updates. The LocationItem-specific fields are the location, and * the in/out edge sets in case of a Node LocationItem. */ final LocationItem afterLocationItemView = (LocationItem) this.afterView; final LocationItem beforeLocationItemViewFromAtlas = (LocationItem) beforeViewFromAtlas; beforeViewUpdatesOnly = CompleteEntity.shallowFrom(beforeLocationItemViewFromAtlas); if (afterLocationItemView.getLocation() != null) { ((CompleteLocationItem) beforeViewUpdatesOnly) .withLocation(beforeLocationItemViewFromAtlas.getLocation()); } if (this.afterView instanceof Node) { final Node afterNodeView = (Node) afterLocationItemView; final Node beforeNodeViewFromAtlas = (Node) beforeViewFromAtlas; if (afterNodeView.inEdges() != null) { ((CompleteNode) beforeViewUpdatesOnly) .withInEdges(beforeNodeViewFromAtlas.inEdges()); } if (afterNodeView.outEdges() != null) { ((CompleteNode) beforeViewUpdatesOnly) .withOutEdges(beforeNodeViewFromAtlas.outEdges()); } } } else if (this.afterView instanceof Relation) { /* * Relation specific updates. There are quite a few Relation specific fields: members, * allRelationsWithSameOsmIdentifier, allKnownOsmMembers, and osmRelationIdentifier. */ final Relation afterRelationView = (Relation) this.afterView; final Relation beforeRelationViewFromAtlas = (Relation) beforeViewFromAtlas; beforeViewUpdatesOnly = CompleteRelation.shallowFrom(afterRelationView); if (afterRelationView.members() != null) { ((CompleteRelation) beforeViewUpdatesOnly) .withMembers(beforeRelationViewFromAtlas.members()); } if (afterRelationView.allRelationsWithSameOsmIdentifier() != null) { ((CompleteRelation) beforeViewUpdatesOnly).withAllRelationsWithSameOsmIdentifier( beforeRelationViewFromAtlas.allRelationsWithSameOsmIdentifier().stream() .map(Relation::getIdentifier).collect(Collectors.toList())); } if (afterRelationView.allKnownOsmMembers() != null) { ((CompleteRelation) beforeViewUpdatesOnly).withAllKnownOsmMembers( beforeRelationViewFromAtlas.allKnownOsmMembers().asBean()); } if (afterRelationView.osmRelationIdentifier() != null) { ((CompleteRelation) beforeViewUpdatesOnly).withOsmRelationIdentifier( beforeRelationViewFromAtlas.osmRelationIdentifier()); } final Optional afterGeom = afterRelationView.asMultiPolygon(); final Optional beforeGeom = beforeRelationViewFromAtlas.asMultiPolygon(); if (afterGeom.isPresent() && beforeGeom.isPresent()) { ((CompleteRelation) beforeViewUpdatesOnly) .withMultiPolygonGeometry(beforeGeom.get()); } } else { throw new CoreException("Unknown entity type {}", this.afterView.getType()); } /* * Add before view of the tags if the updatedView updated the tags. */ final Map updatedViewTags = this.afterView.getTags(); if (updatedViewTags != null) { ((CompleteEntity) beforeViewUpdatesOnly).withTags(beforeViewFromAtlas.getTags()); } /* * Add before view of relations if updatedView updated relations. */ final Set updatedViewRelations = this.afterView.relations(); if (updatedViewRelations != null) { ((CompleteEntity) beforeViewUpdatesOnly).withRelations(beforeViewFromAtlas.relations()); } this.beforeView = beforeViewUpdatesOnly; } /** * Compute information needed for an OpenStreetMap Change file * * @param atlas * The atlas with all the needed information (all nodes, etc.) * @param changeType * The type of change */ private void computeRequiredOpenStreetMapChangeInformation(final Atlas atlas, final ChangeType changeType) { final Collection locationsToFind = new HashSet<>(); if (changeType == ChangeType.ADD) { if (Arrays.asList(ItemType.AREA, ItemType.EDGE, ItemType.LINE) .contains(this.afterView.getType())) { final PolyLine polyLine = getPolyline(atlas, this.afterView); if (polyLine == null) { return; } locationsToFind.addAll(polyLine); } } else if (changeType == ChangeType.REMOVE && Arrays.asList(ItemType.AREA, ItemType.EDGE, ItemType.LINE) .contains(this.afterView.getType())) // Only add remove points if there is no chance that a point is used by another // object { // In contrast with ChangeType.ADD, we must use the beforeView. final PolyLine polyLine = getPolyline(atlas, this.beforeView); if (polyLine == null) { return; } this.findNodesToRemove(atlas, polyLine, locationsToFind); } this.buildNodes(atlas, locationsToFind); } /** * Find nodes to remove * * @param atlas * The atlas with the information needed to determine if a node should be removed * @param polyLine * The polyline that we are deleting -- we check if the only parent of a node is this * line, and if so, remove it. * @param locationsToFind * The collection to add the locations to remove to */ private void findNodesToRemove(final Atlas atlas, final PolyLine polyLine, final Collection locationsToFind) { for (final Location point : polyLine) { final Rectangle pointBounds = point.bounds(); final Set lines = Iterables.asSet(atlas.lineItemsContaining(point)); if (this.afterView instanceof LineItem) { lines.removeIf( line -> line.getOsmIdentifier() == this.afterView.getOsmIdentifier()); } final Set areas = Iterables.asSet(atlas.areasIntersecting(pointBounds)); if (this.afterView instanceof Area) { areas.removeIf( area -> area.getOsmIdentifier() == this.afterView.getOsmIdentifier()); } final Set relations = Iterables .asSet(atlas.relationsWithEntitiesIntersecting(pointBounds)); atlas.relationsWithEntitiesWithin(point.bounds()).forEach(relations::add); if (this.afterView instanceof Relation) { relations.removeIf(relation -> relation.getOsmIdentifier() == this.afterView .getOsmIdentifier()); } if (lines.isEmpty() && relations.isEmpty() && areas.isEmpty()) { locationsToFind.add(point); } } } /** * Check that this {@link FeatureChange} is not shallow. A shallow {@link FeatureChange} is one * whose CompleteEntity only contains an identifier. */ private void validateNotShallow() { if (this.changeType == ChangeType.ADD && ((CompleteEntity) this.afterView).isShallow()) { throw new CoreException("{} was shallow (i.e. it contained only an identifier)", this); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/FeatureChangeBoundsExpander.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.utilities.maps.MultiMapWithSet; /** * Expand the size of bounds for features that belong to a relation, or for nodes that are connected * to edges, thus expanding the full geographical impact of a FeatureChange * * @author matthieun */ public class FeatureChangeBoundsExpander { private final Set featureChanges; private Atlas atlas; private final Predicate needsUpdate = featureChange -> { if (featureChange.getItemType() == ItemType.NODE) { return true; } if (featureChange.getChangeType() == ChangeType.ADD) { // For relation members, only look at removes return false; } final Set relations = featureChange.getAfterView().relations(); if (relations != null && !relations.isEmpty()) { return true; } final AtlasEntity entity = this.atlas.entity(featureChange.getIdentifier(), featureChange.getItemType()); return entity != null && !entity.relations().isEmpty(); }; private final Set result = new HashSet<>(); private final Set featureChangesToUpdate = new HashSet<>(); private final MultiMapWithSet typeIdentifierToExtensionBounds = new MultiMapWithSet<>(); private final Map typeIdentifierToFeatureChange = new HashMap<>(); public FeatureChangeBoundsExpander(final Set featureChanges, final Atlas atlas) { this.featureChanges = featureChanges; this.atlas = atlas; } public Set apply() { if (!this.result.isEmpty()) { throw new CoreException("Cannot apply the same bounds expander twice!"); } this.featureChanges.forEach(featureChange -> this.typeIdentifierToFeatureChange.put( AtlasEntityKey.from(featureChange.getItemType(), featureChange.getIdentifier()), featureChange)); findBounds(); for (final FeatureChange featureChange : this.featureChangesToUpdate) { final Set expansionRectangles = this.typeIdentifierToExtensionBounds .get(AtlasEntityKey.from(featureChange.getItemType(), featureChange.getIdentifier())); FeatureChange newFeatureChange = featureChange; if (expansionRectangles != null) { newFeatureChange = new FeatureChange(featureChange.getChangeType(), expanded(featureChange.getAfterView(), expansionRectangles), featureChange.getBeforeView()); } this.result.add(newFeatureChange); } return this.result; } private AtlasEntity expanded(final AtlasEntity other, final Set expansionRectangles) { final Rectangle newBounds = Rectangle.forLocated(expansionRectangles); if (other instanceof CompleteNode) { return ((CompleteNode) other).withBoundsExtendedBy(newBounds); } if (other instanceof CompleteEdge) { return ((CompleteEdge) other).withBoundsExtendedBy(newBounds); } if (other instanceof CompleteArea) { return ((CompleteArea) other).withBoundsExtendedBy(newBounds); } if (other instanceof CompleteLine) { return ((CompleteLine) other).withBoundsExtendedBy(newBounds); } if (other instanceof CompletePoint) { return ((CompletePoint) other).withBoundsExtendedBy(newBounds); } if (other instanceof CompleteRelation) { return ((CompleteRelation) other).withBoundsExtendedBy(newBounds); } throw new CoreException("AtlasEntity is of a non-workable type: {}", other.getClass().getName()); } private void findBounds() // NOSONAR { for (final FeatureChange featureChange : this.featureChanges) { final ItemType itemType = featureChange.getItemType(); if (this.needsUpdate.test(featureChange)) { this.featureChangesToUpdate.add(featureChange); } else { this.result.add(featureChange); } if (itemType == ItemType.RELATION) { findBoundsFromRelation((Relation) featureChange.getAfterView()); final Relation relationFromAtlas = this.atlas .relation(featureChange.getIdentifier()); if (relationFromAtlas != null) { findBoundsFromRelation(relationFromAtlas); } } if (itemType == ItemType.EDGE) { findBoundsFromEdge((Edge) featureChange.getAfterView()); final Edge edgeFromAtlas = this.atlas.edge(featureChange.getIdentifier()); if (edgeFromAtlas != null) { findBoundsFromEdge(edgeFromAtlas); } } } for (final Edge edge : this.atlas.edges()) { final AtlasEntityKey startKey = AtlasEntityKey.from(ItemType.NODE, edge.start().getIdentifier()); final AtlasEntityKey endKey = AtlasEntityKey.from(ItemType.NODE, edge.end().getIdentifier()); if (this.typeIdentifierToFeatureChange.containsKey(startKey) || this.typeIdentifierToFeatureChange.containsKey(endKey)) { findBoundsFromEdge(edge); } } for (final Relation relation : this.atlas.relations()) { final Set memberKeys = new HashSet<>(); relation.members().forEach(member -> memberKeys.add(AtlasEntityKey .from(member.getEntity().getType(), member.getEntity().getIdentifier()))); if (memberKeys.stream().anyMatch(this.typeIdentifierToFeatureChange::containsKey)) { findBoundsFromRelation(relation); } } } private void findBoundsFromEdge(final Edge edge) { final Node start = edge.start(); final Node end = edge.end(); final Rectangle bounds = edge.bounds(); if (start != null) { this.typeIdentifierToExtensionBounds .add(AtlasEntityKey.from(ItemType.NODE, start.getIdentifier()), bounds); } if (end != null) { this.typeIdentifierToExtensionBounds .add(AtlasEntityKey.from(ItemType.NODE, end.getIdentifier()), bounds); } } private void findBoundsFromRelation(final Relation relation) { final RelationMemberList members = relation.members(); if (members != null && !members.isEmpty()) { final Rectangle bounds = relation.bounds(); members.forEach(relationMember -> { final AtlasEntity entity = relationMember.getEntity(); this.typeIdentifierToExtensionBounds .add(AtlasEntityKey.from(entity.getType(), entity.getIdentifier()), bounds); }); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/FeatureChangeMergeGroup.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * Defines the guard-rails that guides the groups of {@link FeatureChange}s that can be merged * together in a {@link Change}. * * @author Yazad Khambata */ public class FeatureChangeMergeGroup { private final ItemType itemType; private final Long identifier; private final ChangeType changeType; public static FeatureChangeMergeGroup from(final FeatureChange featureChange) { return new FeatureChangeMergeGroup(featureChange.getItemType(), featureChange.getIdentifier(), featureChange.getChangeType()); } public FeatureChangeMergeGroup(final ItemType itemType, final Long identifier, final ChangeType changeType) { super(); this.itemType = itemType; this.identifier = identifier; this.changeType = changeType; } public ChangeType getChangeType() { return this.changeType; } public Long getIdentifier() { return this.identifier; } public ItemType getItemType() { return this.itemType; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/FeatureChangeMergingHelpers.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.change.MemberMerger.MergedMemberBean; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A utility class for the various {@link FeatureChange} merge helper functions. * * @author lcram */ public final class FeatureChangeMergingHelpers { static final String IN_EDGE_IDENTIFIERS_FIELD = "inEdgeIdentifiers"; static final String OUT_EDGE_IDENTIFIERS_FIELD = "outEdgeIdentifiers"; static final String GEOMETRIC_RELATIONS_FIELD = "geometricRelations"; private static final String AFTER_ENTITY_RIGHT_WAS_NULL = "afterEntityRight was null, this should never happen!"; private static final String AFTER_ENTITY_LEFT_WAS_NULL = "afterEntityLeft was null, this should never happen!"; private static final Logger logger = LoggerFactory.getLogger(FeatureChangeMergingHelpers.class); /** * Merge two {@link ChangeType#ADD} {@link FeatureChange}s into a single {@link FeatureChange}. * * @param left * the left {@link FeatureChange} * @param right * the right {@link FeatureChange} * @return the merged {@link FeatureChange}s */ public static FeatureChange mergeADDFeatureChangePair(final FeatureChange left, final FeatureChange right) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final MergedMemberBean> mergedTagsBean = new MemberMerger.Builder>() .withMemberName("tags").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight).withMemberExtractor(Taggable::getTags) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleTagMerger) .withAfterViewConsistentBeforeViewMerger(MemberMergeStrategies.diffBasedTagMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryTagMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryTagMerger).build() .mergeMember(); final MergedMemberBean> mergedParentRelationsBean = new MemberMerger.Builder>() .withMemberName("parentRelations").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> atlasEntity.relations() == null ? null : atlasEntity.relations().stream().map(Relation::getIdentifier) .collect(Collectors.toSet())) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSetMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongSetMerger).build() .mergeMember(); if (afterEntityLeft instanceof LocationItem) { return mergeLocationItems(left, right, mergedTagsBean, mergedParentRelationsBean); } else if (afterEntityLeft instanceof LineItem) { return mergeLineItems(left, right, mergedTagsBean, mergedParentRelationsBean); } else if (afterEntityLeft instanceof Area) { return mergeAreas(left, right, mergedTagsBean, mergedParentRelationsBean); } else if (afterEntityLeft instanceof Relation) { return mergeRelations(left, right, mergedTagsBean, mergedParentRelationsBean); } else { throw new CoreException("Unknown AtlasEntity subtype {}", afterEntityLeft.getClass().getName()); } } /** * Merge the meta-data of two {@link FeatureChange}s * * @param left * The left {@link FeatureChange} * @param right * The right {@link FeatureChange} * @return The concatenated meta-data map */ public static Map mergeMetaData(final FeatureChange left, final FeatureChange right) { final Map result = new HashMap<>(); final Map leftMap = left.getMetaData(); final Map rightMap = right.getMetaData(); for (final Map.Entry leftEntry : leftMap.entrySet()) { if (rightMap.containsKey(leftEntry.getKey())) { final List leftValues = Arrays.asList(leftEntry.getValue().split(",")); final List rightValues = Arrays .asList(rightMap.get(leftEntry.getKey()).split(",")); final String mergedValue; if (leftValues.equals(rightValues)) { mergedValue = new StringList(leftValues).join(","); } else { final SortedSet values = new TreeSet<>(); values.addAll(leftValues); values.addAll(rightValues); mergedValue = new StringList(values).join(","); } result.put(leftEntry.getKey(), mergedValue); } else { result.put(leftEntry.getKey(), leftEntry.getValue()); } } for (final Map.Entry rightEntry : rightMap.entrySet()) { if (!result.containsKey(rightEntry.getKey())) { result.put(rightEntry.getKey(), rightEntry.getValue()); } } return result; } /** * Merge two {@link ChangeType#REMOVE} {@link FeatureChange}s into a single * {@link FeatureChange}. This method only needs to handle merging the beforeViews, since the * afterViews would be null. Additionally, there are only a few beforeView fields that even need * to be merged in the first place, namely {@link RelationBean}s and {@link Node} in/out edge * identifier sets. * * @param left * the left {@link FeatureChange} * @param right * the right {@link FeatureChange} * @return the merged {@link FeatureChange}s */ public static FeatureChange mergeREMOVEFeatureChangePair(final FeatureChange left, final FeatureChange right) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); /* * For nodes, we need to merge the beforeViews of the in/out edge identifier sets. */ if (beforeEntityLeft instanceof Node) { final Node beforeNodeLeft = (Node) beforeEntityLeft; final Node beforeNodeRight = (Node) beforeEntityRight; final CompleteNode mergedBeforeNode = CompleteNode.from(beforeNodeLeft); final SortedSet leftInEdgeIdentifiers = new TreeSet<>(beforeNodeLeft.inEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())); final SortedSet rightInEdgeIdentifiers = new TreeSet<>(beforeNodeRight.inEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())); final SortedSet leftOutEdgeIdentifiers = new TreeSet<>(beforeNodeLeft.outEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())); final SortedSet rightOutEdgeIdentifiers = new TreeSet<>(beforeNodeRight.outEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())); if (!leftInEdgeIdentifiers.equals(rightInEdgeIdentifiers)) { mergedBeforeNode .withInEdgeIdentifiers(MemberMergeStrategies.simpleLongSortedSetMerger .apply(leftInEdgeIdentifiers, rightInEdgeIdentifiers)); } if (!leftOutEdgeIdentifiers.equals(rightOutEdgeIdentifiers)) { mergedBeforeNode .withOutEdgeIdentifiers(MemberMergeStrategies.simpleLongSortedSetMerger .apply(leftOutEdgeIdentifiers, rightOutEdgeIdentifiers)); } return new FeatureChange(ChangeType.REMOVE, left.getAfterView(), mergedBeforeNode); } /* * For relations, we need to merge the beforeViews of the members RelationBean. */ else if (beforeEntityLeft instanceof Relation) { final Relation beforeRelationLeft = (Relation) beforeEntityLeft; final Relation beforeRelationRight = (Relation) beforeEntityRight; final CompleteRelation mergedBeforeRelation = CompleteRelation.from(beforeRelationLeft); final RelationBean leftMembers = beforeRelationLeft.members().asBean(); final RelationBean rightMembers = beforeRelationRight.members().asBean(); if (!leftMembers.equalsIncludingExplicitlyExcluded(rightMembers)) { mergedBeforeRelation.withMembers(RelationBean.mergeBeans(leftMembers, rightMembers), Rectangle.forLocated(beforeRelationLeft, beforeRelationRight)); } return new FeatureChange(ChangeType.REMOVE, left.getAfterView(), mergedBeforeRelation); } /* * For any other case, there is no need to merge anything. Just arbitrarily return the left * side. */ else { return left; } } private static FeatureChange mergeAreas(final FeatureChange left, final FeatureChange right, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } /* * The polygon merger ensure that the afterEntity polygons either: 1) exactly match or 2) * are reconcilable based on the beforeView. Additionally, this step also ensures that the * beforeViews, if present, had equivalent geometry. */ final MergedMemberBean mergedPolygonBean = new MemberMerger.Builder() .withMemberName("polygon").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((Area) atlasEntity).asPolygon()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryPolygonMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedPolygonMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryPolygonMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryPolygonMerger).build() .mergeMember(); final MergedMemberBean> mergedGeometricParentRelationsBean = new MemberMerger.Builder>() .withMemberName(GEOMETRIC_RELATIONS_FIELD).withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor( atlasEntity -> ((CompleteArea) atlasEntity).geometricRelationIdentifiers()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSetMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongSetMerger).build() .mergeMember(); final CompleteArea mergedAfterArea = new CompleteArea(left.getIdentifier(), mergedPolygonBean.getMergedAfterMember(), mergedTagsBean.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterArea.withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedAfterMember()); mergedAfterArea.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterArea.withBoundsExtendedBy(afterEntityRight.bounds()); final CompleteArea mergedBeforeArea; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforeArea = CompleteArea.shallowFrom((Area) beforeEntityLeft) .withTags(mergedTagsBean.getMergedBeforeMember()) .withPolygon(mergedPolygonBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()) .withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforeArea = null; } return new FeatureChange(ChangeType.ADD, mergedAfterArea, mergedBeforeArea); } private static FeatureChange mergeEdges(final FeatureChange left, final FeatureChange right, final MergedMemberBean mergedPolyLineBean, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final MergedMemberBean mergedStartNodeIdentifierBean = new MemberMerger.Builder() .withMemberName("startNode").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(edge -> ((Edge) edge).start() == null ? null : ((Edge) edge).start().getIdentifier()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryLongMerger) .withAfterViewConsistentBeforeViewMerger(MemberMergeStrategies.diffBasedLongMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongMerger).build() .mergeMember(); final MergedMemberBean mergedEndNodeIdentifierBean = new MemberMerger.Builder() .withMemberName("endNode").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(edge -> ((Edge) edge).end() == null ? null : ((Edge) edge).end().getIdentifier()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryLongMerger) .withAfterViewConsistentBeforeViewMerger(MemberMergeStrategies.diffBasedLongMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongMerger).build() .mergeMember(); final MergedMemberBean> mergedGeometricParentRelationsBean = new MemberMerger.Builder>() .withMemberName(GEOMETRIC_RELATIONS_FIELD).withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor( atlasEntity -> ((CompleteEdge) atlasEntity).geometricRelationIdentifiers()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSetMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongSetMerger).build() .mergeMember(); final CompleteEdge mergedAfterEdge = new CompleteEdge(left.getIdentifier(), mergedPolyLineBean.getMergedAfterMember(), mergedTagsBean.getMergedAfterMember(), mergedStartNodeIdentifierBean.getMergedAfterMember(), mergedEndNodeIdentifierBean.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterEdge.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterEdge.withBoundsExtendedBy(afterEntityRight.bounds()); mergedAfterEdge.withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedAfterMember()); final CompleteEdge mergedBeforeEdge; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforeEdge = CompleteEdge.shallowFrom((Edge) beforeEntityLeft) .withStartNodeIdentifier(mergedStartNodeIdentifierBean.getMergedBeforeMember()) .withEndNodeIdentifier(mergedEndNodeIdentifierBean.getMergedBeforeMember()) .withTags(mergedTagsBean.getMergedBeforeMember()) .withPolyLine(mergedPolyLineBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()) .withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforeEdge = null; } return new FeatureChange(ChangeType.ADD, mergedAfterEdge, mergedBeforeEdge); } private static FeatureChange mergeLineItems(final FeatureChange left, final FeatureChange right, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } /* * The polyline merger ensure that the afterEntity polylines either: 1) exactly match or 2) * are reconcilable based on the beforeView. Additionally, this step also ensures that the * beforeViews, if present, had equivalent geometry. */ final MergedMemberBean mergedPolyLineBean = new MemberMerger.Builder() .withMemberName("polyLine").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((LineItem) atlasEntity).asPolyLine()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryPolyLineMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedPolyLineMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryPolyLineMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryPolyLineMerger).build() .mergeMember(); if (afterEntityLeft instanceof Edge) { return mergeEdges(left, right, mergedPolyLineBean, mergedTagsBean, mergedParentRelationsBean); } else if (afterEntityLeft instanceof Line) { return mergeLines(left, right, mergedPolyLineBean, mergedTagsBean, mergedParentRelationsBean); } else { throw new CoreException("Unknown AtlasEntity subtype {}", afterEntityLeft.getClass().getName()); } } private static FeatureChange mergeLines(final FeatureChange left, final FeatureChange right, final MergedMemberBean mergedPolyLineBean, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final MergedMemberBean> mergedGeometricParentRelationsBean = new MemberMerger.Builder>() .withMemberName(GEOMETRIC_RELATIONS_FIELD).withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor( atlasEntity -> ((CompleteLine) atlasEntity).geometricRelationIdentifiers()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSetMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongSetMerger).build() .mergeMember(); final CompleteLine mergedAfterLine = new CompleteLine(left.getIdentifier(), mergedPolyLineBean.getMergedAfterMember(), mergedTagsBean.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterLine.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterLine.withBoundsExtendedBy(afterEntityRight.bounds()); mergedAfterLine.withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedAfterMember()); final CompleteLine mergedBeforeLine; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforeLine = CompleteLine.shallowFrom((Line) beforeEntityLeft) .withTags(mergedTagsBean.getMergedBeforeMember()) .withPolyLine(mergedPolyLineBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()) .withGeometricRelationIdentifiers( mergedGeometricParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforeLine = null; } return new FeatureChange(ChangeType.ADD, mergedAfterLine, mergedBeforeLine); } private static FeatureChange mergeLocationItems(final FeatureChange left, final FeatureChange right, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } /* * The location merger ensure that the afterEntity locations either: 1) exactly match or 2) * are reconcilable based on the beforeView. Additionally, this step also ensures that the * beforeViews, if present, had equivalent geometry. */ final MergedMemberBean mergedLocationBean = new MemberMerger.Builder() .withMemberName("location").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((LocationItem) atlasEntity).getLocation()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryLocationMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLocationMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLocationMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLocationMerger).build() .mergeMember(); if (afterEntityLeft instanceof Node) { return mergeNodes(left, right, mergedLocationBean, mergedTagsBean, mergedParentRelationsBean); } else if (afterEntityLeft instanceof Point) { return mergePoints(left, right, mergedLocationBean, mergedTagsBean, mergedParentRelationsBean); } else { throw new CoreException("Unknown LocationItem subtype {}", afterEntityLeft.getClass().getName()); } } private static FeatureChange mergeNodes(final FeatureChange left, final FeatureChange right, final MergedMemberBean mergedLocationBean, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final MergedMemberBean> mergedInEdgeIdentifiersBean = new MemberMerger.Builder>() .withMemberName(IN_EDGE_IDENTIFIERS_FIELD).withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((Node) atlasEntity).inEdges() == null ? null : ((Node) atlasEntity).inEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new))) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSortedSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSortedSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSortedSetMerger) .withBeforeViewMerger(MemberMergeStrategies.simpleLongSortedSetMerger) .useHackForMergingConflictingConnectedEdgeSetBeforeViews( (CompleteNode) afterEntityLeft, (CompleteNode) afterEntityRight) .build().mergeMember(); final MergedMemberBean> mergedOutEdgeIdentifiersBean = new MemberMerger.Builder>() .withMemberName(OUT_EDGE_IDENTIFIERS_FIELD).withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((Node) atlasEntity).outEdges() == null ? null : ((Node) atlasEntity).outEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new))) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSortedSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSortedSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSortedSetMerger) .withBeforeViewMerger(MemberMergeStrategies.simpleLongSortedSetMerger) .useHackForMergingConflictingConnectedEdgeSetBeforeViews( (CompleteNode) afterEntityLeft, (CompleteNode) afterEntityRight) .build().mergeMember(); final CompleteNode mergedAfterNode = new CompleteNode(left.getIdentifier(), mergedLocationBean.getMergedAfterMember(), mergedTagsBean.getMergedAfterMember(), mergedInEdgeIdentifiersBean.getMergedAfterMember(), mergedOutEdgeIdentifiersBean.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterNode.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterNode.withBoundsExtendedBy(afterEntityRight.bounds()); /* * We need to merge the explicitlyExcluded sets from the left and right CompleteNodes. This * simple merge will always succeed, since the sets are key only. */ mergedAfterNode.setExplicitlyExcludedInEdgeIdentifiers( MemberMergeStrategies.simpleLongSetMerger.apply( ((CompleteNode) afterEntityLeft).explicitlyExcludedInEdgeIdentifiers(), ((CompleteNode) afterEntityRight).explicitlyExcludedInEdgeIdentifiers())); mergedAfterNode.setExplicitlyExcludedOutEdgeIdentifiers( MemberMergeStrategies.simpleLongSetMerger.apply( ((CompleteNode) afterEntityLeft).explicitlyExcludedOutEdgeIdentifiers(), ((CompleteNode) afterEntityRight).explicitlyExcludedOutEdgeIdentifiers())); final CompleteNode mergedBeforeNode; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforeNode = CompleteNode.shallowFrom((Node) beforeEntityLeft) .withInEdgeIdentifiers(mergedInEdgeIdentifiersBean.getMergedBeforeMember()) .withOutEdgeIdentifiers(mergedOutEdgeIdentifiersBean.getMergedBeforeMember()) .withTags(mergedTagsBean.getMergedBeforeMember()) .withLocation(mergedLocationBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforeNode = null; } return new FeatureChange(ChangeType.ADD, mergedAfterNode, mergedBeforeNode); } private static FeatureChange mergePoints(final FeatureChange left, final FeatureChange right, final MergedMemberBean mergedLocationBean, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final CompletePoint mergedAfterPoint = new CompletePoint(left.getIdentifier(), mergedLocationBean.getMergedAfterMember(), mergedTagsBean.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterPoint.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterPoint.withBoundsExtendedBy(afterEntityRight.bounds()); final CompletePoint mergedBeforePoint; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforePoint = CompletePoint.shallowFrom((Point) beforeEntityLeft) .withTags(mergedTagsBean.getMergedBeforeMember()) .withLocation(mergedLocationBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforePoint = null; } return new FeatureChange(ChangeType.ADD, mergedAfterPoint, mergedBeforePoint); } private static FeatureChange mergeRelations(final FeatureChange left, final FeatureChange right, final MergedMemberBean> mergedTagsBean, final MergedMemberBean> mergedParentRelationsBean) { final AtlasEntity beforeEntityLeft = left.getBeforeView(); final AtlasEntity afterEntityLeft = left.getAfterView(); final AtlasEntity beforeEntityRight = right.getBeforeView(); final AtlasEntity afterEntityRight = right.getAfterView(); if (afterEntityLeft == null) { throw new CoreException(AFTER_ENTITY_LEFT_WAS_NULL); } if (afterEntityRight == null) { throw new CoreException(AFTER_ENTITY_RIGHT_WAS_NULL); } final MergedMemberBean mergedMembersBean = new MemberMerger.Builder() .withMemberName("relationMembers").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(entity -> ((Relation) entity).members() == null ? null : ((Relation) entity).members().asBean()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleRelationBeanMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedRelationBeanMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.conflictingBeforeViewRelationBeanMerger) .withBeforeViewMerger(MemberMergeStrategies.simpleRelationBeanMerger).build() .mergeMember(); final MergedMemberBean> mergedAllRelationsWithSameOsmIdentifierBean = new MemberMerger.Builder>() .withMemberName("allRelationsWithSameOsmIdentifier") .withBeforeEntityLeft(beforeEntityLeft).withAfterEntityLeft(afterEntityLeft) .withBeforeEntityRight(beforeEntityRight).withAfterEntityRight(afterEntityRight) .withMemberExtractor(atlasEntity -> ((Relation) atlasEntity) .allRelationsWithSameOsmIdentifier() == null ? null : ((Relation) atlasEntity).allRelationsWithSameOsmIdentifier() .stream().map(Relation::getIdentifier) .collect(Collectors.toSet())) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleLongSetMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedLongSetMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongSetMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongSetMerger).build() .mergeMember(); final MergedMemberBean mergedAllKnownMembersBean = new MemberMerger.Builder() .withMemberName("allKnownOsmMembers").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor( entity -> ((Relation) entity).allKnownOsmMembers() == null ? null : ((Relation) entity).allKnownOsmMembers().asBean()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.simpleRelationBeanMerger) .withAfterViewConsistentBeforeViewMerger( MemberMergeStrategies.diffBasedRelationBeanMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.conflictingBeforeViewRelationBeanMerger) .withBeforeViewMerger(MemberMergeStrategies.simpleRelationBeanMerger).build() .mergeMember(); final MergedMemberBean mergedOsmRelationIdentifier = new MemberMerger.Builder() .withMemberName("osmRelationIdentifier").withBeforeEntityLeft(beforeEntityLeft) .withAfterEntityLeft(afterEntityLeft).withBeforeEntityRight(beforeEntityRight) .withAfterEntityRight(afterEntityRight) .withMemberExtractor(entity -> ((Relation) entity).osmRelationIdentifier()) .withAfterViewNoBeforeMerger(MemberMergeStrategies.autofailBinaryLongMerger) .withAfterViewConsistentBeforeViewMerger(MemberMergeStrategies.diffBasedLongMerger) .withAfterViewConflictingBeforeViewMerger( MemberMergeStrategies.autofailQuaternaryLongMerger) .withBeforeViewMerger(MemberMergeStrategies.autofailBinaryLongMerger).build() .mergeMember(); final Rectangle mergedBounds = Rectangle.forLocated(afterEntityLeft, afterEntityRight); final CompleteRelation mergedAfterRelation = new CompleteRelation(left.getIdentifier(), mergedTagsBean.getMergedAfterMember(), mergedBounds, mergedMembersBean.getMergedAfterMember(), mergedAllRelationsWithSameOsmIdentifierBean.getMergedAfterMember() != null ? mergedAllRelationsWithSameOsmIdentifierBean.getMergedAfterMember() .stream().collect(Collectors.toList()) : null, mergedAllKnownMembersBean.getMergedAfterMember(), mergedOsmRelationIdentifier.getMergedAfterMember(), mergedParentRelationsBean.getMergedAfterMember()); mergedAfterRelation.withBoundsExtendedBy(afterEntityLeft.bounds()); mergedAfterRelation.withBoundsExtendedBy(afterEntityRight.bounds()); if (((CompleteRelation) afterEntityLeft).isOverrideGeometry()) { final Optional leftGeom = ((CompleteRelation) afterEntityLeft) .asMultiPolygon(); if (((CompleteRelation) afterEntityRight).isOverrideGeometry()) { final Optional rightGeom = ((CompleteRelation) afterEntityRight) .asMultiPolygon(); if (leftGeom.isPresent() && rightGeom.isPresent() && leftGeom.get().equalsNorm(rightGeom.get())) { mergedAfterRelation.withMultiPolygonGeometry(rightGeom.get()); } else { throw new CoreException( "Problem merging relation override geometries for relation {}!", mergedAfterRelation.getIdentifier()); } } else { if (leftGeom.isPresent()) { mergedAfterRelation.withMultiPolygonGeometry(leftGeom.get()); } } } else if (((CompleteRelation) afterEntityRight).isOverrideGeometry()) { final Optional rightGeom = ((CompleteRelation) afterEntityRight) .asMultiPolygon(); if (rightGeom.isPresent()) { mergedAfterRelation.withMultiPolygonGeometry(rightGeom.get()); } } mergedAfterRelation.getRemovedGeometry() .addAll(((CompleteRelation) afterEntityLeft).getRemovedGeometry()); mergedAfterRelation.getRemovedGeometry() .addAll(((CompleteRelation) afterEntityRight).getRemovedGeometry()); mergedAfterRelation.getAddedGeometry() .addAll(((CompleteRelation) afterEntityLeft).getAddedGeometry()); mergedAfterRelation.getAddedGeometry() .addAll(((CompleteRelation) afterEntityRight).getAddedGeometry()); final CompleteRelation mergedBeforeRelation; /* * Here we just arbitrarily use the left side entity. We have already asserted that both * left and right explicitly provided or explicitly excluded a beforeView. At this point, we * have also ensured that both beforeViews, if present, were consistent (or we merged them * if necessary). Therefore it is safe to arbitrarily choose one from which to "shallowFrom" * clone a new CompleteEntity. */ if (beforeEntityLeft != null) { mergedBeforeRelation = CompleteRelation.shallowFrom((Relation) beforeEntityLeft) .withMembers(mergedMembersBean.getMergedBeforeMember(), beforeEntityLeft.bounds()) .withAllRelationsWithSameOsmIdentifier( mergedAllRelationsWithSameOsmIdentifierBean .getMergedAfterMember() != null ? mergedAllRelationsWithSameOsmIdentifierBean .getMergedBeforeMember().stream() .collect(Collectors.toList()) : null) .withAllKnownOsmMembers(mergedAllKnownMembersBean.getMergedBeforeMember()) .withOsmRelationIdentifier(mergedOsmRelationIdentifier.getMergedBeforeMember()) .withTags(mergedTagsBean.getMergedBeforeMember()) .withRelationIdentifiers(mergedParentRelationsBean.getMergedBeforeMember()); } else { mergedBeforeRelation = null; } return new FeatureChange(ChangeType.ADD, mergedAfterRelation, mergedBeforeRelation); } private FeatureChangeMergingHelpers() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/MemberMergeStrategies.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.BinaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.exception.change.FeatureChangeMergeException; import org.openstreetmap.atlas.exception.change.MergeFailureType; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedRelation; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.Sets; import org.openstreetmap.atlas.utilities.function.QuaternaryOperator; import org.openstreetmap.atlas.utilities.function.SenaryFunction; import org.openstreetmap.atlas.utilities.function.TernaryOperator; /** * A utility class to store the various merge strategies utilized by the {@link FeatureChange} merge * code. These are low-level strategies used to merge the underlying data structures. * * @author lcram */ public final class MemberMergeStrategies { static final BinaryOperator autofailBinaryLongMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_MERGE, "autofailBinaryLongMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator autofailQuaternaryLongMerger = (beforeLeft, beforeRight, afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_MERGE, "autofailQuaternaryLongMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator> autofailBinaryTagMerger = (afterMapLeft, afterMapRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_TAG_MERGE, "autofailBinaryTagMerger:\n{}\nvs\n{}", afterMapLeft, afterMapRight); }; static final QuaternaryOperator> autofailQuaternaryTagMerger = ( beforeMapLeft, afterMapLeft, beforeMapRight, afterMapRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_TAG_MERGE, "autofailQuaternaryTagMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeMapLeft, beforeMapRight, afterMapLeft, afterMapRight); }; static final BinaryOperator> autofailBinaryLongSetMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_SET_MERGE, "autofailBinaryLongSetMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator> autofailQuaternaryLongSetMerger = (beforeLeft, afterLeft, beforeRight, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_SET_MERGE, "autofailQuaternaryLongSetMerger: before\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator> autofailBinaryLongSortedSetMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_SORTED_SET_MERGE, "autofailBinaryLongSortedSetMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator> autofailQuaternaryLongSortedSetMerger = ( beforeLeft, afterLeft, beforeRight, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LONG_SORTED_SET_MERGE, "autofailQuaternaryLongSortedSetMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator autofailBinaryLocationMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LOCATION_MERGE, "autofailBinaryLocationMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator autofailQuaternaryLocationMerger = (beforeLeft, afterLeft, beforeRight, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_LOCATION_MERGE, "autofailQuaternaryLocationMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator autofailBinaryPolyLineMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_POLYLINE_MERGE, "autofailBinaryPolyLineMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator autofailQuaternaryPolyLineMerger = (beforeLeft, afterLeft, beforeRight, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_POLYLINE_MERGE, "autofailQuaternaryPolyLineMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator autofailBinaryPolygonMerger = (afterLeft, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_POLYGON_MERGE, "autofailBinaryPolygonMerger:\n{}\nvs\n{}", afterLeft, afterRight); }; static final QuaternaryOperator autofailQuaternaryPolygonMerger = (beforeLeft, afterLeft, beforeRight, afterRight) -> { throw new FeatureChangeMergeException(MergeFailureType.AUTOFAIL_POLYGON_MERGE, "autofailQuaternaryPolygonMerger: before:\n{}\nvs\n{}\nafter:\n{}\nvs\n{}", beforeLeft, beforeRight, afterLeft, afterRight); }; static final BinaryOperator> simpleTagMerger = (afterMapLeft, afterMapRight) -> { try { return Maps.withMaps(afterMapLeft, afterMapRight); } catch (final Exception exception) { throw new FeatureChangeMergeException(MergeFailureType.SIMPLE_TAG_MERGE_FAIL, "simpleTagMerger failed", exception); } }; static final BinaryOperator> simpleLongSetMerger = (afterSetLeft, afterSetRight) -> { try { return Sets.withSets(false, afterSetLeft, afterSetRight); } catch (final Exception exception) { throw new FeatureChangeMergeException(MergeFailureType.SIMPLE_LONG_SET_MERGE_FAIL, "simpleLongSetMerger failed", exception); } }; static final BinaryOperator> simpleLongSortedSetMerger = (afterSetLeft, afterSetRight) -> { try { return Sets.withSortedSets(false, afterSetLeft, afterSetRight); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.SIMPLE_LONG_SORTED_SET_MERGE_FAIL, "simpleLongSortedSetMerger failed", exception); } }; static final BinaryOperator simpleRelationBeanMerger = (afterBeanLeft, afterBeanRight) -> { try { return RelationBean.mergeBeans(afterBeanLeft, afterBeanRight); } catch (final Exception exception) { throw new FeatureChangeMergeException(MergeFailureType.SIMPLE_RELATION_BEAN_MERGE_FAIL, "simpleRelationBeanMerger failed", exception); } }; static final TernaryOperator diffBasedLongMerger = (beforeLong, afterLongLeft, afterLongRight) -> { try { return (Long) getDiffBasedMutuallyExclusiveMerger().apply(beforeLong, afterLongLeft, afterLongRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure(MergeFailureType.DIFF_BASED_LONG_MERGE_FAIL), "mutually exclusive Long merge failed", exception); } }; static final TernaryOperator diffBasedLocationMerger = (beforeLocation, afterLocationLeft, afterLocationRight) -> { try { return (Location) getDiffBasedMutuallyExclusiveMerger().apply(beforeLocation, afterLocationLeft, afterLocationRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure( MergeFailureType.DIFF_BASED_LOCATION_MERGE_FAIL), "mutually exclusive Location merge failed", exception); } }; static final TernaryOperator diffBasedPolyLineMerger = (beforePolyLine, afterPolyLineLeft, afterPolyLineRight) -> { try { return (PolyLine) getDiffBasedMutuallyExclusiveMerger().apply(beforePolyLine, afterPolyLineLeft, afterPolyLineRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure( MergeFailureType.DIFF_BASED_POLYLINE_MERGE_FAIL), "mutually exclusive PolyLine merge failed", exception); } }; static final TernaryOperator diffBasedPolygonMerger = (beforePolygon, afterPolygonLeft, afterPolygonRight) -> { try { return (Polygon) getDiffBasedMutuallyExclusiveMerger().apply(beforePolygon, afterPolygonLeft, afterPolygonRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception .withNewTopLevelFailure(MergeFailureType.DIFF_BASED_POLYGON_MERGE_FAIL), "mutually exclusive Polygon merge failed", exception); } }; static final TernaryOperator diffBasedRelationBeanMerger = (beforeBean, afterLeftBean, afterRightBean) -> { final Map beforeBeanMap = beforeBean.asMap(); final Map afterLeftBeanMap = afterLeftBean.asMap(); final Map afterRightBeanMap = afterRightBean.asMap(); verifyExplicitlyExcludedSets(beforeBean.asSet(), afterLeftBean.asSet(), afterLeftBean.getExplicitlyExcluded(), beforeBean.asSet(), afterRightBean.asSet(), afterRightBean.getExplicitlyExcluded()); /* * Compute the difference set between the beforeView and the afterViews (which is equivalent * to the keys removed from the after views). We filter any entries that have a count <= 0, * since this corresponds to unchanged/added keys in the afterView. */ final Map removedFromLeftView = computeMapDifferenceCounts( beforeBeanMap, afterLeftBeanMap).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); final Map removedFromRightView = computeMapDifferenceCounts( beforeBeanMap, afterRightBeanMap).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); /* * Compute the difference set between the afterViews and the beforeView (which is equivalent * to the keys added to the after views). We filter any entries that have a count <= 0, * since this corresponds to unchanged/removed keys in the afterView. */ final Map addedToLeftView = computeMapDifferenceCounts( afterLeftBeanMap, beforeBeanMap).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); final Map addedToRightView = computeMapDifferenceCounts( afterRightBeanMap, beforeBeanMap).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); /* * Check for REMOVE/REMOVE conflicts. A REMOVE/REMOVE conflict occurs when * removedFromLeftView and removedFromRightView share a key, but the values differ. */ for (final Map.Entry removedFromLeftEntry : removedFromLeftView .entrySet()) { final RelationBeanItem leftKey = removedFromLeftEntry.getKey(); final Integer leftValue = removedFromLeftEntry.getValue(); final Integer rightValue = removedFromRightView.get(leftKey); if (rightValue != null && !leftValue.equals(rightValue)) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_RELATION_BEAN_REMOVE_REMOVE_CONFLICT, "diffBasedRelationBeanMerger failed due to REMOVE/REMOVE conflict on key: [{}]: beforeValue absolute count was {} but removedLeft/Right diff counts conflict [{} vs {}]", leftKey, beforeBeanMap.get(leftKey), leftValue, rightValue); } } /* * Check for ADD/REMOVE conflicts. An ADD/REMOVE conflict occurs when the addedToLeftView * and removedFromRightView maps share a key (or the addedToRightView and * removedFromLeftView share a key). */ final Set addedLeftRemovedRightConflicts = com.google.common.collect.Sets .intersection(addedToLeftView.keySet(), removedFromRightView.keySet()); if (!addedLeftRemovedRightConflicts.isEmpty()) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_RELATION_BEAN_ADD_REMOVE_CONFLICT, "diffBasedRelationBeanMerger failed due to ADD/REMOVE conflict(s) on key(s): {}", addedLeftRemovedRightConflicts); } final Set addedRightRemovedLeftConflicts = com.google.common.collect.Sets .intersection(addedToRightView.keySet(), removedFromLeftView.keySet()); if (!addedRightRemovedLeftConflicts.isEmpty()) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_RELATION_BEAN_ADD_REMOVE_CONFLICT, "diffBasedRelationBeanMerger failed due to ADD/REMOVE conflict(s) on key(s): {}", addedLeftRemovedRightConflicts); } /* * Check for ADD/ADD conflicts. A ADD/ADD conflict occurs when addedToLeftView and * addedToRightView share a key, but the values differ. */ for (final Map.Entry addedToLeftEntry : addedToLeftView .entrySet()) { final RelationBeanItem leftKey = addedToLeftEntry.getKey(); final Integer leftValue = addedToLeftEntry.getValue(); final Integer rightValue = addedToRightView.get(leftKey); if (rightValue != null && !leftValue.equals(rightValue)) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_RELATION_BEAN_ADD_ADD_CONFLICT, "diffBasedRelationBeanMerger failed due to ADD/ADD conflict on key: [{}]: beforeValue absolute count was {} but addedLeft/Right diff counts conflict [{} vs {}]", leftKey, beforeBeanMap.get(leftKey) != null ? beforeBeanMap.get(leftKey) : 0, leftValue, rightValue); } } /* * Since there were no ADD/REMOVE or REMOVE/REMOVE conflicts, we can safely merge the * REMOVED maps. */ final Map removedMergedView = new HashMap<>(); removedFromLeftView.entrySet().stream() .forEach(entry -> removedMergedView.put(entry.getKey(), entry.getValue())); removedFromRightView.entrySet().stream() .forEach(entry -> removedMergedView.put(entry.getKey(), entry.getValue())); /* * Since there were no ADD/REMOVE or ADD/ADD conflicts. we can safely merge the ADD maps. */ final Map addedMergedView = new HashMap<>(); addedToLeftView.entrySet().stream() .forEach(entry -> addedMergedView.put(entry.getKey(), entry.getValue())); addedToRightView.entrySet().stream() .forEach(entry -> addedMergedView.put(entry.getKey(), entry.getValue())); /* * Construct the final product using our merged REMOVE and ADD views. First we create a * resultMap with merged counts. We operate on the beforeView - subtract count for each key * in removedMergedView, add count for each key in addedMergedView. Then, using the * resultMap, we can construct the result bean with the proper key counts. */ final Map resultMap = new HashMap<>(beforeBeanMap); for (final Map.Entry removedEntry : removedMergedView.entrySet()) { final RelationBeanItem removedKey = removedEntry.getKey(); final Integer removedCount = removedEntry.getValue(); final Integer beforeValue = beforeBeanMap.get(removedKey); final Integer newValue = beforeValue - removedCount; resultMap.put(removedKey, newValue); } for (final Map.Entry addedEntry : addedMergedView.entrySet()) { final RelationBeanItem addedKey = addedEntry.getKey(); final Integer addedCount = addedEntry.getValue(); final Integer beforeValue = beforeBeanMap.get(addedKey); final Integer newValue; /* * The beforeValue will be null if the added key is brand new. In that case, we just * need to set the count. Otherwise, we add the count to the before value. */ if (beforeValue == null) { newValue = 0 + addedCount; } else { newValue = beforeValue + addedCount; } resultMap.put(addedKey, newValue); } final RelationBean resultBean = new RelationBean(); for (final Map.Entry resultEntry : resultMap.entrySet()) { final RelationBeanItem resultKey = resultEntry.getKey(); final Integer resultCount = resultEntry.getValue(); for (int count = 0; count < resultCount; count++) { resultBean.addItem(resultKey); } } Stream.concat(afterLeftBean.getExplicitlyExcluded().stream(), afterRightBean.getExplicitlyExcluded().stream()) .forEach(resultBean::addItemExplicitlyExcluded); return resultBean; }; /* * Merge two differing Set created using ADDs and REMOVEs on a common ancestor. For * example, consider set A: [2, 3, 4] and set B: [1, 2, 3, 5]. Assume these were both based on * initial set I: [1, 2, 3, 4]. Neither A nor B make any changes that conflict with each other, * so we should be able to merge them into set C: [2, 3, 5]. */ static final TernaryOperator> diffBasedLongSetMerger = (beforeSet, afterLeftSet, afterRightSet) -> { final Set removedFromLeftView = com.google.common.collect.Sets.difference(beforeSet, afterLeftSet); final Set removedFromRightView = com.google.common.collect.Sets.difference(beforeSet, afterRightSet); final Set addedToLeftView = com.google.common.collect.Sets.difference(afterLeftSet, beforeSet); final Set addedToRightView = com.google.common.collect.Sets.difference(afterRightSet, beforeSet); /* * Easy key-merge of left and right ADDs and REMOVEs. We can safely ignore duplicate keys, * since it is feasible that two FeatureChanges made the same ADD or REMOVE. We also do not * need to rectify the ADD/ADD or ADD/REMOVE conflicts. Since these are keys-only, there is * no possibility of an ADD/ADD conflict. And because we enforce a shared beforeView, there * is no possibility of an ADD/REMOVE conflict. */ final Set removedMerged = Sets.withSets(false, removedFromLeftView, removedFromRightView); final Set addedMerged = Sets.withSets(false, addedToLeftView, addedToRightView); /* * Build the result set by performing the REMOVEs on the beforeView, then performing the * ADDs on the beforeView. */ final Set result = new HashSet<>(beforeSet); result.removeAll(removedMerged); result.addAll(addedMerged); return result; }; /* * This is essentially the same thing as diffBasedLongSetMerger. However, we construct a * SortedSet before returning the result. This is useful for members that require a sorted * property. */ static final TernaryOperator> diffBasedLongSortedSetMerger = (beforeSet, afterLeftSet, afterRightSet) -> new TreeSet<>( diffBasedLongSetMerger.apply(beforeSet, afterLeftSet, afterRightSet)); /* * Merge two differing Map created using ADD/MODIFYs and REMOVEs on a common * ancestor. For example, consider map A: [a=1, b=12, c=3] and map B: [a=1, b=2, c=3, d=4, e=5]. * Assume these were both based on initial map I: [a=1, b=2, c=3, d=4]. Neither A nor B make any * changes that conflict with each other, so we should be able to merge them into map C: [a=1, * b=12, c=3, e=5]. */ static final TernaryOperator> diffBasedTagMerger = (beforeMap, afterLeftMap, afterRightMap) -> { /* * Simple key removal. A REMOVE looks like [a=1] -> [] */ final Set keysRemovedFromLeftView = com.google.common.collect.Sets .difference(beforeMap.keySet(), afterLeftMap.keySet()); final Set keysRemovedFromRightView = com.google.common.collect.Sets .difference(beforeMap.keySet(), afterRightMap.keySet()); /* * Consider the difference between an ADD [] -> [a=1] and a MODIFY [a=1] -> [a=2]. Below we * group key ADDs and MODIFYs together, since we operate on the entire key->value pair. In * light of this fact, ADDs and MODIFYs are effectively the same operation as far as merge * logic is concerned. From here on, we will only refer to ADD and never MODIFY. */ final Map addedToLeftView = com.google.common.collect.Sets .difference(afterLeftMap.entrySet(), beforeMap.entrySet()).stream() .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); final Map addedToRightView = com.google.common.collect.Sets .difference(afterRightMap.entrySet(), beforeMap.entrySet()).stream() .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); /* * Check to see if any of the shared ADD keys generate an ADD/ADD conflict. An ADD/ADD * conflict is when the same key maps to two different values. */ final Set sharedAddedKeys = com.google.common.collect.Sets .intersection(addedToLeftView.keySet(), addedToRightView.keySet()); for (final String sharedKey : sharedAddedKeys) { final String leftValue = addedToLeftView.get(sharedKey); final String rightValue = addedToRightView.get(sharedKey); if (!Objects.equals(leftValue, rightValue)) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_TAG_ADD_ADD_CONFLICT, "diffBasedTagMerger failed due to ADD/ADD conflict on keys: [{} -> {}] vs [{} -> {}]", sharedKey, leftValue, sharedKey, rightValue); } } /* * Now, check for any ADD/REMOVE conflicts. An ADD/REMOVE conflict is when one view of the * map contains an update to a key, but another view of the map removes the key entirely. */ final Set keysRemovedMerged = Sets.withSets(false, keysRemovedFromLeftView, keysRemovedFromRightView); final Set keysAddedMerged = Sets.withSets(false, addedToLeftView.keySet(), addedToRightView.keySet()); final Set conflicts = com.google.common.collect.Sets.intersection(keysRemovedMerged, keysAddedMerged); if (!conflicts.isEmpty()) { throw new FeatureChangeMergeException( MergeFailureType.DIFF_BASED_TAG_ADD_REMOVE_CONFLICT, "diffBasedTagMerger failed due to ADD/REMOVE conflict(s) on key(s): {}", conflicts); } /* * Now construct the merged map. Take the beforeView and REMOVE all keys that were in the * removedMerged set. Then, ADD all keys in the addMerged set. To get the values for those * keys, we select from whichever addedTo view (left or right) gives us a non-null mapping. * This correctly handles the case where one of the FeatureChanges is ADDing a brand new key * value pair (since the other FeatureChange will not contain a mapping). */ final Map result = new HashMap<>(beforeMap); keysRemovedMerged.forEach(result::remove); keysAddedMerged.forEach(key -> { String value = addedToLeftView.get(key); if (value == null) { value = addedToRightView.get(key); } result.put(key, value); }); return result; }; /** * A merger for cases when two {@link Set}s have conflicting beforeViews. This is useful for * merging {@link Node} in/out {@link Edge} sets, since different shards may occasionally have * inconsistent views of a {@link Node}'s connected {@link Edge}s. While this merger uses * explicitlyExcluded state to properly compute the merge, it does not handle merging the * explicitlyExcluded state itself. This responsibility lies with the caller. */ static final SenaryFunction, SortedSet, Set, SortedSet, SortedSet, Set, SortedSet> conflictingBeforeViewSetMerger = ( beforeLeftSet, afterLeftSet, explicitlyExcludedLeftSet, beforeRightSet, afterRightSet, explicitlyExcludedRightSet) -> { verifyExplicitlyExcludedSets(beforeLeftSet, afterLeftSet, explicitlyExcludedLeftSet, beforeRightSet, afterRightSet, explicitlyExcludedRightSet); /* * Merge the removed sets. This should just work, since we are doing a key-only merge. */ final Set removedMerged = Sets.withSets(false, explicitlyExcludedLeftSet, explicitlyExcludedRightSet); /* * Compute the added sets. We do this by comparing the before and after views of the given * identifier sets on each side of the merge. */ final Set addedToLeft = com.google.common.collect.Sets.difference(afterLeftSet, beforeLeftSet); final Set addedToRight = com.google.common.collect.Sets.difference(afterRightSet, beforeRightSet); /* * Merge the added sets. This should just work, since we are doing a key-only merge. */ final Set addedMerged = Sets.withSets(false, addedToLeft, addedToRight); /* * Check for ADD/REMOVE conflicts. This occurs if one side of the merge adds an identifier * which was explicitly removed by the other side. */ final Set conflicts = com.google.common.collect.Sets.intersection(removedMerged, addedMerged); if (!conflicts.isEmpty()) { throw new FeatureChangeMergeException( MergeFailureType.CONFLICTING_BEFORE_VIEW_SET_ADD_REMOVE_CONFLICT, "conflictingBeforeViewSetMerger failed due to ADD/REMOVE conflict(s) on key(s): {}", conflicts); } /* * Now, we need to construct a proper merged beforeView for the left and right sides (this * view will be tolerant of inconsistencies in the left and right sides). Once we have this, * we can apply the changes from our removedMerged and addedMerged sets to get the final * result. */ final Set mergedBeforeView = simpleLongSetMerger.apply(beforeLeftSet, beforeRightSet); mergedBeforeView.removeAll(removedMerged); mergedBeforeView.addAll(addedMerged); final SortedSet resultSet = new TreeSet<>(); mergedBeforeView.forEach(resultSet::add); return resultSet; }; /** * A merger for cases when two {@link RelationBean}s have conflicting beforeViews. This can * happen occasionally, since different shards may have slightly inconsistent relation views. *

* Also note that this lambda does not respect duplicate {@link RelationBeanItem}s of a given * value. Technically, OSM allows for duplicate {@link RelationBeanItem}s in a given relation. * However, these duplicates are disallowed by {@link PackedAtlas#relationMembers} and by * extension {@link PackedRelation#members}. As a result, we need not worry about that edge case * here. */ static final QuaternaryOperator conflictingBeforeViewRelationBeanMerger = ( beforeLeftBean, afterLeftBean, beforeRightBean, afterRightBean) -> { verifyExplicitlyExcludedSets(beforeLeftBean.asSet(), afterLeftBean.asSet(), afterLeftBean.getExplicitlyExcluded(), beforeRightBean.asSet(), afterRightBean.asSet(), afterRightBean.getExplicitlyExcluded()); /* * Merge the removed sets. This should just work, since we are doing a key-only merge and * have already assumed that there cannot be duplicate RelationBeanItems. */ final Set removedMerged = Sets.withSets(false, afterLeftBean.getExplicitlyExcluded(), afterRightBean.getExplicitlyExcluded()); /* * Compute the added sets. We do this by comparing the before and after views of the given * RelationBeans on each side of the merge. */ final Set addedToLeft = com.google.common.collect.Sets .difference(afterLeftBean.asSet(), beforeLeftBean.asSet()); final Set addedToRight = com.google.common.collect.Sets .difference(afterRightBean.asSet(), beforeRightBean.asSet()); /* * Merge the added sets. This should just work, since we are doing a key-only merge and have * already assumed that there cannot be duplicate RelationBeanItems. */ final Set addedMerged = Sets.withSets(false, addedToLeft, addedToRight); /* * Check for ADD/REMOVE conflicts. This occurs if one side of the merge adds a member which * was explicitly removed by the other side. */ final Set conflicts = com.google.common.collect.Sets .intersection(removedMerged, addedMerged); if (!conflicts.isEmpty()) { throw new FeatureChangeMergeException( MergeFailureType.CONFLICTING_BEFORE_VIEW_RELATION_BEAN_ADD_REMOVE_CONFLICT, "conflictingBeforeViewRelationBeanMerger failed due to ADD/REMOVE conflict(s) on key(s): {}", conflicts); } /* * Now, we need to construct a proper merged beforeView for the left and right sides (this * view will be tolerant of inconsistencies in the left and right sides). Once we have this, * we can apply the changes from our removedMerged and addedMerged sets to get the final * result. */ final Set mergedBeforeView = simpleRelationBeanMerger .apply(beforeLeftBean, beforeRightBean).asSet(); mergedBeforeView.removeAll(removedMerged); mergedBeforeView.addAll(addedMerged); final RelationBean resultBean = new RelationBean(); mergedBeforeView.forEach(resultBean::addItem); Stream.concat(afterLeftBean.getExplicitlyExcluded().stream(), afterRightBean.getExplicitlyExcluded().stream()) .forEach(resultBean::addItemExplicitlyExcluded); return resultBean; }; /** * Compute the difference counts between two Map. The difference is computing by * subtracting the after Integer value from the before Integer value. If the before key is not * present in the after map, then we use the before Integer value for that key. Any after keys * not present in the before map are ignored. * * @param before * the before view of the map * @param after * the after view of the map * @return the difference map */ private static Map computeMapDifferenceCounts(final Map before, final Map after) { final Map result = new HashMap<>(); for (final Map.Entry beforeEntry : before.entrySet()) { final T beforeKey = beforeEntry.getKey(); final Integer beforeCount = beforeEntry.getValue(); final Integer afterCount = after.get(beforeKey); if (afterCount != null) { result.put(beforeKey, beforeCount - afterCount); } else { result.put(beforeKey, beforeCount); } } return result; } /** * Returns a TernaryOperator that acts as a diff based, mutually exclusive chooser. The operator * can successfully merge two afterViews if: 1) both afterViews match OR 2) the afterViews are * mismatched, but one of the afterViews matches the beforeView. In case 2) the merger will * select the afterView which differs from the beforeView. In any other case, the operator will * fail with an ADD/ADD conflict. * * @return the operator */ private static TernaryOperator getDiffBasedMutuallyExclusiveMerger() { return (beforeView, afterViewLeft, afterViewRight) -> { /* * If the afterViews are equivalent, arbitrarily return one of them. */ if (afterViewLeft.equals(afterViewRight)) { return afterViewLeft; } /* * afterViewLeft and afterViewRight were not equivalent. If one of them matches the * beforeView, return the opposing one. */ if (afterViewLeft.equals(beforeView)) { return afterViewRight; } if (afterViewRight.equals(beforeView)) { return afterViewLeft; } /* * If we get here, we have an ADD/ADD conflict. */ throw new FeatureChangeMergeException( MergeFailureType.MUTUALLY_EXCLUSIVE_ADD_ADD_CONFLICT, "diffBasedMutuallyExclusiveMerger failed due to ADD/ADD conflict: beforeView was:\n{}\nbut afterViews were:\n{}\nvs\n{}", beforeView, afterViewLeft, afterViewRight); }; } private static void verifyExplicitlyExcludedSets(final Set beforeLeft, final Set afterLeft, final Set explicitlyExcludedLeft, final Set beforeRight, final Set afterRight, final Set explicitlyExcludedRight) { /* * The implicitly computed removed sets. These are computed by comparing the before and * after views of the given set-based entities on each side of the merge. */ final Set implicitlyRemovedFromLeft = com.google.common.collect.Sets .difference(beforeLeft, afterLeft); final Set implicitlyRemovedFromRight = com.google.common.collect.Sets .difference(beforeRight, afterRight); /* * It must be the case that the explicitly and implicitly computed removed sets match for a * given set-based entity from one side of the merge. If they don't, that means the user * likely removed some elements without using the contextual API (withMembersAndSource for * Relations, withXEdgeIdentifiersAndSource for Nodes), which means the explicitlyExcluded * sets are not properly populated. This will lead to corrupt and unexpected results. */ if (!explicitlyExcludedLeft.equals(implicitlyRemovedFromLeft)) { throw new CoreException( "explicitlyExcludedLeft set did not match the implicitly computed removedFromLeft set:\n{}\nvs\n{}\n" + "This is likely because members were removed without using the correct withXAndSource API", explicitlyExcludedLeft, implicitlyRemovedFromLeft); } if (!explicitlyExcludedRight.equals(implicitlyRemovedFromRight)) { throw new CoreException( "explicitlyExcludedRight set did not match the implicitly computed removedFromRight set:\n{}\nvs\n{}\n" + "This is likely because members were removed without using the correct withXAndSource API", explicitlyExcludedRight, implicitlyRemovedFromRight); } } private MemberMergeStrategies() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/MemberMerger.java ================================================ package org.openstreetmap.atlas.geography.atlas.change; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.function.BinaryOperator; import java.util.function.Function; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.exception.change.FeatureChangeMergeException; import org.openstreetmap.atlas.exception.change.MergeFailureType; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.utilities.function.QuaternaryOperator; import org.openstreetmap.atlas.utilities.function.TernaryOperator; /** * This class encapsulates the logic and configuration for {@link CompleteEntity} member merging in * the context of {@link FeatureChange} merges. * * @author lcram * @param * the type of the member this {@link MemberMerger} will be merging */ public final class MemberMerger { /** * A builder class for {@link MemberMerger}. * * @author lcram * @param * the type of the member this {@link MemberMerger} will be merging */ public static class Builder { private String memberName; private AtlasEntity beforeEntityLeft; private AtlasEntity afterEntityLeft; private AtlasEntity beforeEntityRight; private AtlasEntity afterEntityRight; private Function memberExtractor; private BinaryOperator afterViewNoBeforeViewMerger; private TernaryOperator afterViewConsistentBeforeViewMerger; private QuaternaryOperator afterViewConflictingBeforeViewMerger; private BinaryOperator beforeViewMerger; private boolean useHackForMergingConflictingConnectedEdgeSetBeforeViews = false; private Optional leftNode; private Optional rightNode; public MemberMerger build() { assertRequiredFieldsNonNull(); final MemberMerger merger = new MemberMerger<>(); merger.memberName = this.memberName; merger.beforeEntityLeft = this.beforeEntityLeft; merger.afterEntityLeft = this.afterEntityLeft; merger.beforeEntityRight = this.beforeEntityRight; merger.afterEntityRight = this.afterEntityRight; merger.memberExtractor = this.memberExtractor; merger.afterViewNoBeforeViewMerger = this.afterViewNoBeforeViewMerger; merger.afterViewConsistentBeforeViewMerger = this.afterViewConsistentBeforeViewMerger; merger.afterViewConflictingBeforeViewMerger = this.afterViewConflictingBeforeViewMerger; merger.beforeViewMerger = this.beforeViewMerger; merger.useHackForMergingConflictingConnectedEdgeSetBeforeViews = this.useHackForMergingConflictingConnectedEdgeSetBeforeViews; merger.leftNode = this.leftNode; merger.rightNode = this.rightNode; return merger; } public Builder useHackForMergingConflictingConnectedEdgeSetBeforeViews( final CompleteNode left, final CompleteNode right) { this.useHackForMergingConflictingConnectedEdgeSetBeforeViews = true; this.leftNode = Optional.ofNullable(left); this.rightNode = Optional.ofNullable(right); return this; } public Builder withAfterEntityLeft(final AtlasEntity afterEntityLeft) { this.afterEntityLeft = afterEntityLeft; return this; } public Builder withAfterEntityRight(final AtlasEntity afterEntityRight) { this.afterEntityRight = afterEntityRight; return this; } public Builder withAfterViewConflictingBeforeViewMerger( final QuaternaryOperator afterViewConflictingBeforeViewMerger) { this.afterViewConflictingBeforeViewMerger = afterViewConflictingBeforeViewMerger; return this; } public Builder withAfterViewConsistentBeforeViewMerger( final TernaryOperator afterViewConsistentBeforeViewMerger) { this.afterViewConsistentBeforeViewMerger = afterViewConsistentBeforeViewMerger; return this; } public Builder withAfterViewNoBeforeMerger( final BinaryOperator afterViewNoBeforeMerger) { this.afterViewNoBeforeViewMerger = afterViewNoBeforeMerger; return this; } public Builder withBeforeEntityLeft(final AtlasEntity beforeEntityLeft) { this.beforeEntityLeft = beforeEntityLeft; return this; } public Builder withBeforeEntityRight(final AtlasEntity beforeEntityRight) { this.beforeEntityRight = beforeEntityRight; return this; } public Builder withBeforeViewMerger(final BinaryOperator beforeViewMerger) { this.beforeViewMerger = beforeViewMerger; return this; } public Builder withMemberExtractor(final Function memberExtractor) { this.memberExtractor = memberExtractor; return this; } public Builder withMemberName(final String memberName) { this.memberName = memberName; return this; } private void assertRequiredFieldsNonNull() { if (this.memberName == null) { throw new CoreException("Required field \'memberName\' was unset"); } if (this.afterEntityLeft == null) { throw new CoreException("Required field \'afterEntityLeft\' was unset"); } if (this.afterEntityRight == null) { throw new CoreException("Required field \'afterEntityRight\' was unset"); } if (this.beforeEntityLeft != null && this.beforeEntityRight == null || this.beforeEntityLeft == null && this.beforeEntityRight != null) { throw new CoreException("Both \'beforeEntity\' fields must either be set or null"); } if (this.memberExtractor == null) { throw new CoreException("Required field \'memberExtractor\' was unset"); } } } /** * A bean class to store the merged before and after members. This is useful as a return type * for the member merger, which needs to correctly merge the before and after entity view of * each {@link FeatureChange}. * * @author lcram * @param * the member type */ public static class MergedMemberBean { private final M beforeMemberMerged; private final M afterMemberMerged; public MergedMemberBean(final M before, final M after) { this.beforeMemberMerged = before; this.afterMemberMerged = after; } public M getMergedAfterMember() { return this.afterMemberMerged; } public M getMergedBeforeMember() { return this.beforeMemberMerged; } } private String memberName; private AtlasEntity beforeEntityLeft; private AtlasEntity afterEntityLeft; private AtlasEntity beforeEntityRight; private AtlasEntity afterEntityRight; private Function memberExtractor; private BinaryOperator afterViewNoBeforeViewMerger; private TernaryOperator afterViewConsistentBeforeViewMerger; private QuaternaryOperator afterViewConflictingBeforeViewMerger; private BinaryOperator beforeViewMerger; private boolean useHackForMergingConflictingConnectedEdgeSetBeforeViews; private Optional leftNode; private Optional rightNode; private MemberMerger() { } /** * Merge some feature member using a left and right before/after view. * * @return a {@link MergedMemberBean} containing the merged beforeMember view and the merged * afterMember view */ public MergedMemberBean mergeMember() { final M beforeMemberResult; final M afterMemberResult; final M beforeMemberLeft = this.beforeEntityLeft == null ? null : this.memberExtractor.apply(this.beforeEntityLeft); final M afterMemberLeft = this.afterEntityLeft == null ? null : this.memberExtractor.apply(this.afterEntityLeft); final M beforeMemberRight = this.beforeEntityRight == null ? null : this.memberExtractor.apply(this.beforeEntityRight); final M afterMemberRight = this.afterEntityRight == null ? null : this.memberExtractor.apply(this.afterEntityRight); /* * In the case that both beforeMembers are present, we check their equivalence before * continuing. If they are not equivalent, then we try to use our special beforeView * conflict resolution merge logic. Otherwise, we can continue as normal. */ if (beforeMemberLeft != null && beforeMemberRight != null && !beforeMemberLeft.equals(beforeMemberRight)) { /* * In the case that we are merging the inEdges or outEdges members of Node, we perform a * different merge logic. The in/outEdge sets have a possibility of beforeView * conflicts, and since we are unable to attach additional explicitlyExcluded state * directly to a set, we cannot use the same logic utilized for merging other members * with conflicting beforeViews. */ if (this.useHackForMergingConflictingConnectedEdgeSetBeforeViews) { return mergeMemberHackForConflictingConnectedEdgeSetBeforeViews(beforeMemberLeft, afterMemberLeft, beforeMemberRight, afterMemberRight); } return mergeMemberWithConflictingBeforeViews(beforeMemberLeft, afterMemberLeft, beforeMemberRight, afterMemberRight); } beforeMemberResult = chooseNonNullMemberIfPossible(beforeMemberLeft, beforeMemberRight); /* * In the case that both afterMembers are present, then we will need to resolve the * afterMember merge using one of the supplied merge strategies. In this case, beforeMembers * are either consistent or both null - so we can use the merged beforeMemberResult. */ if (afterMemberLeft != null && afterMemberRight != null) { return mergeMembersWithConsistentBeforeViews(beforeMemberResult, afterMemberLeft, afterMemberRight); } /* * If only one of the afterMembers is present, we just take whichever one is present. */ if (afterMemberLeft != null) { afterMemberResult = afterMemberLeft; } else if (afterMemberRight != null) { afterMemberResult = afterMemberRight; } /* * If neither afterMember is present, then just move on. */ else { afterMemberResult = null; } return new MergedMemberBean<>(beforeMemberResult, afterMemberResult); } /** * Choose the non-null member between two choices if possible. If both the left and right * members are non-null, then this method will arbitrarily select one of them. Due to this * condition, you may see unexpected results if you pass two non-null members that are unequal. * * @param memberLeft * the left side before view of the member * @param memberRight * the right side before view of the member * @return The non-null beforeMember among the two if present. Otherwise, returns {@code null}; */ private M chooseNonNullMemberIfPossible(final M memberLeft, final M memberRight) { /* * Properly merge the members. If both are non-null, we arbitrarily take the left (since * this method makes no guarantee on which side it will select when both are non-null). If * one is null and one is not, then we take the non-null. If both were null, then the result * remains null. */ if (memberLeft != null && memberRight != null) { return memberLeft; } else if (memberLeft != null) { return memberLeft; } else if (memberRight != null) { return memberRight; } else { return null; } } @SuppressWarnings("unchecked") private MergedMemberBean mergeMemberHackForConflictingConnectedEdgeSetBeforeViews( final M beforeMemberLeft, final M afterMemberLeft, final M beforeMemberRight, final M afterMemberRight) { final M beforeMemberResult; final M afterMemberResult; final Set explicitlyExcludedLeft; final Set explicitlyExcludedRight; if (!this.leftNode.isPresent()) { throw new CoreException( "Attempted merge failed for {}: tried to use hackForConflictingConnectedEdgeSet but was missing leftNode", this.memberName); } if (!this.rightNode.isPresent()) { throw new CoreException( "Attempted merge failed for {}: tried to use hackForConflictingConnectedEdgeSet but was missing rightNode", this.memberName); } if (FeatureChangeMergingHelpers.IN_EDGE_IDENTIFIERS_FIELD.equals(this.memberName)) { explicitlyExcludedLeft = this.leftNode.get().explicitlyExcludedInEdgeIdentifiers(); explicitlyExcludedRight = this.rightNode.get().explicitlyExcludedInEdgeIdentifiers(); } else if (FeatureChangeMergingHelpers.OUT_EDGE_IDENTIFIERS_FIELD.equals(this.memberName)) { explicitlyExcludedLeft = this.leftNode.get().explicitlyExcludedOutEdgeIdentifiers(); explicitlyExcludedRight = this.rightNode.get().explicitlyExcludedOutEdgeIdentifiers(); } else { throw new CoreException( "Attempted merge failed for {}: hackForConflictingConnectedEdgeSet is not a valid strategy for {}", this.memberName, this.memberName); } if (this.beforeViewMerger == null) { throw new FeatureChangeMergeException( MergeFailureType.MISSING_BEFORE_VIEW_MERGE_STRATEGY, "Conflicting beforeMembers {} and no beforeView merge strategy was provided; beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight); } try { beforeMemberResult = this.beforeViewMerger.apply(beforeMemberLeft, beforeMemberRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure( MergeFailureType.BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Attempted beforeView merge strategy failed for {} with beforeView:\n{}\nvs\n{}", // NOSONAR this.memberName, beforeMemberLeft, beforeMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Attempted beforeView merge strategy failed for {} with beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, exception); } /* * Here we hardcode the application of the SenaryOperator node connected edge set merger. We * can cast back and forth between M and SortedSet here, since we know that M is of * type SortedSet based on the constraints imposed when calling this function. */ try { final SortedSet mergeResult = MemberMergeStrategies.conflictingBeforeViewSetMerger .apply((SortedSet) beforeMemberLeft, (SortedSet) afterMemberLeft, explicitlyExcludedLeft, (SortedSet) beforeMemberRight, (SortedSet) afterMemberRight, explicitlyExcludedRight); afterMemberResult = (M) mergeResult; } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException(exception.withNewTopLevelFailure( MergeFailureType.AFTER_VIEW_CONFLICTING_BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Tried merge strategy for hackForConflictingConnectedEdgeSet, but it failed for {}" + "\nbeforeView:\n{}\nvs\n{}\nafterView:\n{}\nvs\n{}", // NOSONAR this.memberName, beforeMemberLeft, beforeMemberRight, afterMemberLeft, afterMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.AFTER_VIEW_CONFLICTING_BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Tried merge strategy for hackForConflictingConnectedEdgeSet, but it failed for {}" + "\nbeforeView:\n{}\nvs\n{}\nafterView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, afterMemberLeft, afterMemberRight, exception); } return new MergedMemberBean<>(beforeMemberResult, afterMemberResult); } /** * Merge a member that has conflicting beforeViews. This can happen occasionally with * {@link RelationBean}s and the in/out {@link Edge} identifier sets in {@link Node}, since * these may be inconsistent across shards. * * @param beforeMemberLeft * the left side before view of the member * @param afterMemberLeft * the left side after view of the member * @param beforeMemberRight * the right side before view of the member * @param afterMemberRight * the right side after view of the member * @return a {@link MergedMemberBean} containing the merged beforeMember view and the merged * afterMember view */ private MergedMemberBean mergeMemberWithConflictingBeforeViews(final M beforeMemberLeft, final M afterMemberLeft, final M beforeMemberRight, final M afterMemberRight) { final M beforeMemberResult; final M afterMemberResult; if (this.afterViewConflictingBeforeViewMerger == null) { throw new FeatureChangeMergeException( MergeFailureType.MISSING_AFTER_VIEW_MERGE_STRATEGY_WITH_BEFORE_MEMBER_CONFLICT_HANDLING, "Conflicting beforeMembers {} and no afterView merge strategy capable of handling" + " conflicting beforeViews was provided; beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight); } if (this.beforeViewMerger == null) { throw new FeatureChangeMergeException( MergeFailureType.MISSING_BEFORE_VIEW_MERGE_STRATEGY, "Conflicting beforeMembers {} and no beforeView merge strategy was provided; beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight); } try { beforeMemberResult = this.beforeViewMerger.apply(beforeMemberLeft, beforeMemberRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure( MergeFailureType.BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Attempted beforeView merge strategy failed for {} with beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Attempted beforeView merge strategy failed for {} with beforeView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, exception); } try { afterMemberResult = this.afterViewConflictingBeforeViewMerger.apply(beforeMemberLeft, afterMemberLeft, beforeMemberRight, afterMemberRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException(exception.withNewTopLevelFailure( MergeFailureType.AFTER_VIEW_CONFLICTING_BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Tried merge strategy for handling conflicting beforeViews. but it failed for {}" + "\nbeforeView:\n{}\nvs\n{}\nafterView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, afterMemberLeft, afterMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.AFTER_VIEW_CONFLICTING_BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Tried merge strategy for handling conflicting beforeViews. but it failed for {}" + "\nbeforeView:\n{}\nvs\n{}\nafterView:\n{}\nvs\n{}", this.memberName, beforeMemberLeft, beforeMemberRight, afterMemberLeft, afterMemberRight, exception); } return new MergedMemberBean<>(beforeMemberResult, afterMemberResult); } /** * Merge a member that has consistent (possibly null) beforeViews. * * @param beforeMemberResult * the pre-merged before member view * @param afterMemberLeft * the left side after view of the member * @param afterMemberRight * the right side after view of the member * @return a {@link MergedMemberBean} containing the merged beforeMember view and the merged * afterMember view */ private MergedMemberBean mergeMembersWithConsistentBeforeViews(final M beforeMemberResult, final M afterMemberLeft, final M afterMemberRight) { final M afterMemberResult; /* * In the case that both afterMembers are non-null and equivalent, we arbitrarily pick the * left one. */ if (afterMemberLeft.equals(afterMemberRight)) { return new MergedMemberBean<>(beforeMemberResult, afterMemberLeft); } /* * If both beforeMembers are present (we have already asserted their equivalence so we just * arbitrarily use beforeMemberLeft), we use the diffBased strategy if present. */ if (beforeMemberResult != null && this.afterViewConsistentBeforeViewMerger != null) { try { afterMemberResult = this.afterViewConsistentBeforeViewMerger .apply(beforeMemberResult, afterMemberLeft, afterMemberRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException(exception.withNewTopLevelFailure( MergeFailureType.AFTER_VIEW_CONSISTENT_BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Attempted afterViewConsistentBeforeMerge failed for {} with beforeView:\n{}\nafterView:\n{}\nvs\n{}", this.memberName, beforeMemberResult, afterMemberLeft, afterMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.AFTER_VIEW_CONSISTENT_BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Attempted afterViewConsistentBeforeMerge failed for {} with beforeView:\n{}\nafterView:\n{}\nvs\n{}", this.memberName, beforeMemberResult, afterMemberLeft, afterMemberRight, exception); } } /* * If the beforeMember is not present, or we don't have a diffBased strategy, we try the * simple strategy. */ else if (this.afterViewNoBeforeViewMerger != null) { try { afterMemberResult = this.afterViewNoBeforeViewMerger.apply(afterMemberLeft, afterMemberRight); } catch (final FeatureChangeMergeException exception) { throw new FeatureChangeMergeException( exception.withNewTopLevelFailure( MergeFailureType.AFTER_VIEW_NO_BEFORE_VIEW_MERGE_STRATEGY_FAILED), "Attempted afterViewNoBeforeMerge failed for {}; afterView:\n{}\nvs\n{}", this.memberName, afterMemberLeft, afterMemberRight, exception); } catch (final Exception exception) { throw new FeatureChangeMergeException( MergeFailureType.AFTER_VIEW_NO_BEFORE_VIEW_MERGE_STRATEGY_FAILED, "Attempted afterViewNoBeforeMerge failed for {}; afterView:\n{}\nvs\n{}", this.memberName, afterMemberLeft, afterMemberRight, exception); } } /* * If there was no simple strategy, we have to fail. */ else { throw new FeatureChangeMergeException( MergeFailureType.MISSING_AFTER_VIEW_MERGE_STRATEGY, "Conflicting members and no merge strategy for {}; afterView:\n{}\nvs\n{}", this.memberName, afterMemberLeft, afterMemberRight); } return new MergedMemberBean<>(beforeMemberResult, afterMemberResult); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/ChangeDescription.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.change.ChangeType; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptorComparator; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.tags.AtlasTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * A basic description of the internal contents of a {@link FeatureChange}. A * {@link ChangeDescription} consists of a {@link List} of {@link ChangeDescriptor}s as well as some * other details (like an identifier, an {@link ItemType}, and a {@link ChangeDescriptorType}). * * @author lcram */ public class ChangeDescription { /** * A transformer factory for writing data -- creation is surprisingly expensive (~90% of * the cost for saveAsOsc) */ private static final TransformerFactory TRANSFORMER_FACTORY = TransformerFactory.newInstance(); /** This improves performance for writing OSC files */ private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory .newInstance(); private static final Logger logger = LoggerFactory.getLogger(ChangeDescription.class); private static final ChangeDescriptorComparator COMPARATOR = new ChangeDescriptorComparator(); /** `visible` is used in OSC files to denote whether or not an object should be shown */ private static final String VISIBLE = "visible"; /** `version` is used in OSC files to help determine if there is a change conflict */ private static final String VERSION = "version"; /** * `if-unused` is used in OSC files to indicate that something should be deleted, if it is * unused. Any value enables it. */ private static final String IF_UNUSED = "if-unused"; /** A relation is a collection of other objects */ private static final String RELATION = "relation"; /** A relation member is part of a relation */ private static final String RELATION_MEMBER = "member"; /** The action for JOSM to use */ private static final String ACTION = "action"; /** A modify string */ private static final String MODIFY = "modify"; /** A delete string */ private static final String DELETE = "delete"; /** The specified line separator for json (windows \r\n works, since \r is whitespace) */ private static final String JSON_LINE_SEPARATOR = System.lineSeparator(); private final long identifier; private final ItemType itemType; private final ChangeDescriptorType changeDescriptorType; private final List descriptors; private final Collection nodes; private final Map originalTags; private final AtlasEntity afterView; private final AtlasEntity beforeView; private String osc; /** * Convert a LocationItem to OSC information * * @param information * The information to add to * @param entity * The node */ private static void atlasEntityToOscInformationNode(final JsonObject information, final LocationItem entity) { // Nodes have two additional attributes for latitude and longitude information.addProperty("type", "node"); information.addProperty("lat", entity.getLocation().getLatitude().asDegrees()); information.addProperty("lon", entity.getLocation().getLongitude().asDegrees()); } /** * Convert a Relation object to OSC information * * @param information * The information to add to * @param entity * The relation */ private static void atlasEntityToOscInformationRelation(final JsonObject information, final Relation entity) { information.addProperty("type", RELATION); // Relations have an array of members // final var members = new JsonArray(); // Try to account for relations spread across atlases for (final RelationMember memberInformation : entity.allKnownOsmMembers()) { final var memberObject = new JsonObject(); final String type; if (memberInformation.getEntity() instanceof LocationItem) { type = "node"; } else if (memberInformation.getEntity() instanceof LineItem) { type = "way"; } else { type = RELATION; } memberObject.addProperty("type", type); if (memberInformation.getEntity().getOsmIdentifier() > 0) { memberObject.addProperty("ref", memberInformation.getEntity().getOsmIdentifier()); } else { memberObject.addProperty("ref", newId(memberInformation.getEntity().getIdentifier())); } memberObject.addProperty("role", memberInformation.getRole()); members.add(memberObject); } } private static void createOscXmlElement(final Document document, final Element parentElement, final JsonObject object) { if (!object.has("type")) { return; } final var type = object.get("type").getAsString(); final var objectElement = document.createElement(type); objectElement.setAttribute(VISIBLE, Optional.ofNullable(object.get(VISIBLE)) .orElse(new JsonPrimitive(true)).getAsString()); objectElement.setAttribute("id", Optional.ofNullable(object.get("id")).orElse(new JsonPrimitive(0)).getAsString()); objectElement.setAttribute(VERSION, Optional.ofNullable(object.get(VERSION)) .orElse(new JsonPrimitive(1)).getAsString()); if (Long.parseLong(objectElement.getAttribute("id")) <= 0) { // New elements are "modified", at least as far as JOSM is concerned. objectElement.setAttribute(ACTION, MODIFY); } Optional.ofNullable(object.get(ACTION)).map(JsonElement::getAsString) .ifPresent(action -> objectElement.setAttribute(ACTION, action)); Optional.ofNullable(object.get(IF_UNUSED)).map(JsonElement::getAsString) .ifPresent(ifUnused -> objectElement.setAttribute(IF_UNUSED, ifUnused)); if (object.has("tags")) { for (final Map.Entry tag : object.get("tags").getAsJsonObject() .entrySet()) { final var tagElement = document.createElement("tag"); tagElement.setAttribute("k", tag.getKey()); tagElement.setAttribute("v", tag.getValue().getAsString()); objectElement.appendChild(tagElement); } } if ("node".equals(type)) { if (object.has("lat") && object.has("lon")) { final var lat = object.get("lat").getAsDouble(); final var lon = object.get("lon").getAsDouble(); objectElement.setAttribute("lat", Double.toString(lat)); objectElement.setAttribute("lon", Double.toString(lon)); } parentElement.appendChild(objectElement); } else if ("way".equals(type)) { if (object.has("nd")) { object.get("nd").getAsJsonArray().forEach(element -> { final var ndElement = document.createElement("nd"); ndElement.setAttribute("ref", element.getAsString()); objectElement.appendChild(ndElement); }); } parentElement.appendChild(objectElement); } else if (RELATION.equals(type)) { if (object.has(RELATION_MEMBER)) { object.get(RELATION_MEMBER).getAsJsonArray().forEach(element -> { final var member = element.getAsJsonObject(); final var memberElement = document.createElement(RELATION_MEMBER); memberElement.setAttribute("type", member.get("type").getAsString()); memberElement.setAttribute("ref", member.get("ref").getAsString()); memberElement.setAttribute("role", member.get("role").getAsString()); objectElement.appendChild(memberElement); }); } parentElement.appendChild(objectElement); } } /** * Check if the entity has tags. * * @param entity * The entity to check * @return {@code true} if the entity has tags */ private static boolean hasTags(@Nullable final AtlasEntity entity) { return entity != null && entity.getTags() != null && !entity.getTags().isEmpty(); } /** * Convert an identifier to a negative number, as this is commonly seen as a placeholder for new * objects * * @param identifier * the current id * @return The new id (always negative) */ private static long newId(final long identifier) { if (identifier < 0) { return identifier; } return -identifier; } /** * Convert create, modify, and delete objects to an OSC type json object * * @param description * The JsonObject to add the OSC info to * @param create * The objects being created * @param modify * The objects being modified * @param delete * The objects being deleted * @throws ParserConfigurationException * if a parser cannot be configured * @throws TransformerException * if we could not transform the generated XML to a string. */ // Suppress java:S2755 -- external entity vulnerabilities. We are building the XML, not parsing // it. @SuppressWarnings("java:S2755") private static void saveAsOsc(final JsonObject description, final Collection create, final Collection modify, final Collection delete) throws ParserConfigurationException, TransformerException { DOCUMENT_BUILDER_FACTORY.setExpandEntityReferences(false); final var builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); final var document = builder.newDocument(); final var rootElement = document.createElement("osmChange"); rootElement.setAttribute(VERSION, "0.6"); // Increment the version when this code changes rootElement.setAttribute("generator", "atlas ChangeDescription v0.0.1"); document.appendChild(rootElement); if (!create.isEmpty()) { final var createElement = document.createElement("create"); create.forEach( createObject -> createOscXmlElement(document, createElement, createObject)); rootElement.appendChild(createElement); } if (!modify.isEmpty()) { final var modifyElement = document.createElement(MODIFY); modify.forEach(jsonObject -> createOscXmlElement(document, modifyElement, jsonObject)); rootElement.appendChild(modifyElement); } if (!delete.isEmpty()) { final var deleteElement = document.createElement(DELETE); delete.forEach( deleteObject -> createOscXmlElement(document, deleteElement, deleteObject)); rootElement.appendChild(deleteElement); } if (rootElement.hasChildNodes()) { final var transformer = TRANSFORMER_FACTORY.newTransformer(); final var stringWriter = new StringWriter(); transformer.transform(new DOMSource(document), new StreamResult(stringWriter)); final var docString = stringWriter.getBuffer().toString(); // MapRoulette uses OSC in base64 encoded format, and this is a good way to ensure that // everything is in place saveOsc(description, Base64.getEncoder().encodeToString(docString.getBytes(StandardCharsets.UTF_8))); } } /** * Save OSC to a JsonObject * * @param description * The object to save to * @param osc * The osc to save */ private static void saveOsc(final JsonObject description, final String osc) { description.add("osc", new JsonPrimitive(osc)); } public ChangeDescription(final long identifier, final ItemType itemType, final AtlasEntity beforeView, final AtlasEntity afterView, final ChangeType sourceFeatureChangeType) { this(identifier, itemType, beforeView, afterView, sourceFeatureChangeType, null, Collections.emptyList()); } /** * Create a new ChangeDescription * * @param identifier * The identifier for the change object * @param itemType * The item type * @param beforeView * The unmodified object * @param afterView * The modified object * @param sourceFeatureChangeType * Change type * @param originalTags * The original object's tags * @param nodes * The nodes to be used for way geometry changes, in order. If a collection has * multiple nodes, then no geometry changes for an OSC will be written. */ public ChangeDescription(final long identifier, final ItemType itemType, final AtlasEntity beforeView, final AtlasEntity afterView, final ChangeType sourceFeatureChangeType, final Map originalTags, final Collection nodes) { this.identifier = identifier; this.itemType = itemType; this.descriptors = new ArrayList<>(); this.nodes = nodes != null ? nodes : Collections.emptyList(); this.originalTags = originalTags != null ? originalTags : Collections.emptyMap(); // Avoid saving before/after views if we don't need them to generate the json (if no nodes, // then not needed) if (!this.nodes.isEmpty()) { this.afterView = afterView; this.beforeView = beforeView; } else if (sourceFeatureChangeType == ChangeType.REMOVE) { // We need before views for removal this.afterView = null; this.beforeView = beforeView; } else { this.afterView = null; this.beforeView = null; } if (sourceFeatureChangeType == ChangeType.ADD) { if (beforeView != null) { this.changeDescriptorType = ChangeDescriptorType.UPDATE; } else { this.changeDescriptorType = ChangeDescriptorType.ADD; } } else { this.changeDescriptorType = ChangeDescriptorType.REMOVE; } this.descriptors.addAll( new ChangeDescriptorGenerator(beforeView, afterView, this.changeDescriptorType) .generate()); } public ChangeDescriptorType getChangeDescriptorType() { return this.changeDescriptorType; } /** * Get a sorted copy of the underlying {@link ChangeDescriptor} list. * * @return the sorted list */ public List getChangeDescriptors() { return new ArrayList<>(this.descriptors); } /** * Get the identifier of the feature described by this {@link ChangeDescription}. * * @return the identifier */ public long getIdentifier() { return this.identifier; } /** * Get the {@link ItemType} of the feature described by this {@link ChangeDescription}. * * @return the type */ public ItemType getItemType() { return this.itemType; } /** * Get the OSC, if one exists * * @return An optional with the osc, if present */ public Optional getOsc() { return Optional.ofNullable(this.osc); } /** * Set the OSC information (this is for deserialization, please do not call when not * deserializing) * * @param osc * The osc to set (base64 encoded) */ public void setOsc(final String osc) { this.osc = osc; } public JsonElement toJsonElement() { final JsonObject description = new JsonObject(); description.addProperty("type", this.changeDescriptorType.toString()); final JsonArray descriptorArray = new JsonArray(); for (final ChangeDescriptor descriptor : this.descriptors) { descriptorArray.add(descriptor.toJsonElement()); } description.add("descriptors", descriptorArray); if (this.osc == null) { this.createOsc(description); } else { saveOsc(description, this.osc); } return description; } @Override public String toString() { this.descriptors.sort(COMPARATOR); final var builder = new StringBuilder(22 + 2 * JSON_LINE_SEPARATOR.length()); builder.append("ChangeDescription ["); builder.append(JSON_LINE_SEPARATOR); builder.append(this.changeDescriptorType); builder.append(" "); builder.append(this.itemType); builder.append(" "); builder.append(this.getIdentifier()); builder.append(JSON_LINE_SEPARATOR); if (this.descriptors.isEmpty()) { builder.append("]"); return builder.toString(); } for (int i = 0; i < this.descriptors.size() - 1; i++) { builder.append(this.descriptors.get(i)); builder.append(JSON_LINE_SEPARATOR); } builder.append(this.descriptors.get(this.descriptors.size() - 1)); builder.append(JSON_LINE_SEPARATOR); builder.append("]"); return builder.toString(); } /** * Convert an entity to a JsonObject which can be used to create an OSC file. Note: You still * have to add the {@code visible="true|false"} information, as we don't know if this is * creating, updating, or removing. * * @param entity * the entity to convert * @param nodes * nodes for the entity (may include unrelated nodes) * @param tags * The OSM tags for the entity. May be {@code null}. If {@code null}, tags from the * entity are used. * @return A JsonObject with the information needed to create an OSC file */ private JsonObject atlasEntityToOscInformation(final AtlasEntity entity, final Collection nodes, @Nullable final Map tags) { final var information = new JsonObject(); final var tagsToUse = Optional.ofNullable(tags).orElseGet(entity::getTags); final var osmTagsToUse = Optional.ofNullable(tags).orElseGet(Collections::emptyMap) .entrySet().stream() .filter(tag -> !AtlasTag.TAGS_FROM_OSM.contains(tag.getKey()) && !AtlasTag.TAGS_FROM_ATLAS.contains(tag.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); // Common requirements if (entity.getOsmIdentifier() > 0) { information.addProperty("id", entity.getOsmIdentifier()); } else { information.addProperty("id", newId(entity.getIdentifier())); } // This is used to ensure that this will apply cleanly if (tagsToUse != null) { final Optional lastEditVersion = Optional .ofNullable(tagsToUse.get("last_edit_version")); lastEditVersion.ifPresent(s -> information.addProperty(VERSION, Long.parseLong(s))); } // Add the tags (OSC files are idempotent, in that they require all information for // an object) final Map tagsToAdd; if (!osmTagsToUse.isEmpty()) { tagsToAdd = osmTagsToUse; } else if (entity.getTags() != null && !entity.getOsmTags().isEmpty()) { tagsToAdd = entity.getOsmTags(); } else { tagsToAdd = null; } if (tagsToAdd != null) { final var tagsObject = new JsonObject(); tagsToAdd.forEach(tagsObject::addProperty); information.add("tags", tagsObject); } if (entity instanceof LocationItem) { atlasEntityToOscInformationNode(information, (LocationItem) entity); } else if (entity instanceof LineItem && ((LineItem) entity).asPolyLine() != null) { return this.atlasEntityToOscInformationWay(information, ((LineItem) entity).asPolyLine(), nodes); } else if (entity instanceof Area && ((Area) entity).asPolygon() != null) { return this.atlasEntityToOscInformationWay(information, ((Area) entity).asPolygon(), nodes); } else if (entity instanceof Relation) { atlasEntityToOscInformationRelation(information, (Relation) entity); } return information; } /** * Convert a LineItem to OSC information * * @param information * The information to add to * @param polyLine * The polyline for the way * @return {@code null} if we cannot create the way with the available information */ private JsonObject atlasEntityToOscInformationWay(final JsonObject information, final PolyLine polyLine, final Collection nodes) { information.addProperty("type", "way"); // Lines have an array of node references // final var nodeIds = new JsonArray(); for (final Location location : polyLine) { // Don't short-circuit on the first found, since there is no guarantee that it is // the only location at that point. final List foundNodes = nodes.stream() .filter(locationItem -> location.equals(locationItem.getLocation())) .collect(Collectors.toList()); // Atlases don't store what nodes belong to what ways, so we cannot create an OSC // change for this way. final long nodeCount = foundNodes.stream().mapToLong(LocationItem::getIdentifier) .distinct().count(); if (nodeCount == 1) { if (foundNodes.get(0).getOsmIdentifier() > 0) { nodeIds.add(new JsonPrimitive(foundNodes.get(0).getOsmIdentifier())); } else { nodeIds.add(new JsonPrimitive(newId(foundNodes.get(0).getIdentifier()))); } } else if (nodeCount != 0 || this.changeDescriptorType != ChangeDescriptorType.REMOVE) { // Ways that are being removed may have no nodes to remove (for example, if all // nodes are part of other ways) return null; } } information.add("nd", nodeIds); return information; } /** * Create an object for use in generating an OSC * * @param description * The object to add the OSC info to */ private void createOsc(final JsonObject description) { final List create = new ArrayList<>(); final List modify = new ArrayList<>(); final List delete = new ArrayList<>(); final Collection requiredLocations = new HashSet<>(); this.oscCreateUpdate(create, modify); if (this.updateRequiredLocations(create, modify, requiredLocations)) { this.oscDelete(delete, requiredLocations); try { saveAsOsc(description, create, modify, delete); } catch (final TransformerException | ParserConfigurationException e) { logger.error("Could not save OpenStreetMap Change information", e); } } } /** * Fill information in for create/update objects * * @param create * The collection of objects that will be created (this is added to) * @param modify * The collection of objects that will be modified (this is added to) */ private void oscCreateUpdate(final List create, final Collection modify) { if ((this.changeDescriptorType == ChangeDescriptorType.ADD || this.changeDescriptorType == ChangeDescriptorType.UPDATE) && this.afterView != null) { final Map tags; if (!hasTags(this.afterView) && !hasTags(this.beforeView) && !this.originalTags.isEmpty()) { tags = this.originalTags; } else if (hasTags(this.afterView)) { tags = this.afterView.getTags(); } else { tags = null; } final JsonObject createObject = this.atlasEntityToOscInformation(this.afterView, this.nodes, tags); if (createObject != null) { createObject.addProperty(VISIBLE, true); // This is needed for JOSM to know that something has been modified, and needs to be // uploaded. createObject.addProperty(ACTION, MODIFY); if (this.changeDescriptorType == ChangeDescriptorType.ADD) { create.add(createObject); } else { modify.add(createObject); } } } } /** * Fill items in for delete objects * * @param delete * The collection of objects to delete * @param requiredLocations * The locations where nodes must not be deleted */ private void oscDelete(final List delete, final Collection requiredLocations) { if (this.changeDescriptorType == ChangeDescriptorType.REMOVE && this.beforeView != null) { final JsonObject deleteObject = this.atlasEntityToOscInformation(this.beforeView, this.nodes, null); if (deleteObject != null) { // JOSM does not mark an object as modified if it is visible. // See following link for logic in JOSM // https://josm.openstreetmap.de/browser/josm/trunk/src/org/openstreetmap/josm/io/AbstractReader.java?rev=17822#L512 deleteObject.addProperty(VISIBLE, true); // This helps ensure that we don't accidentally delete something if the OSC is // directly applied to the OSM API. deleteObject.addProperty(IF_UNUSED, true); // Remove lat/lon, since we are deleting the object. deleteObject.remove("lat"); deleteObject.remove("lon"); // Needed for JOSM to know that it needs to be uploaded deleteObject.addProperty(ACTION, DELETE); delete.add(deleteObject); } } this.oscDeleteNodes(delete, requiredLocations); } /** * Delete nodes * * @param delete * The JsonObjects that may have deletable nodes * @param requiredLocations * The locations that must not be deleted */ private void oscDeleteNodes(final List delete, final Collection requiredLocations) { // new ArrayList avoids a concurrent modification exception for (final JsonObject entityObject : new ArrayList<>(delete)) { if (entityObject.get("nd") != null) { final var localNodes = entityObject.get("nd").getAsJsonArray(); for (final JsonElement nodeElement : localNodes) { final var nodeId = nodeElement.getAsLong(); final LocationItem node = this.nodes.stream() .filter(node1 -> node1.getOsmIdentifier() == nodeId).findAny() .filter(node1 -> node1.getTags() != null) .filter(node1 -> node1.getOsmTags().isEmpty()) .filter(node1 -> !requiredLocations.contains(node1.getLocation())) .orElse(null); // Don't delete nodes with tags if (node != null) { final JsonObject nodeDelete = this.atlasEntityToOscInformation(node, null, Collections.emptyMap()); nodeDelete.addProperty(VISIBLE, false); // This helps ensure that we don't accidentally delete something if the // OSC is directly applied to the OSM API. nodeDelete.addProperty(IF_UNUSED, true); // JOSM needs the action prop to mark the object as modified nodeDelete.addProperty(ACTION, DELETE); // Remove lat/lon, since we are deleting the object. nodeDelete.remove("lat"); nodeDelete.remove("lon"); delete.add(nodeDelete); } } entityObject.addProperty(ACTION, DELETE); } } } /** * Fill information in for create/update objects * * @param create * The collection of objects that will be created * @param modify * The collection of objects that will be modified * @param requiredLocations * The collection of locations that must be present (this is added to) * @return {@code true} if we can continue with the osc creation process */ private boolean updateRequiredLocations(final List create, final Collection modify, final Collection requiredLocations) { for (final JsonObject entityObject : Stream.concat(create.stream(), modify.stream()) .collect(Collectors.toList())) { if (entityObject.get("nd") == null) { continue; } final var localNodes = entityObject.get("nd").getAsJsonArray(); for (final JsonElement nodeElement : localNodes) { final var nodeId = nodeElement.getAsLong(); final List nodesFound = this.nodes.stream().filter( node -> node.getIdentifier() == nodeId || node.getOsmIdentifier() == nodeId) .collect(Collectors.toList()); if (nodesFound.size() != 1) { return false; } if (nodeId < 0) { // New nodes should come prior to anything else final JsonObject newNode = this.atlasEntityToOscInformation(nodesFound.get(0), null, null); newNode.addProperty(VISIBLE, true); create.add(0, newNode); } requiredLocations.add(nodesFound.get(0).getLocation()); } } return true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/ChangeDescriptorGenerator.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptorComparator; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptorName; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.GeometricRelationGeometryChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.GeometryChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.LongElementChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.RelationMemberChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.change.description.descriptors.TagChangeDescriptor; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * A helper class for generating a list of {@link ChangeDescriptor}s based on some * {@link AtlasEntity} beforeView and afterView. * * @author lcram */ public final class ChangeDescriptorGenerator { private static final ChangeDescriptorComparator COMPARATOR = new ChangeDescriptorComparator(); private static final String CORRUPTED_FEATURECHANGE_MESSAGE = "Corrupted FeatureChange: afterView {} != null but beforeView {} == null"; private final AtlasEntity beforeView; private final AtlasEntity afterView; private final ChangeDescriptorType changeDescriptorType; /** * Expected nodes for a way. This may be empty. */ private final List nodes = new ArrayList<>(0); ChangeDescriptorGenerator(final AtlasEntity beforeView, final AtlasEntity afterView, final ChangeDescriptorType changeDescriptorType) { this(beforeView, afterView, changeDescriptorType, null); } /** * Create a new change description * * @param beforeView * The object prior to the change * @param afterView * The view after the change * @param changeDescriptorType * The change type * @param nodes * If modifying a way geometry, this must contain all nodes needed for * the way, if openStreetChange is needed (MapRoulette challenges can use this for * quick fixes). */ ChangeDescriptorGenerator(final AtlasEntity beforeView, final AtlasEntity afterView, final ChangeDescriptorType changeDescriptorType, final Collection nodes) { this.beforeView = beforeView; this.afterView = afterView; this.changeDescriptorType = changeDescriptorType; this.setNodes(nodes); } /** * Set nodes needed for a full OSC file * * @param nodes * The nodes needed for an OSC. * @return this, for easy chaining */ public ChangeDescriptorGenerator setNodes(final Collection nodes) { this.nodes.clear(); if (nodes != null) { this.nodes.addAll(nodes); } return this; } List generate() { final List descriptors = new ArrayList<>(); /* * For the REMOVE case, there's no point showing any details. Users can just look at the * FeatureChange output itself to see the beforeView and afterView. */ if (this.changeDescriptorType == ChangeDescriptorType.REMOVE) { return descriptors; } descriptors.addAll(generateTagDescriptors()); descriptors.addAll(generateGeometryDescriptors()); descriptors.addAll(generateParentRelationDescriptors(CompleteEntity::relationIdentifiers)); if (this.afterView.getType() == ItemType.NODE) { descriptors.addAll(generateNodeInOutDescriptors(ChangeDescriptorName.IN_EDGE, CompleteNode::inEdgeIdentifiers)); descriptors.addAll(generateNodeInOutDescriptors(ChangeDescriptorName.OUT_EDGE, CompleteNode::outEdgeIdentifiers)); } if (this.afterView.getType() == ItemType.EDGE) { descriptors.addAll(generateEdgeStartEndDescriptors(ChangeDescriptorName.START_NODE, CompleteEdge::startNodeIdentifier)); descriptors.addAll(generateEdgeStartEndDescriptors(ChangeDescriptorName.END_NODE, CompleteEdge::endNodeIdentifier)); } if (this.afterView.getType() == ItemType.RELATION) { descriptors.addAll(generateRelationMemberDescriptors()); /* * Should we handle the other special relation fields here? * allRelationsWithSameOsmIdentifier, allKnownOsmMembers, and osmRelationIdentifier are * fields that may be altered. */ } descriptors.sort(COMPARATOR); return descriptors; } ChangeDescriptorType getChangeDescriptorType() { return this.changeDescriptorType; } private List generateEdgeStartEndDescriptors( final ChangeDescriptorName name, final Function memberExtractor) // NOSONAR { final CompleteEdge beforeEntity = (CompleteEdge) this.beforeView; final CompleteEdge afterEntity = (CompleteEdge) this.afterView; /* * If the afterView identifier was null, then we know that it was not updated. We can just * return nothing. */ if (memberExtractor.apply(afterEntity) == null) { return new ArrayList<>(); } final Long beforeIdentifier; if (beforeEntity != null) { if (memberExtractor.apply(beforeEntity) == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, name, name); } beforeIdentifier = memberExtractor.apply(beforeEntity); } else { beforeIdentifier = null; } final Long afterIdentifier = memberExtractor.apply(afterEntity); return generateLongValueDescriptors(name, beforeIdentifier, afterIdentifier); } private List generateGeometryDescriptors() { final List descriptors = new ArrayList<>(); /* * Relations do not have explicit geometry, so return nothing. */ if (this.afterView.getType() == ItemType.RELATION) { final Relation beforeRelation = (Relation) this.beforeView; final Relation afterRelation = (Relation) this.afterView; final ChangeDescriptor descriptor = GeometricRelationGeometryChangeDescriptor .getDescriptorsForGeometricRelations(beforeRelation, afterRelation); if (descriptor != null) { descriptors.add(descriptor); } return descriptors; } final CompleteEntity> beforeEntity = (CompleteEntity>) this.beforeView; final CompleteEntity> afterEntity = (CompleteEntity>) this.afterView; /* * If the afterView geometry is null, then we know the geometry was not updated. */ if (afterEntity.getGeometry() == null) { return descriptors; } final List beforeGeometry = new ArrayList<>(); final List afterGeometry = new ArrayList<>(); afterEntity.getGeometry().forEach(afterGeometry::add); if (beforeEntity != null) { if (beforeEntity.getGeometry() == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, "geometry", "geometry"); } beforeEntity.getGeometry().forEach(beforeGeometry::add); } descriptors.addAll( GeometryChangeDescriptor.getDescriptorsForGeometry(beforeGeometry, afterGeometry)); return descriptors; } private List generateLongSetDescriptors( final ChangeDescriptorName name, final Set beforeSet, final Set afterSet) { final List descriptors = new ArrayList<>(); final Set removedFromAfterView = com.google.common.collect.Sets.difference(beforeSet, afterSet); final Set addedToAfterView = com.google.common.collect.Sets.difference(afterSet, beforeSet); for (final Long identifier : removedFromAfterView) { descriptors.add(new LongElementChangeDescriptor(ChangeDescriptorType.REMOVE, identifier, null, name)); } for (final Long identifier : addedToAfterView) { descriptors.add( new LongElementChangeDescriptor(ChangeDescriptorType.ADD, identifier, name)); } return descriptors; } private List generateLongValueDescriptors( final ChangeDescriptorName name, final Long beforeIdentifier, final Long afterIdentifier) { final List descriptors = new ArrayList<>(); /* * This case occurs when an brand new Long value (e.g. startNode, endNode, etc.) is being * added, and so there is no beforeElement. */ if (beforeIdentifier == null) { descriptors.add(new LongElementChangeDescriptor(ChangeDescriptorType.ADD, null, afterIdentifier, name)); } else { descriptors.add(new LongElementChangeDescriptor(ChangeDescriptorType.UPDATE, beforeIdentifier, afterIdentifier, name)); } return descriptors; } private List generateNodeInOutDescriptors( final ChangeDescriptorName name, final Function> memberExtractor) { final CompleteNode beforeEntity = (CompleteNode) this.beforeView; final CompleteNode afterEntity = (CompleteNode) this.afterView; /* * If the afterView in/out edge set was null, then we know that it was not updated. We can * just return nothing. */ if (memberExtractor.apply(afterEntity) == null) { return new ArrayList<>(); } final Set beforeSet; if (beforeEntity != null) { if (memberExtractor.apply(beforeEntity) == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, name, name); } beforeSet = memberExtractor.apply(beforeEntity); } else { beforeSet = new HashSet<>(); } final Set afterSet = memberExtractor.apply(afterEntity); return generateLongSetDescriptors(name, beforeSet, afterSet); } private List generateParentRelationDescriptors( final Function> memberExtractor) { final ChangeDescriptorName name = ChangeDescriptorName.PARENT_RELATION; final CompleteEntity> beforeEntity = (CompleteEntity>) this.beforeView; final CompleteEntity> afterEntity = (CompleteEntity>) this.afterView; /* * If the afterView parent relations were null, then we know that they were not updated. We * can just return nothing. */ if (memberExtractor.apply(afterEntity) == null) { return new ArrayList<>(); } final Set beforeSet; if (beforeEntity != null) { if (memberExtractor.apply(beforeEntity) == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, name, name); } beforeSet = memberExtractor.apply(beforeEntity); } else { beforeSet = new HashSet<>(); } final Set afterSet = memberExtractor.apply(afterEntity); return generateLongSetDescriptors(name, beforeSet, afterSet); } private List generateRelationMemberDescriptors() { final List descriptors = new ArrayList<>(); final CompleteRelation beforeEntity = (CompleteRelation) this.beforeView; final CompleteRelation afterEntity = (CompleteRelation) this.afterView; /* * If the afterView members were null, then we know that the members were not updated. We * can just return nothing. */ if (afterEntity.members() == null) { return descriptors; } final Set beforeBeanSet; if (beforeEntity != null) { if (beforeEntity.members() == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, "relation members", "relation members"); } beforeBeanSet = beforeEntity.members().asBean().asSet(); } else { beforeBeanSet = new HashSet<>(); } final Set afterBeanSet = afterEntity.members().asBean() .asSet(); final Set itemsRemovedFromAfterView = com.google.common.collect.Sets .difference(beforeBeanSet, afterBeanSet); final Set itemsAddedToAfterView = com.google.common.collect.Sets .difference(afterBeanSet, beforeBeanSet); for (final RelationBean.RelationBeanItem item : itemsRemovedFromAfterView) { descriptors.add(new RelationMemberChangeDescriptor(ChangeDescriptorType.REMOVE, item.getIdentifier(), item.getType(), item.getRole())); } for (final RelationBean.RelationBeanItem item : itemsAddedToAfterView) { descriptors.add(new RelationMemberChangeDescriptor(ChangeDescriptorType.ADD, item.getIdentifier(), item.getType(), item.getRole())); } return descriptors; } private List generateTagDescriptors() { final List descriptors = new ArrayList<>(); /* * If the afterView tags were null, then we know that the tags were not updated. We can just * return nothing. */ if (this.afterView.getTags() == null) { return descriptors; } final Map beforeTags; if (this.beforeView != null) { if (this.beforeView.getTags() == null) { throw new CoreException(CORRUPTED_FEATURECHANGE_MESSAGE, "tags", "tags"); } beforeTags = this.beforeView.getTags(); } else { beforeTags = new HashMap<>(); } final Map afterTags = this.afterView.getTags(); final Set keysRemovedFromAfterView = com.google.common.collect.Sets .difference(beforeTags.keySet(), afterTags.keySet()); final Set keysAddedToAfterView = com.google.common.collect.Sets .difference(afterTags.keySet(), beforeTags.keySet()); final Set keysShared = com.google.common.collect.Sets .intersection(beforeTags.keySet(), afterTags.keySet()); for (final String key : keysRemovedFromAfterView) { descriptors.add( new TagChangeDescriptor(ChangeDescriptorType.REMOVE, key, beforeTags.get(key))); } for (final String key : keysAddedToAfterView) { descriptors.add( new TagChangeDescriptor(ChangeDescriptorType.ADD, key, afterTags.get(key))); } for (final String key : keysShared) { if (!beforeTags.get(key).equals(afterTags.get(key))) { descriptors.add(new TagChangeDescriptor(ChangeDescriptorType.UPDATE, key, afterTags.get(key), beforeTags.get(key))); } } return descriptors; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/ChangeDescriptorType.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description; import org.openstreetmap.atlas.geography.atlas.change.ChangeType; /** * Three basic types to characterize the change descriptions. This is similar to the * {@link ChangeType} enum, but with additional support for a more granular * {@link ChangeDescriptorType#UPDATE} type. * * @author lcram */ public enum ChangeDescriptorType { ADD, UPDATE, REMOVE } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/ChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescription; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A basic change unit, which when grouped together form a {@link ChangeDescription}. * * @author lcram */ public interface ChangeDescriptor { ChangeDescriptorType getChangeDescriptorType(); ChangeDescriptorName getName(); default JsonElement toJsonElement() { final JsonObject descriptor = new JsonObject(); descriptor.addProperty("name", getName().toString()); descriptor.addProperty("type", getChangeDescriptorType().toString()); return descriptor; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/ChangeDescriptorComparator.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import java.util.Comparator; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescription; /** * A {@link Comparator} for {@link ChangeDescriptor}s, which defines an ordering. This is useful * when printing {@link ChangeDescription}s, so the display can show a consistent element ordering * * @author lcram */ public class ChangeDescriptorComparator implements Comparator { @Override public int compare(final ChangeDescriptor left, final ChangeDescriptor right) { if (left.getName() != right.getName()) { return left.getName().compareTo(right.getName()); } return complexCompare(left, right); } private int complexCompare(final ChangeDescriptor left, final ChangeDescriptor right) { if (left instanceof TagChangeDescriptor && right instanceof TagChangeDescriptor) { return tagChangeCompare((TagChangeDescriptor) left, (TagChangeDescriptor) right); } if (left instanceof GeometryChangeDescriptor && right instanceof GeometryChangeDescriptor) { return geometryChangeCompare((GeometryChangeDescriptor) left, (GeometryChangeDescriptor) right); } if (left instanceof LongElementChangeDescriptor && right instanceof LongElementChangeDescriptor) { return genericSetChangeCompare((LongElementChangeDescriptor) left, (LongElementChangeDescriptor) right); } if (left instanceof RelationMemberChangeDescriptor && right instanceof RelationMemberChangeDescriptor) { return relationMemberChangeCompare((RelationMemberChangeDescriptor) left, (RelationMemberChangeDescriptor) right); } throw new CoreException("Could not compare {} vs {}", left, right); } private int genericSetChangeCompare(final LongElementChangeDescriptor left, final LongElementChangeDescriptor right) { if (left.getName() != right.getName()) { return left.getName().compareTo(right.getName()); } if (left.getChangeDescriptorType() != right.getChangeDescriptorType()) { return left.getChangeDescriptorType().compareTo(right.getChangeDescriptorType()); } final Comparable leftBeforeComparable = (Comparable) left.getBeforeElement(); final Comparable rightBeforeComparable = (Comparable) right.getBeforeElement(); if (leftBeforeComparable != null && rightBeforeComparable != null && !leftBeforeComparable.equals(rightBeforeComparable)) { return leftBeforeComparable.compareTo(rightBeforeComparable); } final Comparable leftAfterComparable = (Comparable) left.getAfterElement(); final Comparable rightAfterComparable = (Comparable) right.getAfterElement(); if (leftAfterComparable != null && rightAfterComparable != null && !leftAfterComparable.equals(rightAfterComparable)) { return leftAfterComparable.compareTo(rightAfterComparable); } else { /* * If this message appears in production, then that means either this comparison logic * or ChangeDescriptor generation is dubious. But based on the way ChangeDescriptors are * generated, it should never show up in practice. */ throw new CoreException("No comparable criteria for {} vs {}", left, right); } } private int geometryChangeCompare(final GeometryChangeDescriptor left, final GeometryChangeDescriptor right) { if (left.getChangeDescriptorType() != right.getChangeDescriptorType()) { return left.getChangeDescriptorType().compareTo(right.getChangeDescriptorType()); } if (left.getSourcePosition() != right.getSourcePosition()) { return Integer.compare(left.getSourcePosition(), right.getSourcePosition()); } final Optional leftBeforeViewWkt = left.getBeforeViewWkt(); final Optional rightBeforeViewWkt = right.getBeforeViewWkt(); if (leftBeforeViewWkt.isPresent() && rightBeforeViewWkt.isPresent() && !leftBeforeViewWkt.get().equals(rightBeforeViewWkt.get())) { return leftBeforeViewWkt.get().compareTo(rightBeforeViewWkt.get()); } final Optional leftAfterViewWkt = left.getAfterViewWkt(); final Optional rightAfterViewWkt = right.getAfterViewWkt(); if (leftAfterViewWkt.isPresent() && rightAfterViewWkt.isPresent()) { return leftAfterViewWkt.get().compareTo(rightAfterViewWkt.get()); } else { /* * If this message appears in production, then that means either this comparison logic * or ChangeDescriptor generation is dubious. But based on the way ChangeDescriptors are * generated, it should never show up in practice. */ throw new CoreException("No comparable criteria for {} vs {}", left, right); } } private int relationMemberChangeCompare(final RelationMemberChangeDescriptor left, final RelationMemberChangeDescriptor right) { if (left.getChangeDescriptorType() != right.getChangeDescriptorType()) { return left.getChangeDescriptorType().compareTo(right.getChangeDescriptorType()); } if (left.getItemType() != right.getItemType()) { return left.getItemType().compareTo(right.getItemType()); } if (left.getIdentifier() != right.getIdentifier()) { return Long.compare(left.getIdentifier(), right.getIdentifier()); } return left.getRole().compareTo(right.getRole()); } private int tagChangeCompare(final TagChangeDescriptor left, final TagChangeDescriptor right) { if (left.getChangeDescriptorType() != right.getChangeDescriptorType()) { return left.getChangeDescriptorType().compareTo(right.getChangeDescriptorType()); } return left.getKey().compareTo(right.getKey()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/ChangeDescriptorName.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; /** * The various values of the name field in the {@link ChangeDescriptor} JSON serialization. This * enum mostly exists to prevent us from having to hardcode strings all over the place. * * @author lcram */ public enum ChangeDescriptorName { TAG, GEOMETRY, PARENT_RELATION, RELATION_MEMBER, IN_EDGE, OUT_EDGE, START_NODE, END_NODE } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/GeometricRelationGeometryChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import java.util.Optional; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * @author samg */ public final class GeometricRelationGeometryChangeDescriptor implements ChangeDescriptor { private static final Logger logger = LoggerFactory .getLogger(GeometricRelationGeometryChangeDescriptor.class); private final ChangeDescriptorType changeType; private final String diff; public static GeometricRelationGeometryChangeDescriptor getDescriptorsForGeometricRelations( final Relation before, final Relation after) { if (before == null) { if (after == null) { return null; } final Optional afterGeometry = after.asMultiPolygon(); if (afterGeometry.isPresent()) { return new GeometricRelationGeometryChangeDescriptor(afterGeometry.get().toText(), ChangeDescriptorType.ADD); } return null; } else if (after == null) { final Optional beforeGeometry = before.asMultiPolygon(); if (beforeGeometry.isPresent()) { return new GeometricRelationGeometryChangeDescriptor(beforeGeometry.get().toText(), ChangeDescriptorType.REMOVE); } return null; } final Optional beforeGeometry = before.asMultiPolygon(); final Optional afterGeometry = after.asMultiPolygon(); if (beforeGeometry.isEmpty() && afterGeometry.isEmpty()) { return null; } else if (beforeGeometry.isPresent() && afterGeometry.isEmpty()) { return new GeometricRelationGeometryChangeDescriptor(beforeGeometry.get().toText(), ChangeDescriptorType.REMOVE); } else if (beforeGeometry.isEmpty() && afterGeometry.isPresent()) { return new GeometricRelationGeometryChangeDescriptor(afterGeometry.get().toText(), ChangeDescriptorType.ADD); } try { if (beforeGeometry.get().equals(afterGeometry.get())) { return null; } } catch (final Exception exc) { logger.error("Geometry equals failed for relation {}", before.getIdentifier(), exc); } try { if (beforeGeometry.get().getArea() == afterGeometry.get().getArea()) { return null; } return new GeometricRelationGeometryChangeDescriptor( Double.toString(beforeGeometry.get().getArea() - afterGeometry.get().getArea()), ChangeDescriptorType.UPDATE); } catch (final Exception exc) { logger.error("Geometry intersection failed for relation {}", before.getIdentifier(), exc); throw new CoreException("Couldn't calculate diff for relation {}", before.getIdentifier()); } } private GeometricRelationGeometryChangeDescriptor(final String diff, final ChangeDescriptorType changeType) { this.changeType = changeType; this.diff = diff; } @Override public ChangeDescriptorType getChangeDescriptorType() { return this.changeType; } @Override public ChangeDescriptorName getName() { return ChangeDescriptorName.GEOMETRY; } @Override public JsonElement toJsonElement() { final JsonObject descriptor = (JsonObject) ChangeDescriptor.super.toJsonElement(); descriptor.addProperty("diff", this.diff); return descriptor; } @Override public String toString() { final StringBuilder diffString = new StringBuilder(); diffString.append(this.changeType.toString()); diffString.append(", "); switch (this.changeType) { case UPDATE: diffString.append(", "); diffString.append(this.diff); break; case REMOVE: diffString.append(", "); diffString.append(this.diff); break; case ADD: diffString.append(", "); diffString.append(this.diff); break; default: throw new CoreException("Unexpected ChangeType value: " + this.changeType); } return getName().toString() + "(" + diffString.toString() + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/GeometryChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import com.github.difflib.DiffUtils; import com.github.difflib.algorithm.DiffException; import com.github.difflib.patch.AbstractDelta; import com.github.difflib.patch.Patch; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A {@link ChangeDescriptor} for geometry changes. Utilizes a granular diff algorithm to show the * individual {@link Location}s within the linestring that actually changed. * * @author lcram */ public final class GeometryChangeDescriptor implements ChangeDescriptor { private final ChangeDescriptorType changeType; private final AbstractDelta delta; private final int sourceMaterialSize; /** * Create a descriptor for a geometry change * * @param beforeList * The node locations of the geometry prior to the change * @param afterList * The node locations of the geometry after the change * @return A collection of GeometryChangeDescriptors */ public static List getDescriptorsForGeometry( final List beforeList, final List afterList) { final Patch diff; try { diff = DiffUtils.diff(beforeList, afterList); } catch (final DiffException exception) { throw new CoreException("Failed to compute diff for GeometryChangeDescriptor", exception); } final List descriptors = new ArrayList<>(); for (final AbstractDelta delta : diff.getDeltas()) { descriptors.add(new GeometryChangeDescriptor(delta, beforeList.size())); } return descriptors; } public static Patch getPatch(final List descriptors) { final Patch patch = new Patch<>(); for (final GeometryChangeDescriptor descriptor : descriptors) { patch.addDelta(descriptor.getDelta()); } return patch; } private GeometryChangeDescriptor(final AbstractDelta delta, final int sourceMaterialSize) { switch (delta.getType()) { case CHANGE: this.changeType = ChangeDescriptorType.UPDATE; break; case DELETE: this.changeType = ChangeDescriptorType.REMOVE; break; case INSERT: this.changeType = ChangeDescriptorType.ADD; break; default: throw new CoreException("Unexpected Delta value: " + delta.getType()); } this.delta = delta; this.sourceMaterialSize = sourceMaterialSize; } public Optional getAfterViewWkt() { if (this.changeType == ChangeDescriptorType.ADD || this.changeType == ChangeDescriptorType.UPDATE) { return Optional.of(new PolyLine(this.delta.getTarget().getLines()).toWkt()); } return Optional.empty(); } public Optional getBeforeViewWkt() { if (this.changeType == ChangeDescriptorType.REMOVE || this.changeType == ChangeDescriptorType.UPDATE) { return Optional.of(new PolyLine(this.delta.getSource().getLines()).toWkt()); } return Optional.empty(); } @Override public ChangeDescriptorType getChangeDescriptorType() { return this.changeType; } public AbstractDelta getDelta() { return this.delta; } @Override public ChangeDescriptorName getName() { return ChangeDescriptorName.GEOMETRY; } public int getSourcePosition() { return this.delta.getSource().getPosition(); } @Override public JsonElement toJsonElement() { final JsonObject descriptor = (JsonObject) ChangeDescriptor.super.toJsonElement(); descriptor.addProperty("position", this.delta.getSource().getPosition() + "/" + this.sourceMaterialSize); switch (this.changeType) { // Don't truncate, since some geometry changes can be very long (for example, reversing // a way) case UPDATE: descriptor.addProperty("beforeView", new PolyLine(this.delta.getSource().getLines()).toWkt()); descriptor.addProperty("afterView", new PolyLine(this.delta.getTarget().getLines()).toWkt()); break; case REMOVE: descriptor.addProperty("beforeView", new PolyLine(this.delta.getSource().getLines()).toWkt()); break; case ADD: descriptor.addProperty("afterView", new PolyLine(this.delta.getTarget().getLines()).toWkt()); break; default: throw new CoreException("Unexpected ChangeType value: " + this.delta.getType()); } return descriptor; } @Override public String toString() { final StringBuilder diffString = new StringBuilder(); diffString.append(this.changeType.toString()); diffString.append(", "); diffString.append(this.delta.getSource().getPosition()); diffString.append("/"); diffString.append(this.sourceMaterialSize); switch (this.changeType) { case UPDATE: diffString.append(", "); diffString.append(new PolyLine(this.delta.getSource().getLines()).toWkt()); diffString.append(" => "); diffString.append(new PolyLine(this.delta.getTarget().getLines()).toWkt()); break; case REMOVE: diffString.append(", "); diffString.append(new PolyLine(this.delta.getSource().getLines()).toWkt()); break; case ADD: diffString.append(", "); diffString.append(new PolyLine(this.delta.getTarget().getLines()).toWkt()); break; default: throw new CoreException("Unexpected ChangeType value: " + this.delta.getType()); } return getName().toString() + "(" + diffString.toString() + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/LongElementChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A {@link ChangeDescriptor} for an element of type {@link Long}. E.g. cases include * startNodeIdentifiers, inEdgeIdentifiers, parentRelations, etc. * * @author lcram */ public class LongElementChangeDescriptor implements ChangeDescriptor { private final ChangeDescriptorType changeType; private final Long beforeElement; private final Long afterElement; private final ChangeDescriptorName name; public LongElementChangeDescriptor(final ChangeDescriptorType changeType, final Long beforeElement, final Long afterElement, final ChangeDescriptorName name) { this.changeType = changeType; this.beforeElement = beforeElement; this.afterElement = afterElement; this.name = name; } public LongElementChangeDescriptor(final ChangeDescriptorType changeType, final Long afterElement, final ChangeDescriptorName name) { this.changeType = changeType; this.beforeElement = null; this.afterElement = afterElement; this.name = name; } public Long getAfterElement() { return this.afterElement; } public Long getBeforeElement() { return this.beforeElement; } @Override public ChangeDescriptorType getChangeDescriptorType() { return this.changeType; } @Override public ChangeDescriptorName getName() { return this.name; } @Override public JsonElement toJsonElement() { final JsonObject descriptor = (JsonObject) ChangeDescriptor.super.toJsonElement(); if (this.beforeElement != null) { descriptor.addProperty("beforeElement", this.beforeElement.toString()); } if (this.afterElement != null) { descriptor.addProperty("afterElement", this.afterElement.toString()); } return descriptor; } @Override public String toString() { if (this.changeType == ChangeDescriptorType.UPDATE) { return this.name + "(" + this.getChangeDescriptorType() + ", " + this.getBeforeElement() + " => " + this.getAfterElement() + ")"; } if (this.changeType == ChangeDescriptorType.REMOVE) { return this.name + "(" + this.getChangeDescriptorType() + ", " + this.getBeforeElement() + ")"; } return this.name + "(" + this.getChangeDescriptorType() + ", " + this.getAfterElement() + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/RelationMemberChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A {@link ChangeDescriptor} for relation member changes. * * @author lcram */ public class RelationMemberChangeDescriptor implements ChangeDescriptor { private final ChangeDescriptorType changeType; private final long identifier; private final ItemType type; private final String role; public RelationMemberChangeDescriptor(final ChangeDescriptorType changeType, final long identifier, final ItemType type, final String role) { this.changeType = changeType; this.identifier = identifier; this.type = type; this.role = role; } @Override public ChangeDescriptorType getChangeDescriptorType() { return this.changeType; } public long getIdentifier() { return this.identifier; } public ItemType getItemType() { return this.type; } @Override public ChangeDescriptorName getName() { return ChangeDescriptorName.RELATION_MEMBER; } public String getRole() { return this.role; } @Override public JsonElement toJsonElement() { final JsonObject descriptor = (JsonObject) ChangeDescriptor.super.toJsonElement(); descriptor.addProperty("itemType", this.type.toString()); descriptor.addProperty("id", this.identifier); descriptor.addProperty("role", this.role); return descriptor; } @Override public String toString() { return getName().toString() + "(" + this.changeType + ", " + this.type + ", " + this.identifier + ", " + this.role + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/description/descriptors/TagChangeDescriptor.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.description.descriptors; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescriptorType; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A {@link ChangeDescriptor} for tag related changes. * * @author lcram */ public class TagChangeDescriptor implements ChangeDescriptor { private final ChangeDescriptorType changeType; private final String key; private final String value; private final String originalValue; public TagChangeDescriptor(final ChangeDescriptorType changeType, final String key, final String value, final String originalValue) { this.changeType = changeType; this.key = key; this.value = value; this.originalValue = originalValue; } public TagChangeDescriptor(final ChangeDescriptorType changeType, final String key, final String value) { this(changeType, key, value, null); } @Override public ChangeDescriptorType getChangeDescriptorType() { return this.changeType; } public String getKey() { return this.key; } @Override public ChangeDescriptorName getName() { return ChangeDescriptorName.TAG; } public Optional getOriginalValue() { return Optional.ofNullable(this.originalValue); } public String getValue() { return this.value; } @Override public JsonElement toJsonElement() { final JsonObject descriptor = (JsonObject) ChangeDescriptor.super.toJsonElement(); descriptor.addProperty("key", this.key); descriptor.addProperty("value", this.value); if (this.originalValue != null) { descriptor.addProperty("originalValue", this.originalValue); } return descriptor; } @Override public String toString() { String string = getName().toString() + "(" + this.changeType + ", " + this.key + ", "; if (this.originalValue == null) { string += this.value + ")"; } else { string += this.originalValue + " => " + this.value + ")"; } return string; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/diff/AtlasDiff.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.diff; import java.util.HashSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.BareAtlas; import org.openstreetmap.atlas.geography.atlas.change.Change; import org.openstreetmap.atlas.geography.atlas.change.ChangeAtlas; import org.openstreetmap.atlas.geography.atlas.change.ChangeBuilder; import org.openstreetmap.atlas.geography.atlas.change.ChangeType; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Generate {@link Change} objects based on the differences between two {@link Atlas}es. The main * usage of this class is the {@link AtlasDiff#generateChange()} method.
*
* * @author lcram */ public class AtlasDiff { private static final Logger logger = LoggerFactory.getLogger(AtlasDiff.class); private final Atlas before; private final Atlas after; private Change change; /* * By default, AtlasDiff will save geometries of all changed features, even if that feature's * geometry was not changed. This makes it really easy to visualize changed features using * GeoJSON (e.g. if a Node tag changes, we still want to be able to see it in the GeoJSON * serialized Change object). This behaviour can be disabled. */ private boolean saveAllGeometries = true; /** * Construct an {@link AtlasDiff} with a given before {@link Atlas} and after {@link Atlas}. The * resulting {@link Change} will effectively transform the before atlas into the after atlas. * * @param before * the initial {@link Atlas} * @param after * the changed {@link Atlas} */ public AtlasDiff(final Atlas before, final Atlas after) { this.before = before; this.after = after; this.change = null; if (this.before == null) { throw new CoreException("Before atlas cannot be null."); } if (this.after == null) { throw new CoreException("After atlas cannot be null."); } } /** * Generate a {@link Change} that represents a transformation from the before {@link Atlas} to * the after {@link Atlas}. If there were no changes (i.e. the atlases had no differences), then * return an empty {@link Optional}. Repeated calls to this method will return a cached * {@link Change} instead of performing a re-computation.
*
* Now suppose we create a {@link ChangeAtlas} based on the before {@link Atlas} and the * generated {@link Change}. All of the following will be true:
*
* 1) Computing the {@link AtlasDiff} of the {@link ChangeAtlas} with the after {@link Atlas} * would yield an empty {@link Change}.
*
* 2) The {@link ChangeAtlas} will be equivalent to the after {@link Atlas}, * feature-for-feature.
*
* 3) The {@link PackedAtlas}es created by cloning both the {@link ChangeAtlas} and the after * {@link Atlas} will be equivalent under {@link BareAtlas#equals(Object)}, but may not * necessarily be byte-for-byte equivalent.
*
* * @return an {@link Optional} wrapping the generated {@link Change} */ public Optional generateChange() { final Time start = Time.now(); if (this.change != null) { return Optional.of(this.change); } final Set addedEntities = new HashSet<>(); final Set removedEntities = new HashSet<>(); final Set potentiallyModifiedEntities = new HashSet<>(); final ChangeBuilder changeBuilder = new ChangeBuilder(); /* * Check for entities that were removed in the after atlas. If we find any, add them to a * removedEntities set for later processing. We will use this set to create FeatureChanges. * We also check for entities that were potentially modified in the after atlas, i.e. any * entities present in both the before and after atlases. We add them to a * potentiallyModified set, and will check if any modifications occurred later. */ Iterables.stream(this.before).forEach(beforeEntity -> { if (isEntityMissingFromGivenAtlas(beforeEntity, this.after)) { removedEntities.add(beforeEntity); } else { potentiallyModifiedEntities.add(beforeEntity); } }); /* * Check for entities that were added in the after atlas. If we find any, add them to a * addedEntities set for later processing. We will use this set to create FeatureChanges. */ Iterables.stream(this.after) .filter(afterEntity -> isEntityMissingFromGivenAtlas(afterEntity, this.before)) .forEach(addedEntities::add); /* * Aggregate the results stored in the sets, creating the FeatureChange objects if there are * necessary changes. */ createFeatureChangesBasedOnEntitySets(addedEntities, removedEntities, potentiallyModifiedEntities, this.before, this.after, this.saveAllGeometries) .stream().forEach(changeBuilder::add); if (changeBuilder.peekNumberOfChanges() == 0) { logger.debug("Computed AtlasDiff ({} vs {}) in {}", this.before.getName(), this.after.getName(), start.elapsedSince()); return Optional.empty(); } this.change = changeBuilder.get(); logger.debug("Computed AtlasDiff ({} vs {}) in {}", this.before.getName(), this.after.getName(), start.elapsedSince()); return Optional.of(this.change); } public Atlas getAfterAtlas() { return this.after; } public Atlas getBeforeAtlas() { return this.before; } /** * Saving all geometries means that we enrich features with their geometry even if they do not * change the geometry. This is useful for visualization. * * @param saveAllGeometries * save all geometries * @return a configured {@link AtlasDiff} */ public AtlasDiff saveAllGeometries(final boolean saveAllGeometries) { this.saveAllGeometries = saveAllGeometries; return this; } /** * Given a set of added, removed, and potentially modified entities, construct a set of * {@link FeatureChange}s that transform a given before atlas into a given after atlas. * * @param addedEntities * the set of entities added in the after atlas * @param removedEntities * the set of entities removed in the after atlas * @param potentiallyModifiedEntities * the set of entities potentially modified in the after atlas (i.e. any entity, with * the same ID, found in both before and after) * @param beforeAtlas * the before atlas * @param afterAtlas * the after atlas * @param saveAllGeometries * if we are saving all geometries, even when not modified * @return the set of {@link FeatureChange}s that would turn before into after */ private Set createFeatureChangesBasedOnEntitySets( final Set addedEntities, final Set removedEntities, final Set potentiallyModifiedEntities, final Atlas beforeAtlas, final Atlas afterAtlas, final boolean saveAllGeometries) { final Set featureChanges = new HashSet<>(); addedEntities .stream().map(addedEntity -> createSimpleFeatureChangeWithType(ChangeType.ADD, addedEntity, afterAtlas, beforeAtlas, saveAllGeometries)) .forEach(featureChanges::add); removedEntities.stream() .map(removedEntity -> createSimpleFeatureChangeWithType(ChangeType.REMOVE, removedEntity, beforeAtlas, beforeAtlas, saveAllGeometries)) .forEach(featureChanges::add); potentiallyModifiedEntities.stream() .map(modifiedEntity -> createModifyFeatureChanges(modifiedEntity, beforeAtlas, saveAllGeometries)) .forEach(modifyFeatureChangeSet -> modifyFeatureChangeSet .forEach(featureChanges::add)); return featureChanges; } /** * Create a set of modify {@link FeatureChange}s ie. ADDs that change an existing feature. The * set will contain an individual {@link FeatureChange} for certain types of differences. Eg. if * a Point's location AND tags changed, then the set will contain two feature changes, one for * the location and one for the tags. However, if a Node's location and connected edges change, * this will come in as one feature change.
* * @param entity * the entity being modified * @param beforeViewAtlas * the atlas from which to compute the beforeView * @param saveAllGeometries * whether or not we are default saving all geometry * @return a {@link Set} containing the possibly constructed {@link FeatureChange}s */ private Set createModifyFeatureChanges(final AtlasEntity entity, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { final Set featureChanges = new HashSet<>(); final AtlasEntity beforeEntity = entity.getType().entityForIdentifier(this.before, entity.getIdentifier()); final AtlasEntity afterEntity = entity.getType().entityForIdentifier(this.after, entity.getIdentifier()); if (beforeEntity == null || afterEntity == null) { throw new CoreException("Unexpected null entity. This should never happen."); } /* * Determine if the entities are actually different in any relevant way. If so, then we can * decide how to create the feature change. */ /* * Detect changed tags. */ AtlasDiffHelper.getTagChangeIfNecessary(beforeEntity, afterEntity, beforeViewAtlas, saveAllGeometries).ifPresent(featureChanges::add); /* * Detect if the entity changed its parent relation membership. */ AtlasDiffHelper.getParentRelationMembershipChangeIfNecessary(beforeEntity, afterEntity, beforeViewAtlas, saveAllGeometries).ifPresent(featureChanges::add); /* * Detect if the entities were Nodes and some Node properties changed. We check for Node * locations, as well as inEdges and outEdges. */ if (entity instanceof Node) { AtlasDiffHelper.getNodeChangeIfNecessary((Node) beforeEntity, (Node) afterEntity, beforeViewAtlas, saveAllGeometries).ifPresent(featureChanges::add); } /* * Detect if the entities were Edges and some Edge properties changed. We check for Edge * polylines, as well as the start and end Node. */ else if (entity instanceof Edge) { AtlasDiffHelper.getEdgeChangeIfNecessary((Edge) beforeEntity, (Edge) afterEntity, beforeViewAtlas, saveAllGeometries).ifPresent(featureChanges::add); } /* * Detect if the entities were Points and some Point properties changed. We just check the * location. */ else if (entity instanceof Point) { AtlasDiffHelper.getPointChangeIfNecessary((Point) beforeEntity, (Point) afterEntity, beforeViewAtlas).ifPresent(featureChanges::add); } /* * Detect if the entities were Lines and some Line properties changed. We just check the * polyline. */ else if (entity instanceof Line) { AtlasDiffHelper.getLineChangeIfNecessary((Line) beforeEntity, (Line) afterEntity, beforeViewAtlas).ifPresent(featureChanges::add); } /* * Detect if the entities were Areas and some Area properties changed. We just check the * polygon. */ else if (entity instanceof Area) { AtlasDiffHelper.getAreaChangeIfNecessary((Area) beforeEntity, (Area) afterEntity, beforeViewAtlas).ifPresent(featureChanges::add); } /* * Detect if the entities were Relations and some Relation properties changed. We check the * member lists for changes (changes include a removed or added member or a member role * change). Note that the relation diff checking is not deep, i.e. it only looks at the * relation data itself (member ID, role, and type). For example, if a contained Area's * geometry or tags change, this will not generate a relation change for relations * containing that Area. */ else if (entity instanceof Relation) { AtlasDiffHelper.getRelationChangeIfNecessary((Relation) beforeEntity, (Relation) afterEntity, beforeViewAtlas).ifPresent(featureChanges::add); } return featureChanges; } /** * Create a simple {@link FeatureChange} ie. a REMOVE, or an ADD that is adding a new feature * (as opposed to modifying an existing feature). The feature change will be created from the * given entity in the given atlas. * * @param changeType * the change type to use, e.g. ADD or REMOVE * @param entity * the entity to add or remove * @param atlasContainingTheEntity * the atlas that actually contains this entity. In the ADD case, this should be the * afterAtlas, since we are adding a brand new feature not found in the beforeAtlas. * In the REMOVE case, this should be the beforeAtlas, since we are removing a * feature that only exists in the beforeAtlas. * @param beforeViewAtlas * the atlas from which to compute the {@link FeatureChange} beforeView. This should * always be the beforeAtlas. In the ADD case, the beforeView will be null since this * is a brand new feature. In the REMOVE case, the beforeView will be fully * populated, since we are wiping the entity. * @param saveAllGeometries * if we are saving all geometries, even when they are not modified * @return the feature change */ private FeatureChange createSimpleFeatureChangeWithType(final ChangeType changeType, final AtlasEntity entity, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { final FeatureChange featureChange; switch (entity.getType()) { case NODE: featureChange = AtlasDiffHelper.simpleCompleteNodeChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity, saveAllGeometries); break; case EDGE: featureChange = AtlasDiffHelper.simpleCompleteEdgeChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity, saveAllGeometries); break; case POINT: featureChange = AtlasDiffHelper.simpleCompletePointChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity, saveAllGeometries); break; case LINE: featureChange = AtlasDiffHelper.simpleCompleteLineChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity, saveAllGeometries); break; case AREA: featureChange = AtlasDiffHelper.simpleCompleteAreaChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity, saveAllGeometries); break; case RELATION: featureChange = AtlasDiffHelper.simpleCompleteRelationChange(changeType, atlasContainingTheEntity, beforeViewAtlas, entity); break; default: throw new CoreException("Unknown item type {}", entity.getType()); } return featureChange; } /** * Check if a given entity is missing from a given atlas. Optionally, we can match using the * underlying geometry if the itemType/identifier check fails. * * @param entity * the entity to check for * @param atlasToCheck * the atlas to check * @return if the entity was missing from the atlas */ private boolean isEntityMissingFromGivenAtlas(final AtlasEntity entity, final Atlas atlasToCheck) { /* * Look up the given entity's ID in the atlasToCheck. If the returned entity is null, we * know it was NOT PRESENT in the atlasToCheck. */ return entity.getType().entityForIdentifier(atlasToCheck, entity.getIdentifier()) == null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/diff/AtlasDiffHelper.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.diff; import java.util.Iterator; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.change.ChangeType; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.converters.jts.JtsPrecisionManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A helper class for {@link AtlasDiff}. Contains lots of static utilities. * * @author lcram */ public final class AtlasDiffHelper { private static final Logger logger = LoggerFactory.getLogger(AtlasDiffHelper.class); public static Optional getAreaChangeIfNecessary(final Area beforeArea, final Area afterArea, final Atlas beforeViewAtlas) { try { boolean featureChangeWouldBeUseful = false; final CompleteArea completeArea = CompleteArea.shallowFrom(afterArea); if (!beforeArea.asPolygon().equals(afterArea.asPolygon())) { completeArea.withPolygon(afterArea.asPolygon()); featureChangeWouldBeUseful = true; } if (featureChangeWouldBeUseful) { return Optional.of(FeatureChange.add(completeArea, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare areas {} and {}", beforeArea, afterArea, exception); } return Optional.empty(); } public static Optional getEdgeChangeIfNecessary(final Edge beforeEdge, final Edge afterEdge, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { try { boolean featureChangeWouldBeUseful = false; final CompleteEdge completeEdge = CompleteEdge.shallowFrom(afterEdge); if (!beforeEdge.asPolyLine().equals(afterEdge.asPolyLine())) { completeEdge.withPolyLine(afterEdge.asPolyLine()); featureChangeWouldBeUseful = true; } if (beforeEdge.start().getIdentifier() != afterEdge.start().getIdentifier()) { completeEdge.withStartNodeIdentifier(afterEdge.start().getIdentifier()); featureChangeWouldBeUseful = true; } if (beforeEdge.end().getIdentifier() != afterEdge.end().getIdentifier()) { completeEdge.withEndNodeIdentifier(afterEdge.end().getIdentifier()); featureChangeWouldBeUseful = true; } if (featureChangeWouldBeUseful) { /* * Explicitly check for saveAllGeometry. This will resave if the featureChange was * due to a geometry change, but that is OK. We want to ensure we save in the cases * where a start/end node was changed (for visualization purposes). */ if (saveAllGeometries) { completeEdge.withPolyLine(afterEdge.asPolyLine()); } return Optional.of(FeatureChange.add(completeEdge, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare edges {} and {}", beforeEdge, afterEdge, exception); } return Optional.empty(); } public static Optional getLineChangeIfNecessary(final Line beforeLine, final Line afterLine, final Atlas beforeViewAtlas) { try { boolean featureChangeWouldBeUseful = false; final CompleteLine completeLine = CompleteLine.shallowFrom(afterLine); if (!beforeLine.asPolyLine().equals(afterLine.asPolyLine())) { completeLine.withPolyLine(afterLine.asPolyLine()); featureChangeWouldBeUseful = true; } if (featureChangeWouldBeUseful) { return Optional.of(FeatureChange.add(completeLine, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare lines {} and {}", beforeLine, afterLine, exception); } return Optional.empty(); } public static Optional getNodeChangeIfNecessary(final Node beforeNode, final Node afterNode, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { try { boolean featureChangeWouldBeUseful = false; final CompleteNode completeNode = CompleteNode.shallowFrom(afterNode); if (!beforeNode.getLocation().equals(afterNode.getLocation())) { completeNode.withLocation(afterNode.getLocation()); featureChangeWouldBeUseful = true; } if (differentEdgeSet(beforeNode.inEdges(), afterNode.inEdges())) { completeNode.withInEdgeIdentifiersAndSource(new TreeSet<>(afterNode.inEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())), beforeNode); if (saveAllGeometries) { completeNode.withLocation(afterNode.getLocation()); } featureChangeWouldBeUseful = true; } if (differentEdgeSet(beforeNode.outEdges(), afterNode.outEdges())) { completeNode.withOutEdgeIdentifiersAndSource(new TreeSet<>(afterNode.outEdges() .stream().map(Edge::getIdentifier).collect(Collectors.toSet())), beforeNode); if (saveAllGeometries) { completeNode.withLocation(afterNode.getLocation()); } featureChangeWouldBeUseful = true; } if (featureChangeWouldBeUseful) { return Optional.of(FeatureChange.add(completeNode, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare nodes {} and {}", beforeNode, afterNode, exception); } return Optional.empty(); } public static Optional getParentRelationMembershipChangeIfNecessary( final AtlasEntity beforeEntity, final AtlasEntity afterEntity, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { try { final Set beforeRelationIdentifiers = beforeEntity.relations().stream() .map(Relation::getIdentifier).collect(Collectors.toSet()); final Set afterRelationIdentifiers = afterEntity.relations().stream() .map(Relation::getIdentifier).collect(Collectors.toSet()); /* * We never had any parent relations. We want to explicitly return empty so as not to * generate a misleading FeatureChange changing a null into an empty set. */ if (beforeRelationIdentifiers.isEmpty() && afterRelationIdentifiers.isEmpty()) { return Optional.empty(); } /* * If the relation identifier sets are equivalent, then there was no parent relation * membership change. */ if (beforeRelationIdentifiers.equals(afterRelationIdentifiers)) { return Optional.empty(); } /* * OK! We made it here because we have confirmed that the entities have differing * declared parent relation identifiers. We create a feature change to reflect this. */ final AtlasEntity completeEntity; switch (afterEntity.getType()) { case AREA: CompleteArea area = CompleteArea.shallowFrom((Area) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); if (saveAllGeometries) { area = area.withPolygon(((Area) afterEntity).asPolygon()); } completeEntity = area; break; case EDGE: CompleteEdge edge = CompleteEdge.shallowFrom((Edge) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); if (saveAllGeometries) { edge = edge.withPolyLine(((Edge) afterEntity).asPolyLine()); } completeEntity = edge; break; case LINE: CompleteLine line = CompleteLine.shallowFrom((Line) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); if (saveAllGeometries) { line = line.withPolyLine(((Line) afterEntity).asPolyLine()); } completeEntity = line; break; case NODE: CompleteNode node = CompleteNode.shallowFrom((Node) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); if (saveAllGeometries) { node = node.withLocation(((Node) afterEntity).getLocation()); } completeEntity = node; break; case POINT: CompletePoint point = CompletePoint.shallowFrom((Point) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); if (saveAllGeometries) { point = point.withLocation(((Point) afterEntity).getLocation()); } completeEntity = point; break; case RELATION: final CompleteRelation relation = CompleteRelation .shallowFrom((Relation) afterEntity) .withRelationIdentifiers(afterRelationIdentifiers); completeEntity = relation; break; default: throw new CoreException("Unknown item type {}", afterEntity.getType()); } // featureChange should never be null return Optional.of(FeatureChange.add(completeEntity, beforeViewAtlas)); } catch (final Exception exception) { throw new CoreException("Unable to compare relations for {} and {}", beforeEntity, afterEntity, exception); } } public static Optional getPointChangeIfNecessary(final Point beforePoint, final Point afterPoint, final Atlas beforeViewAtlas) { try { boolean featureChangeWouldBeUseful = false; final CompletePoint completePoint = CompletePoint.shallowFrom(afterPoint); if (!beforePoint.getLocation().equals(afterPoint.getLocation())) { completePoint.withLocation(afterPoint.getLocation()); featureChangeWouldBeUseful = true; } if (featureChangeWouldBeUseful) { return Optional.of(FeatureChange.add(completePoint, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare points {} and {}", beforePoint, afterPoint, exception); } return Optional.empty(); } public static Optional getRelationChangeIfNecessary( final Relation beforeRelation, final Relation afterRelation, final Atlas beforeViewAtlas) { try { boolean featureChangeWouldBeUseful = false; final CompleteRelation completeRelation = CompleteRelation.shallowFrom(afterRelation); final RelationMemberList beforeMembers = beforeRelation.members(); final RelationMemberList afterMembers = afterRelation.members(); if (!afterMembers.equals(beforeMembers)) { completeRelation.withMembersAndSource(afterRelation.members(), beforeRelation); featureChangeWouldBeUseful = true; } if (beforeRelation.isGeometric()) { final Optional afterGeom = afterRelation.asMultiPolygon(); if (afterGeom.isPresent()) { final Optional beforeGeom = beforeRelation.asMultiPolygon(); if (beforeGeom.isPresent() && beforeGeom.get().equals(afterGeom.get())) { // nothing to see here, move along! } else { completeRelation.withMultiPolygonGeometry(afterGeom.get()); featureChangeWouldBeUseful = true; } } else if (beforeRelation.asMultiPolygon().isPresent()) { completeRelation.withMultiPolygonGeometry( JtsPrecisionManager.getGeometryFactory().createMultiPolygon()); featureChangeWouldBeUseful = true; } } if (featureChangeWouldBeUseful) { return Optional.of(FeatureChange.add(completeRelation, beforeViewAtlas)); } } catch (final Exception exception) { throw new CoreException("Unable to compare relations {} and {}", beforeRelation, afterRelation, exception); } return Optional.empty(); } public static Optional getTagChangeIfNecessary(final AtlasEntity beforeEntity, final AtlasEntity afterEntity, final Atlas beforeViewAtlas, final boolean saveAllGeometries) { if (beforeEntity.getTags().equals(afterEntity.getTags())) { return Optional.empty(); } final AtlasEntity completeEntity; switch (afterEntity.getType()) { case AREA: CompleteArea area = CompleteArea.shallowFrom((Area) afterEntity) .withTags(afterEntity.getTags()); if (saveAllGeometries) { area = area.withPolygon(((Area) afterEntity).asPolygon()); } completeEntity = area; break; case EDGE: CompleteEdge edge = CompleteEdge.shallowFrom((Edge) afterEntity) .withTags(afterEntity.getTags()); if (saveAllGeometries) { edge = edge.withPolyLine(((Edge) afterEntity).asPolyLine()); } completeEntity = edge; break; case LINE: CompleteLine line = CompleteLine.shallowFrom((Line) afterEntity) .withTags(afterEntity.getTags()); if (saveAllGeometries) { line = line.withPolyLine(((Line) afterEntity).asPolyLine()); } completeEntity = line; break; case NODE: CompleteNode node = CompleteNode.shallowFrom((Node) afterEntity) .withTags(afterEntity.getTags()); if (saveAllGeometries) { node = node.withLocation(((Node) afterEntity).getLocation()); } completeEntity = node; break; case POINT: CompletePoint point = CompletePoint.shallowFrom((Point) afterEntity) .withTags(afterEntity.getTags()); if (saveAllGeometries) { point = point.withLocation(((Point) afterEntity).getLocation()); } completeEntity = point; break; case RELATION: final CompleteRelation relation = CompleteRelation .shallowFrom((Relation) afterEntity).withTags(afterEntity.getTags()); completeEntity = relation; break; default: throw new CoreException("Unknown item type {}", afterEntity.getType()); } return Optional.of(FeatureChange.add(completeEntity, beforeViewAtlas)); } public static FeatureChange simpleCompleteAreaChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity, final boolean saveAllGeometries) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { CompleteArea completeArea = CompleteArea .shallowFrom(atlasContainingTheEntity.area(entityIdentifier)); if (saveAllGeometries) { completeArea = completeArea.withPolygon(((Area) entity).asPolygon()); } return FeatureChange.remove(completeArea, beforeViewAtlas); } else { return FeatureChange.add( CompleteArea.from(atlasContainingTheEntity.area(entityIdentifier)), beforeViewAtlas); } } public static FeatureChange simpleCompleteEdgeChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity, final boolean saveAllGeometries) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { CompleteEdge completeEdge = CompleteEdge .shallowFrom(atlasContainingTheEntity.edge(entityIdentifier)); if (saveAllGeometries) { completeEdge = completeEdge.withPolyLine(((Edge) entity).asPolyLine()); } return FeatureChange.remove(completeEdge, beforeViewAtlas); } else { return FeatureChange.add( CompleteEdge.from(atlasContainingTheEntity.edge(entityIdentifier)), beforeViewAtlas); } } public static FeatureChange simpleCompleteLineChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity, final boolean saveAllGeometries) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { CompleteLine completeLine = CompleteLine .shallowFrom(atlasContainingTheEntity.line(entityIdentifier)); if (saveAllGeometries) { completeLine = completeLine.withPolyLine(((Line) entity).asPolyLine()); } return FeatureChange.remove(completeLine, beforeViewAtlas); } else { return FeatureChange.add( CompleteLine.from(atlasContainingTheEntity.line(entityIdentifier)), beforeViewAtlas); } } public static FeatureChange simpleCompleteNodeChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity, final boolean saveAllGeometries) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { CompleteNode completeNode = CompleteNode .shallowFrom(atlasContainingTheEntity.node(entityIdentifier)); if (saveAllGeometries) { completeNode = completeNode.withLocation(((Node) entity).getLocation()); } return FeatureChange.remove(completeNode, beforeViewAtlas); } else { return FeatureChange.add( CompleteNode.from(atlasContainingTheEntity.node(entityIdentifier)), beforeViewAtlas); } } public static FeatureChange simpleCompletePointChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity, final boolean saveAllGeometries) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { CompletePoint completePoint = CompletePoint .shallowFrom(atlasContainingTheEntity.point(entityIdentifier)); if (saveAllGeometries) { completePoint = completePoint.withLocation(((Point) entity).getLocation()); } return FeatureChange.remove(completePoint, beforeViewAtlas); } else { return FeatureChange.add( CompletePoint.from(atlasContainingTheEntity.point(entityIdentifier)), beforeViewAtlas); } } public static FeatureChange simpleCompleteRelationChange(final ChangeType changeType, final Atlas atlasContainingTheEntity, final Atlas beforeViewAtlas, final AtlasEntity entity) { final Long entityIdentifier = entity.getIdentifier(); if (changeType == ChangeType.REMOVE) { final CompleteRelation completeRelation = CompleteRelation .shallowFrom(atlasContainingTheEntity.relation(entityIdentifier)); return FeatureChange.remove(completeRelation, beforeViewAtlas); } else { return FeatureChange.add( CompleteRelation.from(atlasContainingTheEntity.relation(entityIdentifier)), beforeViewAtlas); } } /* * NOTE: this method considers two edges to be distinct if and only if they have distinct * identifiers. */ private static boolean differentEdgeSet(final SortedSet beforeEdges, final SortedSet afterEdges) { if (beforeEdges.size() != afterEdges.size()) { return true; } final Iterator beforeInEdgeIterator = beforeEdges.iterator(); final Iterator afterInEdgeIterator = afterEdges.iterator(); for (int i = 0; i < beforeEdges.size(); i++) { final Edge beforeInEdge = beforeInEdgeIterator.next(); final Edge afterInEdge = afterInEdgeIterator.next(); if (beforeInEdge.getIdentifier() != afterInEdge.getIdentifier()) { return true; } } return false; } private AtlasDiffHelper() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/event/EntityChangeEvent.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.event; import java.io.Serializable; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.complete.CompleteItemType; /** * An abstract representation of a change in a * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}. * * @author Yazad Khambata */ public abstract class EntityChangeEvent implements Serializable { private static final long serialVersionUID = -9159065869173338344L; private final CompleteItemType completeItemType; private final long identifier; private final Optional newValue; public EntityChangeEvent(final CompleteItemType completeItemType, final long identifier) { this(completeItemType, identifier, Optional.empty()); } public EntityChangeEvent(final CompleteItemType completeItemType, final long identifier, final Optional newValue) { super(); this.completeItemType = completeItemType; this.identifier = identifier; this.newValue = newValue; } public CompleteItemType getCompleteItemType() { return this.completeItemType; } public long getIdentifier() { return this.identifier; } public Optional getNewValue() { return this.newValue; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/event/TagChangeEvent.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.event; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.consts.FieldChangeOperation; import org.openstreetmap.atlas.geography.atlas.complete.CompleteItemType; /** * Represents a tag change event in a * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}. * * @author Yazad Khambata */ public class TagChangeEvent extends EntityChangeEvent { private static final long serialVersionUID = -3108915161471760840L; private final FieldChangeOperation fieldOperation; public static TagChangeEvent added(final CompleteItemType completeItemType, final long identifier, final Pair addedTagPair) { return new TagChangeEvent(completeItemType, identifier, Optional.of(addedTagPair), FieldChangeOperation.ADD); } public static TagChangeEvent overwrite(final CompleteItemType completeItemType, final long identifier, final Map newTags) { return new TagChangeEvent(completeItemType, identifier, Optional.ofNullable(newTags), FieldChangeOperation.OVERWRITE); } public static TagChangeEvent remove(final CompleteItemType completeItemType, final long identifier, final String key) { return new TagChangeEvent(completeItemType, identifier, Optional.of(key), FieldChangeOperation.REMOVE); } public static TagChangeEvent replaced(final CompleteItemType completeItemType, final long identifier, final Triple tagReplacementInfo) { return new TagChangeEvent(completeItemType, identifier, Optional.of(tagReplacementInfo), FieldChangeOperation.REPLACE); } protected TagChangeEvent(final CompleteItemType completeItemType, final long identifier, final Optional newValue, final FieldChangeOperation fieldOperation) { super(completeItemType, identifier, newValue); this.fieldOperation = fieldOperation; } public FieldChangeOperation getFieldOperation() { return this.fieldOperation; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/event/consts/FieldChangeOperation.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.consts; /** * Indicates an operation being performed on a field of a * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}, like tags, geometry, * relations, etc. * * @author Yazad Khambata */ public enum FieldChangeOperation { ADD, REMOVE, REPLACE, OVERWRITE; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/listenable/EntityChangeListenable.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.listenable; /** * An object that can be "listened" to. Typically a * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}. * * @author Yazad Khambata */ public interface EntityChangeListenable { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/listenable/TagChangeListenable.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.listenable; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; /** * A {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity} whose tag changes can * be "listened" to. * * @author Yazad Khambata */ public interface TagChangeListenable extends EntityChangeListenable { void addTagChangeListener(TagChangeListener tagChangeListener); void fireTagChangeEvent(TagChangeEvent tagChangeEvent); void removeTagChangeListeners(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/listener/EntityChangeListener.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener; import java.io.Serializable; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.EntityChangeEvent; /** * The basic contract for tracking changes to a * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}. * * @param * - the entity being observed. * @author Yazad Khambata */ @FunctionalInterface public interface EntityChangeListener extends Serializable { void entityChanged(E entityChangeEvent); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/eventhandling/listener/TagChangeListener.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; /** * Tracking changes to the tags of the * {@link org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity}. * * @author Yazad Khambata */ public interface TagChangeListener extends EntityChangeListener { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/exception/EmptyChangeException.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.exception; import org.openstreetmap.atlas.exception.CoreException; /** * @author Yazad Khambata */ public class EmptyChangeException extends CoreException { private static final String MESSAGE = "Change cannot be empty."; public EmptyChangeException() { super(messageWithToken(MESSAGE)); } public EmptyChangeException(final Throwable cause) { super(messageWithToken(MESSAGE), cause); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/serializer/ChangeGeoJsonSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.serializer; import java.io.IOException; import java.io.Writer; import java.lang.reflect.Type; import java.util.function.BiConsumer; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.Change; import org.openstreetmap.atlas.geography.atlas.change.serializer.FeatureChangeGeoJsonSerializer.FeatureChangeTypeHierarchyAdapter; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; /** * @author matthieun */ public class ChangeGeoJsonSerializer implements BiConsumer, Converter { /** * @author matthieun */ private static class ChangeTypeHierarchyAdapter implements JsonSerializer { private final FeatureChangeTypeHierarchyAdapter featureChangeTypeHierarchyAdapter; ChangeTypeHierarchyAdapter(final boolean showDescription) { this.featureChangeTypeHierarchyAdapter = new FeatureChangeTypeHierarchyAdapter( showDescription); } @Override public JsonElement serialize(final Change source, final Type typeOfSource, final JsonSerializationContext context) { final JsonObject result = new JsonObject(); result.addProperty("type", "FeatureCollection"); final Rectangle bounds = source.bounds(); result.add("bbox", bounds.asGeoJsonBbox()); final JsonArray features = new JsonArray(); source.changes().map(this.featureChangeTypeHierarchyAdapter::serialize) .forEach(features::add); result.add("features", features); final JsonObject properties = new JsonObject(); properties.addProperty("bboxWKT", source.bounds().toWkt()); result.add("properties", properties); return result; } } private final Gson jsonSerializer; public ChangeGeoJsonSerializer() { this(true); } public ChangeGeoJsonSerializer(final boolean prettyPrint, final boolean showDescription) { final GsonBuilder gsonBuilder = new GsonBuilder(); if (prettyPrint) { gsonBuilder.setPrettyPrinting(); } gsonBuilder.disableHtmlEscaping(); gsonBuilder.registerTypeHierarchyAdapter(Change.class, new ChangeTypeHierarchyAdapter(showDescription)); this.jsonSerializer = gsonBuilder.create(); } public ChangeGeoJsonSerializer(final boolean prettyPrint) { this(prettyPrint, true); } @Override public void accept(final Change change, final WritableResource resource) { try (Writer writer = resource.writer()) { this.jsonSerializer.toJson(change, writer); } catch (final IOException e) { throw new CoreException("Could not save FeatureChange to resource {}", resource.getName(), e); } } @Override public String convert(final Change change) { return this.jsonSerializer.toJson(change); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/serializer/FeatureChangeGeoJsonSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.serializer; import java.io.IOException; import java.io.Writer; import java.lang.reflect.Type; import java.util.Map; import java.util.Optional; import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.Function; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometryPrintable; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.change.description.ChangeDescription; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; /** * @author matthieun */ public class FeatureChangeGeoJsonSerializer implements BiConsumer, Converter { /** * @author matthieun */ protected static class FeatureChangeTypeHierarchyAdapter implements JsonSerializer { private final boolean showDescription; private static void addGeometryGeojson(final JsonObject result, final GeometryPrintable property) { add(result, "geometry", property, GeometryPrintable::asGeoJson); } private static void addGeometryWkt(final JsonObject result, final GeometryPrintable property) { addProperty(result, "WKT", property, GeometryPrintable::toWkt); } FeatureChangeTypeHierarchyAdapter(final boolean showDescription) { this.showDescription = showDescription; } public JsonElement serialize(final FeatureChange source) { final JsonObject result = new JsonObject(); result.addProperty("type", "Feature"); final Rectangle bounds = source.bounds(); result.add("bbox", bounds.asGeoJsonBbox()); final GeometryPrintable geometryPrintable = new AtlasEntityGeometryPrintableConverter() .convert(source); addGeometryGeojson(result, geometryPrintable); final JsonObject properties = new JsonObject(); properties.addProperty("featureChangeType", source.getChangeType().toString()); add(properties, "metadata", source.getMetaData(), tagPrinter); if (this.showDescription) { add(properties, "description", source.explain(), ChangeDescription::toJsonElement); } new AtlasEntityPropertiesConverter().convert(source.getAfterView()).entrySet() .forEach(entry -> properties.add(entry.getKey(), entry.getValue())); addGeometryWkt(properties, geometryPrintable); properties.addProperty("bboxWKT", source.bounds().toWkt()); result.add("properties", properties); return result; } @Override public JsonElement serialize(final FeatureChange source, final Type typeOfSource, final JsonSerializationContext context) { return serialize(source); } } /** * @author matthieun */ private static class AtlasEntityGeometryPrintableConverter implements Converter { @Override public GeometryPrintable convert(final FeatureChange featureChange) { final AtlasEntity source = featureChange.getAfterView(); GeometryPrintable result; if (source instanceof Area) { result = ((Area) source).asPolygon(); if (result == null && featureChange.getBeforeView() != null) { result = ((Area) featureChange.getBeforeView()).asPolygon(); } } else if (source instanceof LineItem) { result = ((LineItem) source).asPolyLine(); if (result == null && featureChange.getBeforeView() != null) { result = ((LineItem) featureChange.getBeforeView()).asPolyLine(); } } else if (source instanceof LocationItem) { result = ((LocationItem) source).getLocation(); if (result == null && featureChange.getBeforeView() != null) { result = ((LocationItem) featureChange.getBeforeView()).getLocation(); } } else { // Relation final Optional geom = ((Relation) source).asMultiPolygon(); if (geom.isPresent()) { result = new JtsMultiPolygonToMultiPolygonConverter().convert(geom.get()); } else { result = ((Relation) source).bounds(); } } if (result == null) { result = source.bounds(); } return result; } } /** * @author matthieun */ private static class AtlasEntityPropertiesConverter implements Converter { @Override public JsonObject convert(final AtlasEntity source) { final JsonObject properties = new JsonObject(); properties.addProperty("entityType", source.getType().toString()); properties.addProperty("completeEntityClass", source.getClass().getName()); properties.addProperty("identifier", source.getIdentifier()); add(properties, "tags", source.getTags(), tagPrinter); add(properties, "relations", source.relations(), identifierMapper); if (source instanceof Edge) { addProperty(properties, "startNode", ((Edge) source).start(), Node::getIdentifier); addProperty(properties, "endNode", ((Edge) source).end(), Node::getIdentifier); } else if (source instanceof Node) { add(properties, "inEdges", ((Node) source).inEdges(), identifierMapper); add(properties, "outEdges", ((Node) source).outEdges(), identifierMapper); } else if (source instanceof Relation) { // Relation final Relation relation = (Relation) source; add(properties, "members", relation.members(), members -> { final JsonArray beanResult = new JsonArray(); members.forEach(member -> beanResult.add(new JsonPrimitive(member.toString()))); return beanResult; }); } return properties; } } private static final Function, JsonElement> identifierMapper = entity -> { final JsonArray result = new JsonArray(); Iterables.stream(entity).map(AtlasEntity::getIdentifier).collectToSortedSet() .forEach(number -> result.add(new JsonPrimitive(number))); return result; }; private static final Function, JsonElement> tagPrinter = map -> { final JsonObject result = new JsonObject(); final Map sortedMap = new TreeMap<>(map); for (final Map.Entry entry : sortedMap.entrySet()) { result.addProperty(entry.getKey(), entry.getValue()); } return result; }; private final Gson jsonSerializer; private static void add(final JsonObject result, final String name, final T property, final Function writer) { if (property == null) { result.addProperty(name, (String) null); } else { result.add(name, writer.apply(property)); } } private static void addProperty(final JsonObject result, final String name, final T property, final Function writer) { result.addProperty(name, property == null ? null : writer.apply(property).toString()); } public FeatureChangeGeoJsonSerializer(final boolean prettyPrint) { this(prettyPrint, true); } public FeatureChangeGeoJsonSerializer(final boolean prettyPrint, final boolean showDescription) { final GsonBuilder gsonBuilder = new GsonBuilder(); if (prettyPrint) { gsonBuilder.setPrettyPrinting(); } gsonBuilder.disableHtmlEscaping(); gsonBuilder.registerTypeHierarchyAdapter(FeatureChange.class, new FeatureChangeTypeHierarchyAdapter(showDescription)); this.jsonSerializer = gsonBuilder.create(); } @Override public void accept(final FeatureChange featureChange, final WritableResource resource) { try (Writer writer = resource.writer()) { this.jsonSerializer.toJson(featureChange, writer); } catch (final IOException e) { throw new CoreException("Could not save FeatureChange to resource {}", resource.getName(), e); } } @Override public String convert(final FeatureChange featureChange) { return this.jsonSerializer.toJson(featureChange); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/testing/AtlasChangeGeneratorAddTurnRestrictions.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.testing; import java.util.HashSet; import java.util.Set; import java.util.SortedSet; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.change.AtlasChangeGenerator; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.Sets; import com.google.common.collect.Lists; /** * @author matthieun */ public class AtlasChangeGeneratorAddTurnRestrictions implements AtlasChangeGenerator { private static final long serialVersionUID = -518515697422424803L; private static final int MINIMUM_NODE_VALENCE = 3; private final int minimumNodeValence; public AtlasChangeGeneratorAddTurnRestrictions() { this(MINIMUM_NODE_VALENCE); } public AtlasChangeGeneratorAddTurnRestrictions(final int minimumNodeValence) { this.minimumNodeValence = minimumNodeValence; } @Override public Set generateWithoutValidation(final Atlas atlas) { final AtomicLong identifierGenerator = new AtomicLong(); final Set result = new HashSet<>(); final Long parentRelationIdentifier = identifierGenerator.incrementAndGet(); final RelationBean parentMembers = new RelationBean(); Rectangle parentBounds = null; for (final Node node : atlas.nodes(node -> node.valence() > this.minimumNodeValence)) { final SortedSet inEdges = node.inEdges(); final SortedSet outEdges = node.outEdges(); for (final Edge inEdge : inEdges) { for (final Edge outEdge : outEdges) { final RelationBean members = new RelationBean(); members.addItem(inEdge.getIdentifier(), "from", ItemType.EDGE); inEdge.reversed().ifPresent(reversed -> members .addItem(reversed.getIdentifier(), "from", ItemType.EDGE)); members.addItem(node.getIdentifier(), "via", ItemType.NODE); members.addItem(outEdge.getIdentifier(), "to", ItemType.EDGE); outEdge.reversed().ifPresent(reversed -> members .addItem(reversed.getIdentifier(), "to", ItemType.EDGE)); final Long relationIdentifier = identifierGenerator.incrementAndGet(); final Rectangle bounds = Rectangle.forLocated(inEdge, outEdge); if (parentBounds == null) { parentBounds = bounds; } else { parentBounds = Rectangle.forLocated(parentBounds, bounds); } parentMembers.addItem(relationIdentifier, "addition", ItemType.RELATION); result.add(FeatureChange.add(new CompleteRelation(relationIdentifier, Maps.hashMap("type", "restriction", "restriction", "no_left_turn"), bounds, members, Lists.newArrayList(relationIdentifier), members, relationIdentifier, Sets.hashSet(parentRelationIdentifier)))); result.add(FeatureChange .add(CompleteEdge.shallowFrom(inEdge).withRelationIdentifiers( mergeRelationMembers(inEdge.relations(), relationIdentifier)))); if (inEdge.hasReverseEdge()) { result.add( FeatureChange.add(CompleteEdge.shallowFrom(inEdge.reversed().get()) .withRelationIdentifiers(mergeRelationMembers( inEdge.relations(), relationIdentifier)))); } result.add(FeatureChange .add(CompleteNode.shallowFrom(node).withRelationIdentifiers( mergeRelationMembers(node.relations(), relationIdentifier)))); result.add(FeatureChange.add(CompleteEdge.shallowFrom(outEdge) .withRelationIdentifiers(mergeRelationMembers(outEdge.relations(), relationIdentifier)))); if (outEdge.hasReverseEdge()) { result.add( FeatureChange.add(CompleteEdge.shallowFrom(outEdge.reversed().get()) .withRelationIdentifiers(mergeRelationMembers( outEdge.relations(), relationIdentifier)))); } // Break here to avoid too many Relation FeatureChanges and make validation // super slow for unit tests. break; } // Break here to avoid too many Relation FeatureChanges and make validation super // slow for unit tests. break; } } if (!result.isEmpty()) { result.add(FeatureChange.add(new CompleteRelation(parentRelationIdentifier, Maps.hashMap("name", "parent_of_new_restrictions"), parentBounds, parentMembers, Lists.newArrayList(parentRelationIdentifier), parentMembers, parentRelationIdentifier, Sets.hashSet()))); } return result; } private Set mergeRelationMembers(final Set relations, final Long newIdentifier) { return Sets.withSets( relations.stream().map(Relation::getIdentifier).collect(Collectors.toSet()), Sets.hashSet(newIdentifier)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/testing/AtlasChangeGeneratorRemoveReverseEdges.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.testing; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.change.AtlasChangeGenerator; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author matthieun */ public class AtlasChangeGeneratorRemoveReverseEdges implements AtlasChangeGenerator { private static final long serialVersionUID = 2378086577050982603L; @Override public Set generateWithoutValidation(final Atlas atlas) { return Iterables.stream(atlas.edges()).filter(Edge::isMainEdge).filter(Edge::hasReverseEdge) .map(Edge::reversed).filter(Optional::isPresent).map(Optional::get) .map(edge -> FeatureChange.remove(CompleteEntity.shallowFrom(edge), atlas)) .collectToSet(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/testing/AtlasChangeGeneratorSplitRoundabout.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.testing; import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.change.AtlasChangeGenerator; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.tags.JunctionTag; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.Sets; /** * @author matthieun */ public class AtlasChangeGeneratorSplitRoundabout implements AtlasChangeGenerator { private static final long serialVersionUID = -6596053817414805897L; @Override public Set generateWithoutValidation(final Atlas atlas) { final AtomicLong identifierGenerator = new AtomicLong(); final Set result = new HashSet<>(); for (final Edge edge : atlas.edges(JunctionTag::isRoundabout)) { final PolyLine currentShape = edge.asPolyLine(); if (currentShape.size() > 2 && !edge.hasReverseEdge()) { // Prepare members to fill out: shapes, ids, etc. final Location cut = currentShape.get(currentShape.size() / 2); final PolyLine shape1 = currentShape.between(currentShape.first(), 0, cut, 0); final PolyLine shape2 = currentShape.between(cut, 0, currentShape.last(), currentShape.last().equals(currentShape.first()) ? 1 : 0); final long middleNodeIdentifier = identifierGenerator.incrementAndGet(); final long oldEdgeIdentifier = edge.getIdentifier(); final long newEdgeIdentifier1 = identifierGenerator.incrementAndGet(); final long newEdgeIdentifier2 = identifierGenerator.incrementAndGet(); // This is a new Edge, use "from" instead of "shallowFrom" final CompleteEdge firstEdge = CompleteEdge.from(edge) .withIdentifier(newEdgeIdentifier1).withPolyLine(shape1) .withEndNodeIdentifier(middleNodeIdentifier); // This is a new Edge, use "from" instead of "shallowFrom" final CompleteEdge secondEdge = CompleteEdge.from(edge) .withIdentifier(newEdgeIdentifier2).withPolyLine(shape2) .withStartNodeIdentifier(middleNodeIdentifier); // Update relations of edge to instead list first and second edge that have new IDs edge.relations().stream().map(relation -> { // newMembers exclude the old edge explicitly final RelationMemberList newMembers = new RelationMemberList(relation.members() .stream().filter(member -> !member.getEntity().equals(edge)) .collect(Collectors.toList())); return CompleteRelation.shallowFrom(relation) .withMembersAndSource(newMembers, relation) // With the new relation members .withAddedMember(firstEdge, edge).withAddedMember(secondEdge, edge); }).map(relation -> FeatureChange.add(relation, atlas)).forEach(result::add); // Add the two new edges. result.add(FeatureChange.remove(CompleteEdge.shallowFrom(edge), atlas)); result.add(FeatureChange.add(firstEdge, atlas)); result.add(FeatureChange.add(secondEdge, atlas)); // Middle node is new. Create from scratch result.add(FeatureChange.add(new CompleteNode(middleNodeIdentifier, cut, Maps.hashMap(), Sets.treeSet(newEdgeIdentifier1), Sets.treeSet(newEdgeIdentifier2), Sets.hashSet()), atlas)); // End node has a replaced start edge identifier result.add(FeatureChange.add(CompleteNode.shallowFrom(edge.end()) .withInEdges(edge.end().inEdges()).withReplacedInEdgeIdentifier( oldEdgeIdentifier, newEdgeIdentifier2), atlas)); // Start node has a replaced end edge identifier result.add(FeatureChange.add(CompleteNode.shallowFrom(edge.start()) .withOutEdges(edge.start().outEdges()).withReplacedOutEdgeIdentifier( oldEdgeIdentifier, newEdgeIdentifier1), atlas)); } } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/change/validators/ChangeValidator.java ================================================ package org.openstreetmap.atlas.geography.atlas.change.validators; import java.util.Optional; import java.util.function.BiPredicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.change.Change; import org.openstreetmap.atlas.geography.atlas.change.ChangeType; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.change.exception.EmptyChangeException; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Validate a {@link Change} * * @author matthieun */ public class ChangeValidator { private static final Logger logger = LoggerFactory.getLogger(ChangeValidator.class); private final Change change; public ChangeValidator(final Change change) { this.change = change; } public void validate() { logger.trace("Starting validation of Change {}", this.change.getName()); final Time start = Time.now(); validateChangeNotEmpty(); validateReverseEdgesHaveForwardMatchingCounterpart(); validateAreaGeometricRelationsUpdated(); validateLineGeometricRelationsUpdated(); validateEdgeGeometricRelationsUpdated(); logger.trace("Finished validation of Change {} in {}", this.change.getName(), start.elapsedSince()); } protected void validateAreaGeometricRelationsUpdated() { this.change.changesFor(ItemType.AREA) .filter(featureChange -> featureChange.getChangeType() != ChangeType.REMOVE) .filter(featureChange -> ((CompleteArea) featureChange.getAfterView()) .asPolygon() != null) .filter(featureChange -> ((CompleteArea) featureChange.getAfterView()) .geometricRelationIdentifiers() != null && !((CompleteArea) featureChange.getAfterView()) .geometricRelationIdentifiers().isEmpty()) .filter(featureChange -> featureChange.getBeforeView() == null || ((Area) featureChange.getBeforeView()).asPolygon() == null || !((Area) featureChange.getBeforeView()).asPolygon() .equals(((CompleteArea) featureChange.getAfterView()).asPolygon())) .forEach(featureChange -> { final CompleteArea after = (CompleteArea) featureChange.getAfterView(); after.geometricRelationIdentifiers().forEach(relationId -> { final Optional changeRelationOptional = this.change .changeFor(ItemType.RELATION, relationId); if (changeRelationOptional.isEmpty()) { throw new CoreException( "Geometric relation {} had no change for area member {} with updated geometry!", relationId, after.getIdentifier()); } if (!changeRelationOptional.get().getChangeType().equals(ChangeType.REMOVE)) { final CompleteRelation updatedRelation = (CompleteRelation) changeRelationOptional .get().getAfterView(); if (!updatedRelation.isOverrideGeometry()) { if (updatedRelation.getRemovedGeometry().isEmpty() && updatedRelation.getAddedGeometry().isEmpty()) { throw new CoreException( "Geometric relation {} had no change for area member {} with updated geometry!", relationId, after.getIdentifier()); } if (featureChange.getBeforeView() != null && ((Area) featureChange.getBeforeView()) .asPolygon() != null && !updatedRelation.getRemovedGeometry() .contains(new JtsPolyLineConverter().convert( ((Area) featureChange.getBeforeView()) .asPolygon()))) { throw new CoreException( "Geometric relation {} had no removed geometry for area member {} with updated geometry!", relationId, after.getIdentifier()); } if (!updatedRelation.getAddedGeometry().contains( new JtsPolyLineConverter().convert(after.asPolygon()))) { throw new CoreException( "Geometric relation {} had no added geometry for area member {} with updated geometry!", relationId, after.getIdentifier()); } } } }); }); } protected void validateChangeNotEmpty() { if (this.change.changeCount() == 0) { throw new EmptyChangeException(); } } protected void validateEdgeGeometricRelationsUpdated() { this.change.changesFor(ItemType.EDGE) .filter(featureChange -> featureChange.getChangeType() != ChangeType.REMOVE) .filter(featureChange -> ((Edge) featureChange.getAfterView()).asPolyLine() != null) .filter(featureChange -> ((CompleteEdge) featureChange.getAfterView()) .geometricRelationIdentifiers() != null && !((CompleteEdge) featureChange.getAfterView()) .geometricRelationIdentifiers().isEmpty()) .filter(featureChange -> featureChange.getBeforeView() == null || ((Edge) featureChange.getBeforeView()).asPolyLine() == null || !((Edge) featureChange.getBeforeView()).asPolyLine() .equals(((CompleteEdge) featureChange.getAfterView()).asPolyLine())) .forEach(featureChange -> { final CompleteEdge after = (CompleteEdge) featureChange.getAfterView(); after.geometricRelationIdentifiers().forEach(relationId -> { final Optional changeRelationOptional = this.change .changeFor(ItemType.RELATION, relationId); if (changeRelationOptional.isEmpty()) { throw new CoreException( "Geometric relation {} had no change for edge member {} with updated geometry!", relationId, after.getIdentifier()); } if (!changeRelationOptional.get().getChangeType().equals(ChangeType.REMOVE)) { final CompleteRelation updatedRelation = (CompleteRelation) changeRelationOptional .get().getAfterView(); if (!updatedRelation.isOverrideGeometry()) { if (updatedRelation.getRemovedGeometry().isEmpty() && updatedRelation.getAddedGeometry().isEmpty()) { throw new CoreException( "Geometric relation {} had no change for edge member {} with updated geometry!", relationId, after.getIdentifier()); } if (featureChange.getBeforeView() != null && ((Edge) featureChange.getBeforeView()) .asPolyLine() != null && !updatedRelation.getRemovedGeometry() .contains(new JtsPolyLineConverter().convert( ((Edge) featureChange.getBeforeView()) .asPolyLine()))) { throw new CoreException( "Geometric relation {} had no removed geometry for edge member {} with updated geometry!", relationId, after.getIdentifier()); } if (!updatedRelation.getAddedGeometry().contains( new JtsPolyLineConverter().convert(after.asPolyLine()))) { throw new CoreException( "Geometric relation {} had no added geometry for edge member {} with updated geometry!", relationId, after.getIdentifier()); } } } }); }); } protected void validateLineGeometricRelationsUpdated() { this.change.changesFor(ItemType.LINE) .filter(featureChange -> featureChange.getChangeType() != ChangeType.REMOVE) .filter(featureChange -> ((Line) featureChange.getAfterView()).asPolyLine() != null) .filter(featureChange -> ((CompleteLine) featureChange.getAfterView()) .geometricRelationIdentifiers() != null && !((CompleteLine) featureChange.getAfterView()) .geometricRelationIdentifiers().isEmpty()) .filter(featureChange -> featureChange.getBeforeView() == null || ((Line) featureChange.getBeforeView()).asPolyLine() == null || !((Line) featureChange.getBeforeView()).asPolyLine() .equals(((CompleteLine) featureChange.getAfterView()).asPolyLine())) .forEach(featureChange -> { final CompleteLine after = (CompleteLine) featureChange.getAfterView(); after.geometricRelationIdentifiers().forEach(relationId -> { final Optional changeRelationOptional = this.change .changeFor(ItemType.RELATION, relationId); if (changeRelationOptional.isEmpty()) { throw new CoreException( "Geometric relation {} had no change for Line member {} with updated geometry!", relationId, after.getIdentifier()); } if (!changeRelationOptional.get().getChangeType().equals(ChangeType.REMOVE)) { final CompleteRelation updatedRelation = (CompleteRelation) changeRelationOptional .get().getAfterView(); if (!updatedRelation.isOverrideGeometry()) { if (updatedRelation.getRemovedGeometry().isEmpty() && updatedRelation.getAddedGeometry().isEmpty()) { throw new CoreException( "Geometric relation {} had no change for Line member {} with updated geometry!", relationId, after.getIdentifier()); } if (featureChange.getBeforeView() != null && ((Line) featureChange.getBeforeView()) .asPolyLine() != null && !updatedRelation.getRemovedGeometry() .contains(new JtsPolyLineConverter().convert( ((Line) featureChange.getBeforeView()) .asPolyLine()))) { throw new CoreException( "Geometric relation {} had no removed geometry for Line member {} with updated geometry!", relationId, after.getIdentifier()); } if (!updatedRelation.getAddedGeometry().contains( new JtsPolyLineConverter().convert(after.asPolyLine()))) { throw new CoreException( "Geometric relation {} had no added geometry for Line member {} with updated geometry!", relationId, after.getIdentifier()); } } } }); }); } protected void validateReverseEdgesHaveForwardMatchingCounterpart() { this.change.changesFor(ItemType.EDGE) .filter(featureChange -> !((Edge) featureChange.getAfterView()).isMainEdge()) .filter(featureChange -> featureChange.getChangeType() != ChangeType.REMOVE) .forEach(backwardFeatureChange -> { final long backwardEdgeIdentifier = backwardFeatureChange.getAfterView() .getIdentifier(); final long forwardEdgeIdentifier = -backwardEdgeIdentifier; final Edge backwardEdge = (Edge) backwardFeatureChange.getAfterView(); final Optional forwardFeatureChangeOption = this.change .changeFor(ItemType.EDGE, forwardEdgeIdentifier); if (forwardFeatureChangeOption.isPresent()) { final FeatureChange forwardFeatureChange = forwardFeatureChangeOption.get(); if (forwardFeatureChange.getChangeType() != backwardFeatureChange .getChangeType()) { throw new CoreException( "Forward edge {} is {} when backward edge is {}", forwardEdgeIdentifier, forwardFeatureChange.getChangeType(), backwardFeatureChange.getChangeType()); } if (forwardFeatureChange.getChangeType() == ChangeType.ADD) { final Edge forwardEdge = (Edge) forwardFeatureChange.getAfterView(); validateEdgeConnectedNodesMatch(forwardEdge, backwardEdge); validateEdgePolyLinesMatch(forwardEdge, backwardEdge); } } }); } private boolean differ(final T left, final T right, final BiPredicate equal) { if (left != null && right != null) { return !equal.test(left, right); } else { return false; } } private void validateEdgeConnectedNodesMatch(final Edge forwardEdge, final Edge backwardEdge) { final BiPredicate equal = (left, right) -> left.getIdentifier() == right.getIdentifier(); final long forwardEdgeIdentifier = forwardEdge.getIdentifier(); final Node forwardStartNode = forwardEdge.start(); final Node backwardEndNode = backwardEdge.end(); if (differ(forwardStartNode, backwardEndNode, equal)) { throw new CoreException( "Forward edge {} start node {} does not match its backward edge end node {}", forwardEdgeIdentifier, forwardStartNode, backwardEndNode); } final Node forwardEndNode = forwardEdge.end(); final Node backwardStartNode = backwardEdge.start(); if (differ(forwardEndNode, backwardStartNode, equal)) { throw new CoreException( "Forward edge {} end node {} does not match its backward edge start node {}", forwardEdgeIdentifier, forwardEndNode, backwardStartNode); } } private void validateEdgePolyLinesMatch(final Edge forwardEdge, final Edge backwardEdge) { final BiPredicate equal = (left, right) -> left .equals(right.reversed()); final long forwardEdgeIdentifier = forwardEdge.getIdentifier(); final PolyLine forwardPolyLine = forwardEdge.asPolyLine(); final PolyLine backwardPolyLine = backwardEdge.asPolyLine(); if (differ(forwardPolyLine, backwardPolyLine, equal)) { throw new CoreException( "Forward edge {} polyline {} does not match its backward edge polyline {}", forwardEdgeIdentifier, forwardPolyLine, backwardPolyLine); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/BinaryChangeSetDeserializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.EOFException; import java.io.IOException; import java.io.ObjectInputStream; import java.io.OutputStream; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Deserializes {@link ChangeSet} objects from {@link OutputStream}s back into {@link ChangeSet} * objects. * * @author mkalender * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class BinaryChangeSetDeserializer implements ChangeSetDeserializer { private static final Logger logger = LoggerFactory.getLogger(BinaryChangeSetDeserializer.class); private final ObjectInputStream resource; private boolean hasMore; public BinaryChangeSetDeserializer(final Resource resourceToReadFrom) throws IOException { this.resource = new ObjectInputStream(resourceToReadFrom.read()); this.hasMore = true; } @Override public void close() throws Exception { this.resource.close(); } @Override public Optional get() { if (!this.hasMore) { return Optional.empty(); } ChangeSet changeSet = null; try { changeSet = (ChangeSet) this.resource.readObject(); } catch (final EOFException e) { this.hasMore = false; try { this.close(); } catch (final Exception closeException) { logger.error("ChangeSet resource close is failed.", closeException); } } catch (final ClassNotFoundException | IOException e) { logger.error("ChangeSet deserialization is failed.", e); } return Optional.ofNullable(changeSet); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/BinaryChangeSetSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.IOException; import java.io.ObjectOutputStream; import java.io.OutputStream; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Serializes {@link ChangeSet} objects and writes them into {@link OutputStream}s in binary format. * * @author mkalender * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class BinaryChangeSetSerializer implements ChangeSetSerializer { private static final Logger logger = LoggerFactory.getLogger(BinaryChangeSetSerializer.class); private final ObjectOutputStream resource; public BinaryChangeSetSerializer(final WritableResource resourceToWriteInto) throws IOException { this.resource = new ObjectOutputStream(resourceToWriteInto.write()); } @Override public void accept(final ChangeSet changeSet) { try { this.resource.writeObject(changeSet); this.resource.flush(); } catch (final IOException e) { logger.error("ChangeSet serialization is failed.", e); } } @Override public void close() throws Exception { this.resource.close(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeAction.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; /** * Enumeration of all action types of a change. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public enum ChangeAction { CREATE, UPDATE, DELETE, READ } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.Serializable; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.tags.Taggable; /** * A change item represent a result generated from data enhancement process. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface ChangeItem extends Taggable, Serializable { /** * @return action type of this change item */ ChangeAction getAction(); /** * this needs further thought. We should use a Atlas data type but right now there is no generic * type for all geometry types. * * @return the geometry of the change item, if has it. */ Iterable getGeometry(); /** * identifier of change item. It's required to be unique within one change set. For newly * created item please use negative value * * @return the unique identifier of change item */ long getIdentifier(); /** * @return item member list if exists. It should only have value if ItemType is RELATION */ Iterable getMembers(); /** * create a new RelationBean object from item members * * @return a relation bean object to use for AtlasBuilder. */ Optional getRelationBean(); /** * @return the score of the item, actual meaning varies base on source type and conflation * process, value should always be between 0-1 with 0 for lowest rank and 1 for highest * rank. If score is not needed it should always return 1. */ default double getScore() { return 1; } /** * @return the source name from which the change set is generated. */ String getSourceName(); /** * @return tag map of this item */ @Override Map getTags(); /** * @return item type */ ItemType getType(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeItemMember.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * Represents a member of a {@link ChangeItem}, only valid if {@link ChangeItem} type is * {@code ItemType.RELATION}. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface ChangeItemMember { /** * @return id of the item member, which is a reference of the member's entity id. */ long getIdentifier(); /** * @return role of the member. refer to OSM WIKI for details about roles in different relation * type. */ String getRole(); /** * @return ItemType of the member */ ItemType getType(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeSet.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.Serializable; import java.util.Iterator; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * A ChangeSet is a set of {@link ChangeItem} generated from side files for a specific Atlas data * set. A side file is a data file generated from other sources or validation program to improve the * data quality and/or coverage. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface ChangeSet extends Set, Serializable { /** * @param identifier * the identifier of changeItem * @param type * item type * @return true if item exists and false if not */ boolean contains(long identifier, ItemType type); /** * @param identifier * the identifier of changeItem * @param type * item type * @param action * the action of change item * @return true if item exists and false if not */ boolean contains(long identifier, ItemType type, ChangeAction action); /** * Ideally there should only be one ChangeItem for the combination of identifier and ItemType. * the method will ONLY return ONE item that matches. If the implementation allows multiple * ChangeAction in one change set, please use {@link #get(long, ItemType, ChangeAction)} * * @param identifier * the identifier of change item * @param type * the type of item * @return the {@link ChangeItem} that matches the input. * @see #get(long, ItemType, ChangeAction) */ Optional get(long identifier, ItemType type); /** * @param identifier * the identifier of change item * @param type * the type of item * @param action * the action of change item * @return the {@link ChangeItem} that matches the input */ Optional get(long identifier, ItemType type, ChangeAction action); /** * human readable description of the change set * * @return description of change set */ String getDescription(); /** * @return the source name from which the change set is generated. In case of multiple sources * names are joined by comma(,) */ Iterable getSourceNames(); /** * Change set version should be same as atlas version against which it's generated. * * @return version of change set, same as atlas version. */ String getVersion(); /** * @param action * action to filter * @return a iterator for all {@link ChangeItem}s that matches the action */ Iterator iterator(ChangeAction action); /** * @param type * type to filer * @return a iterator for all {@link ChangeItem}s that matches the type */ Iterator iterator(ItemType type); /** * @param type * type to filter * @param action * action to filter * @return a iterator for all {@link ChangeItem}s that matches both the type and the action */ Iterator iterator(ItemType type, ChangeAction action); /** * @param description * description of the change set */ void setDescription(String description); /** * set version of the ChangeSet, the version should be same as atlas version against which it's * generated. * * @param version * version of the change set */ void setVersion(String version); /** * @param action * {@link ChangeAction} the return sub set should have. * @return a set of all items with given action */ Set subSet(ChangeAction action); /** * @param type * the type of item * @return a set of all items with given type */ Set subSet(ItemType type); /** * get all items with given type and action. * * @param type * the type of item * @param action * item action * @return a set of items with given type and action */ Set subSet(ItemType type, ChangeAction action); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeSetAtlasBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * ChangeSetAtlasBuilder takes an original atlas and accept a ChangeSet to build a new atlas with * all changes applied. Changes follows CUD(Create, Update, Delete) pattern with partial support of * edges and relations. *

* this class is not thread safe *

* * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class ChangeSetAtlasBuilder { private static final Logger logger = LoggerFactory.getLogger(ChangeSetAtlasBuilder.class); private static final int MAXIMUM_RELATION_LOOPS = 500; private final PackedAtlasBuilder builder; private final Atlas originalAtlas; private final ChangeSet changeSet; private Atlas conflatedAtlas; public ChangeSetAtlasBuilder(final Atlas atlas, final ChangeSet changeSet) { this.originalAtlas = atlas; this.changeSet = changeSet; this.builder = new PackedAtlasBuilder().withMetaData(new AtlasMetaData()) .withSizeEstimates(atlas.size()); } /** * build and return atlas object * * @return atlas been built */ public Atlas get() { if (this.conflatedAtlas != null) { return this.conflatedAtlas; } handleSimple(this.originalAtlas.nodes()); handleSimple(this.originalAtlas.points()); handleSimple(this.originalAtlas.lines()); handleSimple(this.originalAtlas.edges()); handleSimple(this.originalAtlas.areas()); handleRelations(); this.conflatedAtlas = this.builder.get(); return this.conflatedAtlas; } private void addItem(final long identifier, final ItemType type, final Iterable geometry, final Map tags) { switch (type) { case POINT: this.builder.addPoint(identifier, (Location) geometry, tags); break; case NODE: this.builder.addNode(identifier, (Location) geometry, tags); break; case LINE: this.builder.addLine(identifier, (PolyLine) geometry, tags); break; case EDGE: this.builder.addEdge(identifier, (PolyLine) geometry, tags); break; case AREA: this.builder.addArea(identifier, (Polygon) geometry, tags); break; case RELATION: default: throw new IllegalArgumentException("addItem can't take relation type"); } } private void handleRelations() { this.originalAtlas.relations().forEach(relation -> { if (this.changeSet.contains(relation.getIdentifier(), ItemType.RELATION, ChangeAction.DELETE)) { return; } final Optional optionalItem = this.changeSet.get(relation.getIdentifier(), ItemType.RELATION, ChangeAction.UPDATE); if (optionalItem.isPresent()) { final ChangeItem changeItem = optionalItem.get(); this.builder.addRelation(relation.getIdentifier(), relation.getOsmIdentifier(), changeItem.getRelationBean().get(), changeItem.getTags()); } else { final RelationBean bean = new RelationBean(); relation.members() .forEach(member -> bean.addItem(member.getEntity().getIdentifier(), member.getRole(), member.getEntity().getType())); this.builder.addRelation(relation.getIdentifier(), relation.osmRelationIdentifier(), bean, relation.getTags()); } }); // Add all the relations that do not have members that are relations. Set stagedRelationIdentifiers = new HashSet<>(); final Iterator iterator = this.changeSet.iterator(ItemType.RELATION, ChangeAction.CREATE); while (iterator.hasNext()) { final ChangeItem relationMatchResult = iterator.next(); if (StreamSupport.stream(relationMatchResult.getMembers().spliterator(), false) .anyMatch(member -> member.getType() == ItemType.RELATION)) { stagedRelationIdentifiers.add(relationMatchResult.getIdentifier()); } else { this.builder.addRelation(relationMatchResult.getIdentifier(), relationMatchResult.getIdentifier(), relationMatchResult.getRelationBean().get(), relationMatchResult.getTags()); } } // Add all the other relations int iterations = 0; while (++iterations < MAXIMUM_RELATION_LOOPS && !stagedRelationIdentifiers.isEmpty()) { logger.trace("Copying relations level {} deep.", iterations); final Set stagedRelationIdentifiersCopy = new HashSet<>(); for (final Long relationIdentifier : stagedRelationIdentifiers) { final Optional optionalItem = this.changeSet.get(relationIdentifier, ItemType.RELATION, ChangeAction.CREATE); if (!optionalItem.isPresent()) { logger.error("can't find relation with id {}", relationIdentifier); continue; } final ChangeItem relationItem = optionalItem.get(); final boolean skip = StreamSupport .stream(relationItem.getMembers().spliterator(), false) .allMatch(member -> member.getType() == ItemType.RELATION && this.builder.peek().relation(member.getIdentifier()) == null); if (!skip) { this.builder.addRelation(relationItem.getIdentifier(), relationItem.getIdentifier(), relationItem.getRelationBean().get(), relationItem.getTags()); } else { stagedRelationIdentifiersCopy.add(relationIdentifier); } } stagedRelationIdentifiers = stagedRelationIdentifiersCopy; } if (iterations >= MAXIMUM_RELATION_LOOPS) { throw new CoreException( "There might be a loop in relations! It took more than {} loops to copy the relation.", MAXIMUM_RELATION_LOOPS); } } private void handleSimple(final Iterable items) { items.forEach(item -> { if (this.changeSet.contains(item.getIdentifier(), item.getType(), ChangeAction.DELETE)) { return; } final Optional optionalItem = this.changeSet.get(item.getIdentifier(), item.getType(), ChangeAction.UPDATE); if (optionalItem.isPresent()) { final ChangeItem changeItem = optionalItem.get(); addItem(changeItem.getIdentifier(), item.getType(), changeItem.getGeometry(), changeItem.getTags()); } else { addItem(item.getIdentifier(), item.getType(), item.getRawGeometry(), item.getTags()); } }); final Iterator iterator = items.iterator(); if (!iterator.hasNext()) { return; } final ItemType type = iterator.next().getType(); this.changeSet.iterator(type, ChangeAction.CREATE).forEachRemaining(changeItem -> { addItem(changeItem.getIdentifier(), type, changeItem.getGeometry(), changeItem.getTags()); }); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeSetDeserializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.Optional; import java.util.function.Supplier; /** * Deerializer interface for {@link ChangeSet}. * * @author yiqing-jin * @author mkalender * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface ChangeSetDeserializer extends Supplier>, AutoCloseable { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/ChangeSetSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.function.Consumer; /** * Serializer interface for {@link ChangeSet}. * * @author yiqing-jin * @author mkalender * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface ChangeSetSerializer extends Consumer, AutoCloseable { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/GeoJSONChangeSetSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.streaming.resource.WritableResource; /** * Serializes {@link ChangeSet} objects and writes them into {@link OutputStream}s in geojson * format. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class GeoJSONChangeSetSerializer implements ChangeSetSerializer { private final WritableResource resource; public GeoJSONChangeSetSerializer(final WritableResource resourceToWriteInto) { this.resource = resourceToWriteInto; } @Override public void accept(final ChangeSet changeSet) { toGeoJson(changeSet).save(this.resource); } @Override public void close() throws Exception { // Do nothing as close is handled in GeoJsonObject.save method } public GeoJsonObject toGeoJson(final ChangeSet changeSet) { final List collection = new ArrayList<>(); changeSet.iterator().forEachRemaining(changeItem -> { if (changeItem.getGeometry() == null) { return; } changeItem.getTags().put("action", changeItem.getAction().name()); changeItem.getTags().put("id", String.valueOf(changeItem.getIdentifier())); collection.add(new GeoJsonBuilder.LocationIterableProperties(changeItem.getGeometry(), changeItem.getTags())); }); final GeoJsonBuilder builder = new GeoJsonBuilder(); return builder.create(collection); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/MutableChangeItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * This interface is designed to be used by data enhancing and data merging programs which may * require frequently changing values on the fly. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public interface MutableChangeItem extends ChangeItem { void addAllMembers(Iterable members) throws CoreException; void addMember(ChangeItemMember member) throws CoreException; boolean removeMember(long identifier, String role, ItemType type) throws CoreException; void setAction(ChangeAction action); void setGeometry(Iterable geometry) throws CoreException; void setIdentifier(long identifier); /** * @param score * the score of the item, actual meaning varies base on source type and conflation * process */ void setScore(double score); void setSourceName(String sourceName); void setTags(Map tags); void setType(ItemType type); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/SimpleChangeItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A implementation of {@link ChangeItem} that should work for most cases. * * @author Yiqing Jin * @author mkalender * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class SimpleChangeItem implements MutableChangeItem { private static final long serialVersionUID = 3817694187693336803L; private String sourceName; private Map tags; private long identifier; private ItemType type; private ChangeAction action; private Iterable geometry; private double score = 1; private final Set members; public SimpleChangeItem() { this.members = new HashSet<>(); } public SimpleChangeItem(final long identifier, final String sourceName, final ItemType type, final ChangeAction action, final Iterable geometry, final Map tags) { this.identifier = identifier; this.sourceName = sourceName; this.type = type; this.action = action; this.geometry = geometry; this.tags = tags; this.members = new HashSet<>(); } @Override public void addAllMembers(final Iterable members) throws CoreException { assertTypeIsRelation(); members.forEach(member -> this.members.add(member)); } @Override public void addMember(final ChangeItemMember member) throws CoreException { assertTypeIsRelation(); this.members.add(member); } @Override public boolean equals(final Object other) { if (!(other instanceof SimpleChangeItem)) { return false; } final SimpleChangeItem that = (SimpleChangeItem) other; return this.getIdentifier() == that.getIdentifier() && this.getType() == that.getType() && this.getAction() == that.getAction() && StringUtils.equals(this.getSourceName(), that.getSourceName()) && this.getScore() == that.getScore() && Iterables.equals(this.getMembers(), that.getMembers()) && Iterables.equals(this.getGeometry(), that.getGeometry()); } @Override public ChangeAction getAction() { return this.action; } @Override public Iterable getGeometry() { return this.geometry; } @Override public long getIdentifier() { return this.identifier; } @Override public Iterable getMembers() { return this.members; } @Override public Optional getRelationBean() { assertTypeIsRelation(); if (this.members.isEmpty()) { return Optional.empty(); } final RelationBean bean = new RelationBean(); this.members.forEach( member -> bean.addItem(member.getIdentifier(), member.getRole(), member.getType())); return Optional.of(bean); } @Override public double getScore() { return this.score; } @Override public String getSourceName() { return this.sourceName; } @Override public Optional getTag(final String key) { return Optional.ofNullable(this.tags.get(key)); } @Override public Map getTags() { return this.tags; } @Override public ItemType getType() { return this.type; } @Override /** * ItemType and ChangeAction combined should have no more than 9 variants and ideally should * have less than 3. so identifier itself should be good enough for hash code */ public int hashCode() { return (int) this.identifier; } @Override public boolean removeMember(final long identifier, final String role, final ItemType type) throws CoreException { assertTypeIsRelation(); return this.members.remove(new SimpleChangeItemMember(identifier, role, type)); } @Override public void setAction(final ChangeAction action) { this.action = action; } @Override public void setGeometry(final Iterable geometry) throws CoreException { assertTypeIsNotRelation(); this.geometry = geometry; } @Override public void setIdentifier(final long identifier) { this.identifier = identifier; } public void setMembers(final Iterable members) { this.members.clear(); this.addAllMembers(members); } @Override public void setScore(final double score) { this.score = score; } @Override public void setSourceName(final String sourceName) { this.sourceName = sourceName; } @Override public void setTags(final Map tags) { this.tags = tags; } @Override public void setType(final ItemType type) { this.type = type; } private void assertTypeIsNotRelation() { if (this.getType() == ItemType.RELATION) { throw new CoreException("Cannot execute this on a ChangeItem with type {}", this.getType()); } } private void assertTypeIsRelation() { if (this.getType() != ItemType.RELATION) { throw new CoreException("Cannot execute this on a ChangeItem with type {}", this.getType()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/SimpleChangeItemMember.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.io.Serializable; import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * A simple implementation of {@link ChangeItemMember} interface. * * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class SimpleChangeItemMember implements ChangeItemMember, Serializable { private static final long serialVersionUID = 3261727439156010800L; private long identifier; private ItemType type; private String role; public SimpleChangeItemMember(final long identifier, final String role, final ItemType type) { this.identifier = identifier; this.type = type; this.role = role; } @Override public boolean equals(final Object other) { if (!(other instanceof SimpleChangeItemMember)) { return false; } final SimpleChangeItemMember that = (SimpleChangeItemMember) other; return this.getIdentifier() == that.getIdentifier() && this.getType() == that.getType() && StringUtils.equals(this.getRole(), that.getRole()); } @Override public long getIdentifier() { return this.identifier; } @Override public String getRole() { return this.role; } @Override public ItemType getType() { return this.type; } @Override public int hashCode() { return Objects.hash(this.getIdentifier(), this.getRole(), this.getType()); } public void setIdentifier(final long identifier) { this.identifier = identifier; } public void setRole(final String role) { this.role = role; } public void setType(final ItemType type) { this.type = type; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[Identifier: "); builder.append(this.getIdentifier()); builder.append(", Role: "); builder.append(this.getRole()); builder.append(", Type: "); builder.append(this.getType()); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/changeset/SimpleChangeSet.java ================================================ package org.openstreetmap.atlas.geography.atlas.changeset; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import com.google.common.collect.AbstractIterator; /** * A simple implementation of {@link ChangeSet} interface. This should be good for most common use * cases but if necessary user can choose to implement their own or extend from this one. *

* This implementation is not thread safe *

* * @author Yiqing Jin * @deprecated - see new API under org.openstreetmap.atlas.geography.atlas.change package. */ @Deprecated public class SimpleChangeSet implements ChangeSet { private static final long serialVersionUID = -6499530503182134327L; // internal data structure containing all data objects. key should be combination of identifier, // ItemType and ChangeAction. private final Map map = new HashMap<>(); private String version; private String description; private static String computeKey(final long identifier, final ItemType type, final ChangeAction action) { return identifier + type.toShortString() + action; } public SimpleChangeSet() { this.version = "unknown"; this.description = ""; } @Override public boolean add(final ChangeItem changeItem) { return this.map.put(computeKey(changeItem), changeItem) != null; } @Override public boolean addAll(final Collection changeItems) { boolean state = false; for (final ChangeItem changeItem : changeItems) { state = add(changeItem) || state; } return state; } @Override public void clear() { this.map.clear(); } @Override public boolean contains(final long identifier, final ItemType type) { for (final ChangeAction action : ChangeAction.values()) { if (this.map.containsKey(computeKey(identifier, type, action))) { return true; } } return false; } @Override public boolean contains(final long identifier, final ItemType type, final ChangeAction action) { return this.map.containsKey(computeKey(identifier, type, action)); } @Override public boolean contains(final Object object) { if (!(object instanceof ChangeItem)) { return false; } return this.map.containsKey(computeKey((ChangeItem) object)); } @Override public boolean containsAll(final Collection collection) { return collection.stream().allMatch(this::contains); } @Override public boolean equals(final Object other) { if (!(other instanceof SimpleChangeSet)) { return false; } final SimpleChangeSet that = (SimpleChangeSet) other; return StringUtils.equals(this.getVersion(), that.getVersion()) && StringUtils.equals(this.getDescription(), that.getDescription()) && this.map.equals(that.map); } @Override public Optional get(final long identifier, final ItemType type) { for (final ChangeAction action : ChangeAction.values()) { final ChangeItem item = this.map.get(computeKey(identifier, type, action)); if (item != null) { return Optional.of(item); } } return Optional.empty(); } @Override public Optional get(final long identifier, final ItemType type, final ChangeAction action) { return Optional.ofNullable(this.map.get(computeKey(identifier, type, action))); } @Override public String getDescription() { return this.description; } @Override public Iterable getSourceNames() { final List sourceNames = new ArrayList<>(); this.iterator().forEachRemaining(changeItem -> sourceNames.add(changeItem.getSourceName())); return sourceNames; } @Override public String getVersion() { return this.version; } @Override public int hashCode() { return Objects.hash(this.getVersion(), this.getDescription(), this.map); } @Override public boolean isEmpty() { return this.map.isEmpty(); } @Override public Iterator iterator() { return this.map.values().iterator(); } @Override public Iterator iterator(final ChangeAction action) { return filterIterator(iterator(), item -> item.getAction() == action); } @Override public Iterator iterator(final ItemType type) { return filterIterator(iterator(), item -> item.getType() == type); } @Override public Iterator iterator(final ItemType type, final ChangeAction action) { return filterIterator(iterator(), item -> item.getType() == type && item.getAction() == action); } @Override public boolean remove(final Object obj) { return this.map.remove(computeKey((ChangeItem) obj)) != null; } @Override public boolean removeAll(final Collection collection) { throw new UnsupportedOperationException(); } @Override public boolean retainAll(final Collection collection) { throw new UnsupportedOperationException(); } @Override public void setDescription(final String description) { this.description = description; } @Override public void setVersion(final String version) { this.version = version; } @Override public int size() { return this.map.size(); } @Override public Set subSet(final ChangeAction action) { return this.stream().filter(item -> item.getAction() == action).collect(Collectors.toSet()); } @Override public Set subSet(final ItemType type) { return this.stream().filter(item -> item.getType() == type).collect(Collectors.toSet()); } @Override public Set subSet(final ItemType type, final ChangeAction action) { return this.stream().filter(item -> item.getType() == type && item.getAction() == action) .collect(Collectors.toSet()); } @Override public Object[] toArray() { return this.map.values().toArray(); } @Override public T[] toArray(final T[] array) { return this.map.values().toArray(array); } private String computeKey(final ChangeItem item) { return computeKey(item.getIdentifier(), item.getType(), item.getAction()); } private Iterator filterIterator(final Iterator parent, final Predicate predicate) { return new AbstractIterator() { @Override protected ChangeItem computeNext() { while (parent.hasNext()) { final ChangeItem item = parent.next(); if (item != null && predicate.test(item)) { return item; } } return endOfData(); } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AbstractAtlasOutputTestSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap.KeySetView; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; /** * Shared code for generating a single text-based test atlas for any building that passes the filter * as defined by concrete subclasses. * * @author cstaylor */ abstract class AbstractAtlasOutputTestSubCommand extends AbstractAtlasSubCommand { private static final Switch OUTPUT_TO_PACKED_ATLAS_PARAMETER = new Switch<>( "packed-output", "Outputs the found atlas pieces into a packed atlas for testing", Paths::get, Optionality.OPTIONAL); private static final Switch OUTPUT_TO_TEXT_PARAMETER = new Switch<>("text-output", "Outputs the found atlas pieces into text for testing", Paths::get, Optionality.OPTIONAL); private static final Switch DISTANCE_IN_METERS_PARAMETER = new Switch<>("expand", "Expands the search bounds by this optional parameter in meters", Double::new, Optionality.OPTIONAL); private KeySetView subAtlases; private Optional distanceInMeters; private Path outputTextPath; private Path packedAtlasPath; protected AbstractAtlasOutputTestSubCommand(final String name, final String description) { super(name, description); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_TO_PACKED_ATLAS_PARAMETER, OUTPUT_TO_TEXT_PARAMETER, DISTANCE_IN_METERS_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-packed-output=/output/to/packed/atlas : the file we should send the packed Atlas to if it exists\n"); writer.printf( "-text-output=/output/to/text/file : the file we should send the text data of the Atlas if it exists\n"); writer.printf( "-expand=[distance in meters] : how far we should expand around the building we've found for the subatlas we're saving\n"); } protected abstract boolean filter(AtlasEntity entity); @Override protected int finish(final CommandMap command) { if (this.subAtlases.isEmpty()) { LoggerFactory.getLogger(getClass()).info("No items found"); return -1; } if (this.packedAtlasPath != null) { final Atlas atlas = new MultiAtlas(this.subAtlases); try { final PackedAtlas saveMe = new PackedAtlasCloner().cloneFrom(atlas); Files.createDirectories(this.packedAtlasPath.getParent()); saveMe.save(new File(this.packedAtlasPath.toFile())); LoggerFactory.getLogger(getClass()).info("Packed atlas saved to {}", this.packedAtlasPath); } catch (final IOException oops) { throw new CoreException("Error when saving packed atlas", oops); } } if (this.outputTextPath != null) { new MultiAtlas(ImmutableList.copyOf(this.subAtlases.iterator())) .saveAsText(new File(this.outputTextPath.toFile())); LoggerFactory.getLogger(getClass()).info("Text atlas saved to {}", this.outputTextPath); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { StreamSupport.stream(atlas.entities(i -> i.getOsmIdentifier() > 0).spliterator(), false) .filter(this::filter).forEach(this::output); } @SuppressWarnings("unchecked") @Override protected void start(final CommandMap command) { this.subAtlases = ConcurrentHashMap.newKeySet(); this.distanceInMeters = (Optional) command.getOption(DISTANCE_IN_METERS_PARAMETER); ((Optional) command.getOption(OUTPUT_TO_TEXT_PARAMETER)) .ifPresent(path -> this.outputTextPath = path); ((Optional) command.getOption(OUTPUT_TO_PACKED_ATLAS_PARAMETER)) .ifPresent(path -> this.packedAtlasPath = path); if (this.outputTextPath == null && this.packedAtlasPath == null) { throw new CoreException("Either -packed-output or -text-output must have a value"); } } private void output(final AtlasEntity item) { Rectangle rectangle = item.bounds(); if (this.distanceInMeters.isPresent()) { rectangle = rectangle.expand(Distance.meters(this.distanceInMeters.get())); } item.getAtlas().subAtlas(rectangle, AtlasCutType.SOFT_CUT).ifPresent(this.subAtlases::add); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AbstractAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Flag; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * Helper class that makes it easier to implement ReaderCommands that need loaded atlases * * @author cstaylor */ public abstract class AbstractAtlasSubCommand implements FlexibleSubCommand { private static final Switch INPUT_FOLDER_PARAMETER = new Switch<>("input", "Input atlas file or folder containing atlas files to load", File::new, Optionality.REQUIRED); private static final Flag COMBINE_PARAMETER = new Flag("combine", "Will combine all atlases found into a MultiAtlas before reading the metadata"); private static final Flag PARALLEL_FLAG = new Flag("parallel", "Will process multiple atlases in parallel"); private final String name; private final String description; protected AbstractAtlasSubCommand(final String name, final String description) { this.name = name; this.description = description; } @Override public int execute(final CommandMap command) { start(command); final File path = (File) command.get(INPUT_FOLDER_PARAMETER); Stream atlases = path.listFilesRecursively().stream() .filter(AtlasResourceLoader.HAS_ATLAS_EXTENSION) .map(atlas -> new AtlasResourceLoader().load(atlas)); if ((Boolean) command.get(PARALLEL_FLAG)) { atlases = atlases.parallel(); } if ((Boolean) command.get(COMBINE_PARAMETER)) { handle(new MultiAtlas(atlases.collect(Collectors.toList())), command); } else { atlases.forEach(atlas -> { handle(atlas, command); }); } return finish(command); } @Override public String getDescription() { return this.description; } @Override public final String getName() { return this.name; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_FOLDER_PARAMETER, COMBINE_PARAMETER, PARALLEL_FLAG); } /** * After all atlas files have been handled, the subclass can override this method for a final * notification and processing. The return value is sent back to the caller through System.exit * * @param command * arguments to this subcommand that may affect processing * @return a status value retured through System.exit */ protected int finish(final CommandMap command) { return 0; } /** * Subclasses will implement this method for processing each atlas object as it is loaded. * * @param atlas * the atlas to process * @param command * arguments to this subcommand that may affect processing */ protected abstract void handle(Atlas atlas, CommandMap command); /** * Subclasses can override this method if they want to do something once before processing all * of the atlases * * @param command * arguments to this subcommand that may affect processing */ protected void start(final CommandMap command) { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasCommandConstants.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; /** * Putting all of those strings in one place * * @author cstaylor */ public interface AtlasCommandConstants { String INPUT_PARAMETER_DESCRIPTION = "-input=/path/to/atlas/files : the atlas files we want to load\n"; String INPUT_ZOOM_LEVEL = "-zoom_level=\n"; String OUTPUT_FOLDER_DESCRIPTION = "-output=/path/to/atlas/output/to/save : the path to output atlas files\n"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasCountriesSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.Set; import java.util.TreeSet; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Lists all of the countries found in the metadata within the input atlas files * * @author cstaylor */ public class AtlasCountriesSubCommand extends AbstractAtlasSubCommand { private final Set countries; public AtlasCountriesSubCommand() { super("countries", "lists all of the countries found in alphabetical order"); this.countries = new TreeSet<>(); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected int finish(final CommandMap command) { System.out.printf("Countries: %d\n", this.countries.size()); this.countries.forEach(System.out::println); return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { atlas.metaData().getCountry().ifPresent(this.countries::add); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasFeatureCountsSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.BufferedOutputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNodeFinder; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuildingFinder; import org.openstreetmap.atlas.geography.atlas.items.complex.islands.ComplexIslandFinder; import org.openstreetmap.atlas.geography.atlas.items.complex.restriction.ComplexTurnRestrictionFinder; import org.openstreetmap.atlas.geography.atlas.items.complex.water.finder.ComplexWaterEntityFinder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import com.google.common.collect.Table; import com.google.common.collect.TreeBasedTable; /** * Prints out the number of features for the given atlas files * * @author ahsieh */ public class AtlasFeatureCountsSubCommand extends AbstractAtlasSubCommand { /** * The types of Atlas entities to be processed (edges, lines, areas, etc.) */ protected enum AtlasType { NODE, LINE, AREA, POINT, EDGE, RELATION, COMPLEX_BUILDING, COMPLEX_WATER, COMPLEX_ISLAND, COMPLEX_TURN_RESTRICTION, COMPLEX_BIG_NODE } private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "The output file to save the statistics", value -> new File(value), Optionality.REQUIRED); private final Table featureCounts; public AtlasFeatureCountsSubCommand() { super("featureCounts", "lists counts of objects found"); this.featureCounts = TreeBasedTable.create(); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected int finish(final CommandMap command) { final File file = (File) command.get(OUTPUT_PARAMETER); try (PrintStream out = new PrintStream( new BufferedOutputStream(new FileOutputStream(file.getFile(), true)))) { for (final String country : this.featureCounts.rowKeySet()) { for (final AtlasType type : AtlasType.values()) { out.println(String.format("%s-%s: %d", country, type, this.featureCounts.contains(country, type) ? this.featureCounts.get(country, type) : 0)); } out.println(); } } catch (final IOException oops) { throw new CoreException("Error writing to file: {}", file.getAbsolutePathString().toString(), oops); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { for (final AtlasType type : AtlasType.values()) { updateHashMapForAtlasType(type, atlas); } } private void updateHashMapForAtlasType(final AtlasType type, final Atlas atlas) { long oldCount = 0; final long additionalCount; final String country = atlas.metaData().getCountry().orElseGet(() -> "UNKNOWN"); switch (type) { case NODE: additionalCount = atlas.numberOfNodes(); break; case LINE: additionalCount = atlas.numberOfLines(); break; case AREA: additionalCount = atlas.numberOfAreas(); break; case POINT: additionalCount = atlas.numberOfPoints(); break; case EDGE: additionalCount = atlas.numberOfEdges(); break; case RELATION: additionalCount = atlas.numberOfRelations(); break; case COMPLEX_BUILDING: additionalCount = Iterables.count(new ComplexBuildingFinder().find(atlas), i -> 1L); break; case COMPLEX_WATER: additionalCount = Iterables.count(new ComplexWaterEntityFinder().find(atlas), i -> 1L); break; case COMPLEX_ISLAND: additionalCount = Iterables.count(new ComplexIslandFinder().find(atlas), i -> 1L); break; case COMPLEX_TURN_RESTRICTION: additionalCount = Iterables.count(new ComplexTurnRestrictionFinder().find(atlas), i -> 1L); break; case COMPLEX_BIG_NODE: additionalCount = Iterables.count(new BigNodeFinder().find(atlas), i -> 1L); break; default: throw new CoreException("Unexpected AtlasType: " + type.toString()); } // check if there's already a value in the Table synchronized (this) { if (this.featureCounts.contains(country, type)) { oldCount = this.featureCounts.get(country, type); } this.featureCounts.put(country, type, oldCount + additionalCount); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasFindByAtlasIdentifierSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasObject; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Searches a collection of Atlases for a set of ids. The Atlases containing each id are reported. * Optionally, all Atlases that contain one of the ids are joined and output to a single Atlas file. * * @author bbreithaupt */ public class AtlasFindByAtlasIdentifierSubCommand extends AbstractAtlasSubCommand { private static final Command.Switch> ATLAS_ID_PARAMETER = new Command.Switch<>("id", "List of comma-delimited Atlas identifiers", possibleMultipleOSMIdentifier -> Stream.of(possibleMultipleOSMIdentifier.split(",")) .map(Long::parseLong).collect(Collectors.toSet()), Command.Optionality.REQUIRED); private static final Command.Switch JOINED_OUTPUT_PARAMETER = new Command.Switch<>( "joinedOutput", "The Atlas file to save the joined output to (optional). If not passed the found shards will not be joined and only appear in the console.", String::toString, Command.Optionality.OPTIONAL); private final Set identifiers = new HashSet<>(); private final Set shardNames = new HashSet<>(); public AtlasFindByAtlasIdentifierSubCommand() { super("find-atlas-id", "Find which atlas files contain particular Atlas features using a given set of Atlas identifiers"); } @Override public Command.SwitchList switches() { return super.switches().with(ATLAS_ID_PARAMETER, JOINED_OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.print(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.print("-id=1000000,2000000 : comma separated Atlas identifiers to search for%n"); writer.print("-joinedOutput=path/to/joined.atlas : the path to the output Atlas file%n"); } @Override protected int finish(final CommandMap command) { final Optional output = command.getOption(JOINED_OUTPUT_PARAMETER); // If joining is requested and there are shards to join... if (output.isPresent() && !this.shardNames.isEmpty()) { System.out.printf("Stitching shards and saving to output %s%n", output.get()); // Use AtlasJoinerSubCommand to join found atlases AtlasReader.main("join", String.format("-input=%s", command.get(super.switches().get(0))), String.format("-output=%s", output.get()), String.format("-atlases=%s", String.join(",", this.shardNames))); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { // Get all atlas entities with ids matching the input list atlas.entities(identifierCheck()).forEach(item -> { // Print atlas and item information System.out.print(formatAtlasObject(item)); // Record shard name this.shardNames.add(atlas.getName()); }); } @Override protected void start(final CommandMap command) { // Collect ids this.identifiers.addAll((Set) command.get(ATLAS_ID_PARAMETER)); } /** * Creates an informative {@link String} for an {@link AtlasEntity} and the {@link Atlas} that * contains it. * * @param entity * {@link AtlasEntity} to create the string for * @return formatted string */ private String formatAtlasObject(final AtlasEntity entity) { final String shardName = entity.getAtlas().metaData().getShardName().orElse("UNKNOWN"); return String.format("[%s] [%d] [%d] --> [%s:%s] Tags: [%s]%n", entity.getType(), entity.getOsmIdentifier(), entity.getIdentifier(), shardName, entity.getAtlas().getName(), entity.getTags()); } /** * Predicate to check an {@link AtlasObject} against the list of ids. * * @param * Object type * @return {@link Predicate} */ private Predicate identifierCheck() { return object -> this.identifiers.contains(object.getIdentifier()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasFindByFeatureIdentifierLocatorSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Parses an extended identifier including the way-sectioned id and feature type and searches for * the matching atlas objects in a collection of atlases. * * @author cstaylor */ public class AtlasFindByFeatureIdentifierLocatorSubCommand extends AbstractAtlasSubCommand { private static final Switch> FEATURE_ID_PARAMETER = new Switch<>("id", "list of comma-delimited Atlas extended feature identifier", possibleMultipleOSMIdentifier -> Stream.of(possibleMultipleOSMIdentifier.split(",")) .collect(Collectors.toSet()), Optionality.REQUIRED); private static final int TYPE_INDEX = 1; private static final int ID_INDEX = 2; private static final int EXPECTED_IDENTIFIER_LENGTH = 4; private final Predicate nodeCheck = item -> this.nodeIds .contains(item.getOsmIdentifier()); private final Predicate pointCheck = item -> this.pointIds .contains(item.getOsmIdentifier()); private final Predicate areaCheck = item -> this.areaIds .contains(item.getOsmIdentifier()); private final Predicate edgeCheck = item -> item.getOsmIdentifier() > 0L && this.edgeIds.contains(item.getOsmIdentifier()); private final Predicate lineCheck = item -> this.lineIds .contains(item.getOsmIdentifier()); private final Predicate relationCheck = item -> this.relationIds .contains(item.getOsmIdentifier()); private final Set nodeIds = new HashSet<>(); private final Set pointIds = new HashSet<>(); private final Set areaIds = new HashSet<>(); private final Set edgeIds = new HashSet<>(); private final Set lineIds = new HashSet<>(); private final Set relationIds = new HashSet<>(); public AtlasFindByFeatureIdentifierLocatorSubCommand() { super("find", "find which atlas files contain particular OSM features using a given set of extended identifiers"); } @Override public SwitchList switches() { return super.switches().with(FEATURE_ID_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf("-id=comma-separated atlas extended feature identifiers to search for\n"); } protected String formatAtlasObject(final String type, final Atlas atlas, final AtlasEntity entity) { final String shardName = atlas.metaData().getShardName().orElse("UNKNOWN"); return String.format("[%10s] [%d] [%d] --> [%s:%s] Tags: [%s]\n", type, entity.getOsmIdentifier(), entity.getIdentifier(), shardName, atlas.getName(), entity.getTags()); } @Override protected void handle(final Atlas atlas, final CommandMap command) { final int[] found = { 0 }; atlas.nodes(this.nodeCheck).forEach(item -> { System.out.printf(formatAtlasObject("NODE", atlas, item)); found[0]++; }); atlas.points(this.pointCheck).forEach(item -> { System.out.printf(formatAtlasObject("POINT", atlas, item)); found[0]++; }); atlas.edges(this.edgeCheck).forEach(item -> { System.out.printf(formatAtlasObject("EDGE", atlas, item)); found[0]++; }); atlas.lines(this.lineCheck).forEach(item -> { System.out.printf(formatAtlasObject("LINE", atlas, item)); found[0]++; }); atlas.areas(this.areaCheck).forEach(item -> { System.out.printf(formatAtlasObject("AREA", atlas, item)); found[0]++; }); atlas.relations(this.relationCheck).forEach(item -> { System.out.printf(formatAtlasObject("RELATION", atlas, item)); found[0]++; }); } @Override protected void start(final CommandMap command) { @SuppressWarnings("unchecked") final Set unparsedFeatureIds = (Set) command.get(FEATURE_ID_PARAMETER); for (final String unparsedFeatureId : unparsedFeatureIds) { final String[] parsedFeatureIds = unparsedFeatureId.split("_"); // By convention the third piece is the OSM ID, and the second piece is the type of item // we're searching for. if (parsedFeatureIds.length != EXPECTED_IDENTIFIER_LENGTH) { throw new CoreException( "There should be four pieces in an extended feature identifier: {}", unparsedFeatureId); } final long osmId = Long.parseLong(parsedFeatureIds[ID_INDEX]); if (parsedFeatureIds[TYPE_INDEX].equals("N")) { this.nodeIds.add(osmId); this.pointIds.add(osmId); } else if (parsedFeatureIds[TYPE_INDEX].equals("W")) { this.areaIds.add(osmId); this.lineIds.add(osmId); this.edgeIds.add(osmId); } else if (parsedFeatureIds[TYPE_INDEX].equals("R")) { this.relationIds.add(osmId); } else { throw new CoreException("Unknown type: {}", parsedFeatureIds[TYPE_INDEX]); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasFindEntitiesByIdSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Creates a new packed atlas for testing purposes on any atlas items if an item matches a set of * expected osm identifiers. * * @author cstaylor */ public class AtlasFindEntitiesByIdSubCommand extends AbstractAtlasOutputTestSubCommand { private static final Switch> OSM_ID_PARAMETER = new Switch<>("osmid", "list of comma-delimited OSM Identifiers of the entities we want to export", possibleMultipleOSMIdentifier -> Stream.of(possibleMultipleOSMIdentifier.split(",")) .map(Long::parseLong).collect(Collectors.toSet()), Optionality.REQUIRED); private Set identifiers; public AtlasFindEntitiesByIdSubCommand() { super("atlas-with-this-entity", "Creates a new atlas containing the area around any items found that match the set of ids provided by the -osmid parameter"); } @Override public SwitchList switches() { return super.switches().with(OSM_ID_PARAMETER); } @Override public void usage(final PrintStream writer) { super.usage(writer); writer.println( "-osmid=OSM identifier : comma-separated numeric osm identifiers of the items we're trying to locate"); } @Override protected boolean filter(final AtlasEntity item) { return this.identifiers.contains(item.getOsmIdentifier()); } @SuppressWarnings("unchecked") @Override protected void start(final CommandMap command) { super.start(command); this.identifiers = (Set) command.get(OSM_ID_PARAMETER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasGeoJSONSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.OutputStreamWritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Outputs GeoJSON data to stdout or to an optional file * * @author cstaylor */ public class AtlasGeoJSONSubCommand extends AbstractAtlasSubCommand { private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "The geojson file to save the Atlas as geojson to: otherwise sent to stdout", Paths::get, Optionality.OPTIONAL); private JsonWriter writer; public AtlasGeoJSONSubCommand() { super("geojson", "outputs the atlas as geojson data"); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-output=/path/to/output/geojson/file: the path to the output geojson file\n"); writer.printf( "-combine : merge all of the atlas files into a MultiAtlas before outputting geojson\n"); } @Override protected int finish(final CommandMap command) { this.writer.close(); return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { getWriter(command).write(atlas.asGeoJson()); } @SuppressWarnings("unchecked") private JsonWriter getWriter(final CommandMap map) { if (this.writer == null) { ((Optional) map.getOption(OUTPUT_PARAMETER)).ifPresent(path -> { try { Files.createDirectories(path.getParent()); this.writer = new JsonWriter(new File(path.toString())); } catch (final IOException oops) { throw new CoreException("Error when creating output stream", oops); } }); if (this.writer == null) { this.writer = new JsonWriter(new OutputStreamWritableResource(System.out)); } } return this.writer; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasItemsWithSharedShapepointsSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.BufferedOutputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.Collection; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; /** * Checks atlas items in a given atlas for geometry with consecutive identical shapepoints. While * this is acceptable in OSM data, it could cause problems in downstream processing tools, so this * tool can give us an early warning. *

* Two required parameters: *

    *
  • -input: where we should read the atlas files
    • *
  • -output: where we should write information about consecutive identical shapepoints. These * files are organized by country
    • *
* * @author cstaylor */ public class AtlasItemsWithSharedShapepointsSubCommand extends AbstractAtlasSubCommand { /** * Helper class for capturing the geometry of an item as a PolyLine and deciding if it has any * consecutive identical shapepoints * * @author cstaylor */ private static final class PolyLineTrouble { private final long osmId; private final PolyLine polyline; PolyLineTrouble(final AtlasItem item) { this.osmId = item.getOsmIdentifier(); this.polyline = new PolyLine(item.getRawGeometry()); } Long getOsmId() { return this.osmId; } /** * Simple algorithm for checking if the polyline has consecutive identical shapepoints by * comparing the current point with the previous one * * @return true if there is a least one overlapping point, false otherwise */ boolean hasDuplicatePoints() { Location previous = null; for (final Location location : this.polyline) { if (location.equals(previous)) { return true; } previous = location; } return false; } } private static final Logger logger = LoggerFactory .getLogger(AtlasItemsWithSharedShapepointsSubCommand.class); private static final Switch OUTPUT_FILE_PARAMETER = new Switch<>("output", "Where we want to store the duplicate point error logs", Paths::get, Optionality.REQUIRED); private Multimap countryToOSMids; private Time start; private int atlasFiles; public AtlasItemsWithSharedShapepointsSubCommand() { super("duplicate-points", "Outputs all of the OSM ids for items with consecutive identical shapepoints"); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_FILE_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf("\t-output=/path/to/output/folder/for/each/country%n"); } @Override protected int finish(final CommandMap command) { for (final Entry> badIdsForCountry : this.countryToOSMids.asMap() .entrySet()) { try (PrintStream stream = outputFor(badIdsForCountry.getKey(), command)) { stream.println(badIdsForCountry.getValue().stream().sorted().map(String::valueOf) .collect(Collectors.joining("\n"))); } } final long totalErrors = this.countryToOSMids.asMap().values().parallelStream() .flatMap(Collection::parallelStream).count(); final NumberFormat formatter = DecimalFormat.getIntegerInstance(); if (logger.isInfoEnabled()) { logger.info("Completed: {} atlas files and found {} errors from {} countries took {}", formatter.format(this.atlasFiles), formatter.format(totalErrors), formatter.format(this.countryToOSMids.keySet().size()), this.start.elapsedSince()); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { logger.info("Starting {}", atlas.getName()); this.atlasFiles++; final Set badOsmIDS = ConcurrentHashMap.newKeySet(); StreamSupport.stream(atlas.items().spliterator(), true).map(PolyLineTrouble::new) .filter(PolyLineTrouble::hasDuplicatePoints).map(PolyLineTrouble::getOsmId) .forEach(badOsmIDS::add); final Optional countryNameOption = atlas.metaData().getCountry(); if (countryNameOption.isPresent()) { final String countryName = countryNameOption.get(); badOsmIDS.forEach(id -> { this.countryToOSMids.put(countryName, id); }); if (!badOsmIDS.isEmpty()) { logger.warn("Found {} overlaps in {}", badOsmIDS.size(), atlas.getName()); } } } protected PrintStream outputFor(final String isoCountry, final CommandMap command) { final Path outputDirectory = (Path) command.get(OUTPUT_FILE_PARAMETER); try { Files.createDirectories(outputDirectory); final Path outputFile = outputDirectory .resolve(String.format("%s.duplicatepoints", isoCountry)); return new PrintStream( new BufferedOutputStream(new FileOutputStream(outputFile.toFile()))); } catch (final IOException oops) { throw new CoreException("Failure when creating outputstream for {}", isoCountry, oops); } } @Override protected void start(final CommandMap command) { super.start(command); this.countryToOSMids = ArrayListMultimap.create(); this.start = Time.now(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasJoinerSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Create a MultiAtlas from the set of input Atlas files, creates a PackedAtlas from the MultiAtlas, * and then writes that PackedAtlas to the specified output file. Input files are defined through * the -input parameter as directory of Atlas shards. All shards are joined by default. Only select * shards can be joined by using the -atlases parameter. * * @author cstaylor * @author bbreithaupt */ public class AtlasJoinerSubCommand extends AbstractAtlasSubCommand { private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "The Atlas file to save to", Paths::get, Optionality.REQUIRED); private static final Switch> ATLAS_NAMES_PARAMETER = new Switch<>("atlases", "A comma separated List of Atlas files to join", atlasNames -> Stream.of(atlasNames.split(",")).collect(Collectors.toSet()), Optionality.OPTIONAL); private final List atlases = new ArrayList<>(); public AtlasJoinerSubCommand() { super("join", "joins multiple atlas files into a single packed atlas"); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_PARAMETER, ATLAS_NAMES_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-output=/path/to/atlas/output/to/save : the path to the output atlas file\n"); writer.printf( "-atlases=example.atlas,example2.atlas : comma separated list of atlas file names\n"); } @Override protected int finish(final CommandMap command) { final Atlas atlas = new MultiAtlas(this.atlases); try { final PackedAtlas saveMe = new PackedAtlasCloner().cloneFrom(atlas); final Path path = (Path) command.get(OUTPUT_PARAMETER); Files.createDirectories(path.getParent()); saveMe.save(new File(path.toString())); return 0; } catch (final IOException oops) { throw new CoreException("Error when saving packed atlas", oops); } } @Override protected void handle(final Atlas atlas, final CommandMap command) { final Optional atlasNames = command.getOption(ATLAS_NAMES_PARAMETER); if (atlasNames.isPresent() && ((Set) atlasNames.get()).contains(atlas.getName())) { this.atlases.add(atlas); } else if (!atlasNames.isPresent()) { this.atlases.add(atlas); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasListRestrictedPathsCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.util.Optional; import java.util.TreeSet; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNodeFinder; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.RestrictedPath; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * List all valid turnRestriction Id's in the output file. * * @author ahsieh */ public class AtlasListRestrictedPathsCommand extends AbstractAtlasSubCommand { private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "The output file to list all turn restriction ids", value -> new File(value), Optionality.OPTIONAL); private final TreeSet restrictedPaths; public AtlasListRestrictedPathsCommand() { super("restrictedPaths", "lists restrictedPaths"); this.restrictedPaths = new TreeSet<>((restriction1, restriction2) -> Integer .compare(restriction1.hashCode(), restriction2.hashCode())); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected int finish(final CommandMap command) { @SuppressWarnings("unchecked") final Optional possibleFile = (Optional) command.getOption(OUTPUT_PARAMETER); if (possibleFile.isPresent()) { try (PrintStream out = new PrintStream( new FileOutputStream(possibleFile.get().getFile(), true))) { this.restrictedPaths.forEach(value -> out.println(value)); } catch (final IOException oops) { throw new CoreException("Error writing restrictedPaths to file", oops); } } else { try (PrintStream out = new PrintStream(System.out)) { this.restrictedPaths.forEach(value -> out.println(value)); } } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { StreamSupport.stream(new BigNodeFinder().find(atlas).spliterator(), false) .flatMap(bigNode -> { return bigNode.turnRestrictions().stream(); }).forEach(this.restrictedPaths::add); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasListValidTurnRestrictionIds.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.util.TreeSet; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.restriction.ComplexTurnRestrictionFinder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * List all valid turnRestriction Id's in the output file. * * @author ahsieh */ public class AtlasListValidTurnRestrictionIds extends AbstractAtlasSubCommand { private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "The output file to list all turn restriction ids", value -> new File(value), Optionality.REQUIRED); private final TreeSet turnRestrictions; public AtlasListValidTurnRestrictionIds() { super("turnRestrictions", "lists turn restriction OsmIds"); this.turnRestrictions = new TreeSet<>(); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected int finish(final CommandMap command) { final File file = (File) command.get(OUTPUT_PARAMETER); try (PrintStream out = new PrintStream(new FileOutputStream(file.getFile(), true))) { this.turnRestrictions.forEach(value -> out.println(value)); } catch (final IOException oops) { throw new CoreException("Error writing turnRestriction ids to file", oops); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { StreamSupport.stream(new ComplexTurnRestrictionFinder().find(atlas).spliterator(), false) .filter(turnRestriction -> turnRestriction.isValid()) .map(turnRestriction -> turnRestriction.getOsmIdentifier()) .forEach(this.turnRestrictions::add); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasMetadataSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Reads the metadata from the input atlas files and writes it to stdout * * @author cstaylor */ public class AtlasMetadataSubCommand extends AbstractAtlasSubCommand { public AtlasMetadataSubCommand() { super("metadata", "outputs all of the metadata found in the atlas files"); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-combine : merge all of the atlas files into a MultiAtlas before outputting geojson\n"); } @Override protected void handle(final Atlas atlas, final CommandMap command) { final AtlasMetaData metaData = atlas.metaData(); System.out.printf("Atlas Meta Data for %s:\n%s\n", atlas.getName(), metaData.toReadableString()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasMissingISOSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.text.DecimalFormat; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Command that outputs the atlas file and OSM id of any areas missing ISO country codes from the * supplied atlas * * @author cstaylor */ public class AtlasMissingISOSubCommand extends AbstractAtlasSubCommand { private static final Logger logger = LoggerFactory.getLogger(AtlasMissingISOSubCommand.class); private AtomicInteger counter; private Time start; public AtlasMissingISOSubCommand() { super("isoloss", "outputs all of the atlas objects that are missing ISO country codes"); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-combine : merge all of the atlas files into a MultiAtlas before outputting geojson\n"); } @Override protected int finish(final CommandMap command) { final int value = this.counter.get(); if (value > 0) { logger.error(String.format("Total Items missing ISO Codes: %s", DecimalFormat.getNumberInstance().format(value))); } logger.info("Time elapsed {}", this.start.elapsedSince()); return Math.min(value, 1); } @Override protected void handle(final Atlas atlas, final CommandMap command) { final Predicate missingISOCountryTag = Validators .hasValuesFor(ISOCountryTag.class).negate(); StreamSupport.stream(atlas.items().spliterator(), true).filter(missingISOCountryTag) .forEach(this::log); } @Override protected void start(final CommandMap command) { this.counter = new AtomicInteger(); this.start = Time.now(); } private void log(final AtlasItem item) { this.counter.incrementAndGet(); logger.error(String.format("[item] [%25s] [%9d]", item.getAtlas().getName(), item.getOsmIdentifier())); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; import org.openstreetmap.atlas.utilities.runtime.FlexibleCommand; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfoList; import io.github.classgraph.ScanResult; /** * Shell for running atlas commands. Run this command with no arguments to learn more about it. * * @author cstaylor */ public class AtlasReader extends FlexibleCommand { public static void main(final String... args) { final AtlasReader reader = new AtlasReader(args); try { reader.runWithoutQuitting(args); } catch (final Exception e) { e.printStackTrace(); reader.printUsageAndExit(1); } } public AtlasReader(final String... args) { super(args); } @SuppressWarnings("unchecked") @Override protected Stream> getSupportedCommands() { final List> returnValue = new ArrayList<>(); try (ScanResult scanResult = new ClassGraph().enableAllInfo() .whitelistPackages(AtlasReader.class.getPackage().getName()).scan()) { final ClassInfoList classInfoList = scanResult .getClassesImplementing(FlexibleSubCommand.class.getName()); classInfoList.loadClasses() .forEach(klass -> returnValue.add((Class) klass)); } return returnValue.stream(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasResourceLoaderErrorSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.io.StreamCorruptedException; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.exception.ExceptionSearch; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Quick way of testing various problems when loading atlas files (missing files, corrupt files, * etc...) * * @author cstaylor */ public class AtlasResourceLoaderErrorSubCommand implements FlexibleSubCommand { private static final Logger logger = LoggerFactory .getLogger(AtlasResourceLoaderErrorSubCommand.class); private static final Switch INPUT_PARAMETER = new Switch<>("input", "Path of Atlas file", File::new, Command.Optionality.OPTIONAL); @Override public int execute(final CommandMap map) { final File input = (File) map.get(INPUT_PARAMETER); try { new AtlasResourceLoader().load(input); } catch (final CoreException oops) { logger.error("", oops); ExceptionSearch.find(StreamCorruptedException.class).within(oops) .ifPresent(error -> logger.error("", error)); } return 0; } @Override public String getDescription() { return "Testing the abstract resource loader"; } @Override public String getName() { return "resource-load-testing"; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.print("-input=/path/to/resources/for/loading/atlas/files"); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasSplitterWithSlippyTileCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Create several atlas files from one atlas file as input with fixed zoom level using SlippyTile as * the sharding method * * @author yalimu */ public class AtlasSplitterWithSlippyTileCommand extends AbstractAtlasSubCommand { private static final Logger logger = LoggerFactory .getLogger(AtlasSplitterWithSlippyTileCommand.class); private static final Command.Switch ZOOM_LEVEL = new Command.Switch<>("zoom_level", "Input zoom level", Integer::new, Command.Optionality.REQUIRED); private static final Command.Switch OUTPUT_FOLDER_PARAMETER = new Command.Switch<>( "output", "The path to save Atlas files", Paths::get, Command.Optionality.REQUIRED); public AtlasSplitterWithSlippyTileCommand() { super("split", "Split one Atlas file into several small Atlas files with fixed zoom level"); } @Override public Command.SwitchList switches() { return super.switches().with(ZOOM_LEVEL, OUTPUT_FOLDER_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf(AtlasCommandConstants.INPUT_ZOOM_LEVEL); writer.printf(AtlasCommandConstants.OUTPUT_FOLDER_DESCRIPTION); } @Override protected void handle(final Atlas atlas, final CommandMap command) { final int zoomLevel = (int) command.get(ZOOM_LEVEL); final Path path = (Path) command.get(OUTPUT_FOLDER_PARAMETER); try { Files.createDirectories(path); } catch (final IOException e) { logger.error("Error when creating output directory", e); return; } StreamSupport.stream(SlippyTile.allTiles(zoomLevel, atlas.bounds()).spliterator(), false) .map(tile -> buildAtlasBasedOnTile(tile, atlas)).forEach(newAtlas -> { if (newAtlas != null) { final String outputFileName = path.toString() + "/" + atlas.getName() + "_" + newAtlas.getIdentifier() + FileSuffix.ATLAS; logger.info("Saving Atlas file into {}", outputFileName); newAtlas.save(new File(outputFileName)); } }); } private PackedAtlas buildAtlasBasedOnTile(final SlippyTile tile, final Atlas atlas) { return atlas.subAtlas(tile.bounds(), AtlasCutType.SOFT_CUT) .map(subAtlas -> new PackedAtlasCloner().cloneFrom(subAtlas)).orElse(null); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/FerrySearchSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.Set; import java.util.TreeSet; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.tags.RouteTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Command for listing countries that have at least one ferry line * * @author cstaylor */ public class FerrySearchSubCommand extends AbstractAtlasSubCommand { private Set countries; public FerrySearchSubCommand() { super("ferries", "Searching for ferries"); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected int finish(final CommandMap command) { System.out.println(this.countries); return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { if (StreamSupport.stream(atlas.entities(i -> i.getIdentifier() > 0).spliterator(), true) .filter(entity -> Validators.isOfType(entity, RouteTag.class, RouteTag.FERRY)) .findFirst().isPresent()) { atlas.metaData().getCountry().ifPresent(this.countries::add); } } @Override protected void start(final CommandMap command) { this.countries = new TreeSet<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/OsmPbfToAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.PrintStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.commons.io.FilenameUtils; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.BridgeConfiguredFilter; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.tags.filters.ConfiguredTaggableFilter; import org.openstreetmap.atlas.utilities.configuration.StandardConfiguration; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * This command converts an OSM PBF file to an Atlas file. It requires the path to a pbf and an * output file. It also takes a number of optional parameters. * * @author bbreithaupt */ public class OsmPbfToAtlasSubCommand implements FlexibleSubCommand { private static final String NAME = "pbf-to-atlas"; private static final String DESCRIPTION = "Converts a PBF to an Atlas file."; // Required parameters private static final Switch INPUT_PARAMETER = new Switch<>("pbf", "Input PBF path", File::new, Optionality.REQUIRED); private static final Switch OUTPUT_PARAMETER = new Switch<>("output", "Output Atlas file path", File::new, Optionality.REQUIRED); // Filter parameters private static final Switch EDGE_FILTER_PARAMETER = new Switch<>("edge-filter", "Path to a local json filter for determining Edges", File::new, Optionality.OPTIONAL); private static final Switch NODE_FILTER_PARAMETER = new Switch<>("node-filter", "Path to a local json filter for OSM nodes", File::new, Optionality.OPTIONAL); private static final Switch RELATION_FILTER_PARAMETER = new Switch<>("relation-filter", "Path to a local json filter for OSM relations", File::new, Optionality.OPTIONAL); private static final Switch WAY_FILTER_PARAMETER = new Switch<>("way-filter", "Path to a local json filter for OSM ways", File::new, Optionality.OPTIONAL); private static final Switch WAY_SECTION_FILTER_PARAMETER = new Switch<>( "way-section-filter", "Path to a local json filter for determining where to way section", File::new, Optionality.OPTIONAL); // Load Parameters private static final Switch LOAD_RELATIONS_PARAMETER = new Switch<>("load-relations", "Whether to load Relations (boolean)", Boolean::parseBoolean, Optionality.OPTIONAL, "true"); private static final Switch LOAD_WAYS_PARAMETER = new Switch<>("load-ways", "Whether to load ways (boolean)", Boolean::parseBoolean, Optionality.OPTIONAL, "true"); // Country parameters private static final Switch COUNTRY_CODE_PARAMETER = new Switch<>("country-code", "Country from the country map to convert (comma separated ISO3 codes)", StringConverter.IDENTITY, Optionality.OPTIONAL); private static final Switch COUNTRY_MAP_PARAMETER = new Switch<>("country-boundary-map", "Path to a local WKT or shp file containing a country boundary map", File::new, Optionality.OPTIONAL); private static final Switch COUNTRY_SLICING_PARAMETER = new Switch<>("country-slicing", "Whether to perform country slicing (boolean)", Boolean::parseBoolean, Optionality.OPTIONAL, "true"); @Override public int execute(final CommandMap map) { final AtlasLoadingOption options = this.getAtlasLoadingOption(map); Atlas atlas = new RawAtlasGenerator((File) map.get(INPUT_PARAMETER), options, MultiPolygon.MAXIMUM).build(); if (options.isCountrySlicing()) { atlas = new RawAtlasSlicer(options, atlas).slice(); } atlas = new AtlasSectionProcessor(atlas, options).run(); atlas.save((File) map.get(OUTPUT_PARAMETER)); return 0; } @Override public String getDescription() { return DESCRIPTION; } @Override public String getName() { return NAME; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER, EDGE_FILTER_PARAMETER, NODE_FILTER_PARAMETER, RELATION_FILTER_PARAMETER, WAY_FILTER_PARAMETER, WAY_SECTION_FILTER_PARAMETER, LOAD_RELATIONS_PARAMETER, LOAD_WAYS_PARAMETER, COUNTRY_CODE_PARAMETER, COUNTRY_MAP_PARAMETER, COUNTRY_SLICING_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.println("-pbf=/path/to/pbf : pbf to convert"); writer.println("-output=/path/to/output/atlas : Atlas file to output to"); writer.println("-edge-filter=/path/to/json/edge/filter : json filter to determine Edges"); writer.println("-node-filter=/path/to/json/node/filter : json filter for OSM nodes"); writer.println( "-relation-filter=/path/to/json/relation/filter : json filter for OSM relations"); writer.println("-way-filter=/path/to/json/way/filter : json filter for OSM ways"); writer.println("-load-relations=boolean : whether to load Relations; defaults to true"); writer.println("-load-ways=boolean : whether to load ways; defaults to true"); writer.println( "-country-codes=list,of,ISO3,codes : countries from the country map to convert"); writer.println( "-country-boundary-map=/path/to/WKT/or/shp : a WKT or shp file containing a country boundary map"); writer.println( "-country-slicing=boolean : whether to perform country slicing; defaults to true"); writer.println( "-way-section-filter=/path/to/json/way/section/filter : json filter to determine where to way section"); } /** * Creates an {@link AtlasLoadingOption} using configurable parameters. If any of the parameters * are not set the defaults from {@link AtlasLoadingOption} are used. * * @param map * {@link CommandMap} * @return {@link AtlasLoadingOption} */ private AtlasLoadingOption getAtlasLoadingOption(final CommandMap map) { final CountryBoundaryMap countryMap = this.getCountryBoundaryMap(map); final AtlasLoadingOption options = AtlasLoadingOption .createOptionWithAllEnabled(countryMap); // Set filters map.getOption(EDGE_FILTER_PARAMETER) .ifPresent(filter -> options.setEdgeFilter( new BridgeConfiguredFilter("", AtlasLoadingOption.ATLAS_EDGE_FILTER_NAME, new StandardConfiguration((File) filter)))); map.getOption(NODE_FILTER_PARAMETER).ifPresent(filter -> options.setOsmPbfNodeFilter( new ConfiguredTaggableFilter(new StandardConfiguration((File) filter)))); map.getOption(RELATION_FILTER_PARAMETER) .ifPresent(filter -> options.setOsmPbfRelationFilter( new ConfiguredTaggableFilter(new StandardConfiguration((File) filter)))); map.getOption(WAY_FILTER_PARAMETER).ifPresent(filter -> options.setOsmPbfWayFilter( new ConfiguredTaggableFilter(new StandardConfiguration((File) filter)))); map.getOption(WAY_SECTION_FILTER_PARAMETER) .ifPresent(filter -> options.setWaySectionFilter(new BridgeConfiguredFilter("", AtlasLoadingOption.ATLAS_WAY_SECTION_FILTER_NAME, new StandardConfiguration((File) filter)))); // Set loading options ((Optional) map.getOption(LOAD_RELATIONS_PARAMETER)) .ifPresent(options::setLoadAtlasRelation); ((Optional) map.getOption(LOAD_WAYS_PARAMETER)).ifPresent(bool -> { options.setLoadAtlasLine(bool); options.setLoadAtlasEdge(bool); }); // Set country options ((Optional) map.getOption(COUNTRY_CODE_PARAMETER)) .ifPresent(options::setCountryCode); ((Optional) map.getOption(COUNTRY_SLICING_PARAMETER)) .ifPresent(options::setCountrySlicing); return options; } /** * Get or create a {@link CountryBoundaryMap}. If the country-boundary-map parameter is set, * this will attempt to load the text or shape file from that parameter. Else, this will load * using the entire world as the country UNK (unknown). * * @param map * {@link CommandMap} containing the {@code COUNTRY_MAP_PARAMETER} * @return {@link CountryBoundaryMap} loaded from a file or default */ private CountryBoundaryMap getCountryBoundaryMap(final CommandMap map) { final Optional countryMapOption = (Optional) map .getOption(COUNTRY_MAP_PARAMETER); final List boundaries = new ArrayList<>(); MultiPolygon.MAXIMUM.outers() .forEach(outer -> boundaries.add(new JtsPolygonConverter().convert(outer))); CountryBoundaryMap countryMap = CountryBoundaryMap .fromBoundaryMap(Collections.singletonMap("UNK", boundaries)); if (countryMapOption.isPresent()) { final File countryMapFile = countryMapOption.get(); if (FilenameUtils.isExtension(countryMapFile.getName(), "txt")) { countryMap = CountryBoundaryMap.fromPlainText(countryMapFile); } else if (FilenameUtils.isExtension(countryMapFile.getName(), "shp")) { countryMap = CountryBoundaryMap.fromShapeFile(countryMapFile.getFile()); } } return countryMap; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToTextAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * Flexible command for converting a {@link PackedAtlas} to a text Atlas * * @author cstaylor */ public class PackedToTextAtlasSubCommand implements FlexibleSubCommand { private static final String NAME = "packed-to-text"; private static final String DESCRIPTION = "converts a packed atlas to a text-based atlas"; private static final Switch INPUT_PARAMETER = new Switch<>("packed-atlas", "Input atlas data in packed atlas format", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_PARAMETER = new Switch<>("text-atlas", "Output text atlas data path", Paths::get, Optionality.REQUIRED); private Path inputPath; private Path outputPath; @Override public int execute(final CommandMap map) { this.inputPath = (Path) map.get(INPUT_PARAMETER); this.outputPath = (Path) map.get(OUTPUT_PARAMETER); preVerify(); PackedAtlas.load(new File(this.inputPath.toFile())) .saveAsText(new File(this.outputPath.toFile())); return 0; } @Override public String getDescription() { return DESCRIPTION; } @Override public String getName() { return NAME; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.println("-text-atlas=/output/path/to/text/atlas"); writer.println("-packed-atlas=/input/path/to/packed/atlas"); } private void preVerify() { if (!Files.isRegularFile(this.inputPath)) { throw new CoreException("{} is not a readable file", this.inputPath); } try { if (Files.isDirectory(this.outputPath)) { throw new CoreException("{} is a directory. Aborting", this.outputPath); } Files.createDirectories(this.outputPath.getParent()); } catch (final IOException oops) { throw new CoreException("Error when creating directories {}", this.outputPath.getParent(), oops); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/SubAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Flag; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Create a {@link MultiAtlas} from the set of input atlas files, creates a {@link PackedAtlas} from * the {@link MultiAtlas}, and then writes that {@link PackedAtlas} to the specified output file. * * @author mnahoum */ public class SubAtlasSubCommand extends AbstractAtlasSubCommand { private static final Switch OUTPUT = new Switch<>("output", "The file to save the Atlas to", Paths::get, Optionality.OPTIONAL); private static final Switch SUB = new Switch<>("sub", "The rectangle to soft-cut this Atlas to", Rectangle::forString, Optionality.REQUIRED); private static final Switch GEOJSON = new Switch<>("geojson", "The geojson file to save this sub atlas to", Paths::get, Optionality.OPTIONAL); private static final Flag SAVE_MEMORY = new Flag("saveMemory", "Reduce momery cost if this flag is existed"); private final List atlases = new ArrayList<>(); public SubAtlasSubCommand() { super("sub", "Subs an Atlas into another smaller Atlas"); } @Override public SwitchList switches() { return super.switches().with(OUTPUT, SUB, GEOJSON, SAVE_MEMORY); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf( "-output=/path/to/atlas/output/to/save : the path to the output atlas file\n"); writer.printf( "-geojson=/path/to/atlas/geojson/to/save : the path to the output geojson file (optional)\n"); writer.printf("-sub=minLat,minLon:maxLat,maxLon : the rectangle to sub the Atlas with\n"); writer.printf("-saveMemory : a flag to save memory\n"); } @Override protected int finish(final CommandMap command) { final Atlas atlas = new MultiAtlas(this.atlases); final Rectangle rectangle = (Rectangle) command.get(SUB); try { final Atlas saveMe = new PackedAtlasCloner().cloneFrom(atlas) .subAtlas(rectangle, AtlasCutType.SOFT_CUT).orElseThrow( () -> new CoreException("There are no features in the sub rectangle.")); final Path path = (Path) command.get(OUTPUT); Files.createDirectories(path.getParent()); saveMe.save(new File(path.toString())); final Path path2 = (Path) command.get(GEOJSON); if (path2 != null) { Files.createDirectories(path2.getParent()); saveMe.saveAsGeoJson(new File(path2.toString())); } return 0; } catch (final IOException oops) { throw new CoreException("Error when saving packed atlas", oops); } } @Override protected void handle(final Atlas atlas, final CommandMap command) { final boolean saveMemory = (boolean) command.get(SAVE_MEMORY); final Rectangle rectangle = (Rectangle) command.get(SUB); if (!saveMemory || rectangle.overlaps(atlas.bounds())) { this.atlases.add(atlas); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/TextToPackedAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * Flexible command for converting a text-based Atlas to a serialized PackedAtlas * * @author cstaylor */ public class TextToPackedAtlasSubCommand implements FlexibleSubCommand { private static final String NAME = "text-to-packed"; private static final String DESCRIPTION = "converts a text-based atlas to a packed atlas"; private static final Switch INPUT_PARAMETER = new Switch<>("text-atlas", "Input atlas data in text atlas format", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_PARAMETER = new Switch<>("packed-atlas", "Output atlas data path", Paths::get, Optionality.REQUIRED); private Path inputPath; private Path outputPath; @Override public int execute(final CommandMap map) { this.inputPath = (Path) map.get(INPUT_PARAMETER); this.outputPath = (Path) map.get(OUTPUT_PARAMETER); preVerify(); new TextAtlasBuilder().read(new File(this.inputPath.toFile())) .save(new File(this.outputPath.toFile())); return 0; } @Override public String getDescription() { return DESCRIPTION; } @Override public String getName() { return NAME; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.println("-text-atlas=/input/path/to/text/atlas"); writer.println("-packed-atlas=/output/path/to/packed/atlas"); } private void preVerify() { if (!Files.isRegularFile(this.inputPath)) { throw new CoreException("{} is not a readable file", this.inputPath); } try { if (Files.isDirectory(this.outputPath)) { throw new CoreException("{} is a directory. Aborting", this.outputPath); } Files.createDirectories(this.outputPath.getParent()); } catch (final IOException oops) { throw new CoreException("Error when creating directories {}", this.outputPath.getParent(), oops); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/buildings/AtlasFindBuildingPartsSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command.buildings; import java.io.PrintStream; import java.util.function.Predicate; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.command.AbstractAtlasSubCommand; import org.openstreetmap.atlas.geography.atlas.command.AtlasCommandConstants; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.tags.BuildingPartTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Outputs all of the atlas items that have building:part tags * * @author cstaylor */ public class AtlasFindBuildingPartsSubCommand extends AbstractAtlasSubCommand { private static void output(final AtlasItem item) { System.out.printf("[%25s] [%9d] %s\n", item.getAtlas().getName(), item.getOsmIdentifier(), item); } public AtlasFindBuildingPartsSubCommand() { super("building-parts", "Task for finding all of the OSM ids with building:part=yes"); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); } @Override protected void handle(final Atlas atlas, final CommandMap command) { final Predicate filter = Validators.hasValuesFor(BuildingPartTag.class); StreamSupport.stream(atlas.items().spliterator(), true).filter(filter) .forEach(AtlasFindBuildingPartsSubCommand::output); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/buildings/BuildingsWithHeightSearchSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command.buildings; import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; import java.util.Optional; import java.util.TreeSet; import java.util.function.Consumer; import java.util.stream.StreamSupport; import org.apache.commons.compress.utils.IOUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.command.AbstractAtlasSubCommand; import org.openstreetmap.atlas.geography.atlas.command.AtlasCommandConstants; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuilding; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuildingFinder; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import com.google.common.collect.ComparisonChain; /** * Outputs information about buildings that have heights (making them 3D) * * @author cstaylor */ public class BuildingsWithHeightSearchSubCommand extends AbstractAtlasSubCommand { /** * Data item class for holding onto the information we need to log about building heights * * @author cstaylor */ private static final class BuildingHeightItem implements Comparable { private final String iso3; private double height; private final long osmIdentifier; private final long atlasIdentifier; private final String latitude; private final String longitude; private final String url; private final boolean valid; BuildingHeightItem(final ComplexBuilding building) { this.url = String.format("http://www.openstreetmap.org/%s/%d", building.getSource().getType() == ItemType.AREA ? "way" : "relation", building.getOsmIdentifier()); final Optional outline = building.getOutline(); this.valid = outline.isPresent(); if (outline.isPresent()) { final Location location = outline.get().outers().iterator().next().first(); this.iso3 = building.getTag(ISOCountryTag.class, Optional.empty()).orElse("UNK"); this.atlasIdentifier = building.getIdentifier(); this.osmIdentifier = building.getOsmIdentifier(); building.topHeight().ifPresent(height -> { this.height = height.asMeters(); }); this.latitude = location.getLatitude().toString(); this.longitude = location.getLongitude().toString(); } else { this.iso3 = null; this.atlasIdentifier = -1; this.osmIdentifier = -1; this.latitude = null; this.longitude = null; } } @Override public int compareTo(final BuildingHeightItem otherBuilding) { return ComparisonChain.start().compare(this.iso3, otherBuilding.iso3) .compare(this.height, otherBuilding.height) .compare(this.atlasIdentifier, otherBuilding.atlasIdentifier).result(); } @Override public boolean equals(final Object otherObject) { if (this == otherObject) { return true; } if (otherObject instanceof BuildingHeightItem) { final BuildingHeightItem otherItem = (BuildingHeightItem) otherObject; return this.atlasIdentifier == otherItem.atlasIdentifier; } return false; } @Override public int hashCode() { return Objects.hash(this.atlasIdentifier); } private boolean isValid() { return this.valid; } private void output(final int count, final PrintStream stream) { stream.printf( "%d%s%s%s%s%.2f%s,%s\n", count, this.iso3, this.atlasIdentifier, this.osmIdentifier, this.url, this.url, this.height, this.latitude, this.longitude); } } /** * Logs some basic information about each building that has a height * * @author cstaylor */ private static final class BuildingsWithHeightLogger implements Consumer, Closeable { private final PrintStream output; private final TreeSet items; private BuildingsWithHeightLogger(final PrintStream output) { this.output = output; output.printf( "\n"); this.items = new TreeSet<>(); } @Override public void accept(final ComplexBuilding buildingsWithHeight) { final BuildingHeightItem item = new BuildingHeightItem(buildingsWithHeight); if (item.isValid()) { this.items.add(item); } } @Override public void close() throws IOException { final int[] counter = { 0 }; this.items.stream().forEach(item -> { item.output(++counter[0], this.output); }); this.output.printf("
\n"); IOUtils.closeQuietly(this.output); } } private static final Switch OUTPUT_FILE_PARAMETER = new Switch<>("output", "HTML file containing information about each 3D building", Paths::get, Optionality.REQUIRED); private BuildingsWithHeightLogger counter; public BuildingsWithHeightSearchSubCommand() { super("3d-buildings", "Lists all of the buildings that have a height value"); } @Override public SwitchList switches() { return super.switches().with(OUTPUT_FILE_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf("-output=/path/to/output/file\n"); } @Override protected int finish(final CommandMap command) { try { this.counter.close(); } catch (final IOException oops) { throw new CoreException("Failure to close", oops); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { StreamSupport.stream(new ComplexBuildingFinder().find(atlas).spliterator(), false) .map(ComplexBuilding.class::cast).filter(this::hasHeight).forEach(this.counter); } @Override protected void start(final CommandMap command) { super.start(command); this.counter = new BuildingsWithHeightLogger(createStream(command)); } private PrintStream createStream(final CommandMap command) { try { final Path output = (Path) command.get(OUTPUT_FILE_PARAMETER); try { Files.createDirectories(output.getParent()); } catch (final IOException oops) { throw new CoreException("Error when creating output directory", oops); } return new PrintStream(new BufferedOutputStream(new FileOutputStream(output.toFile()))); } catch (final IOException oops) { throw new CoreException("Failure to open output", oops); } } private boolean hasHeight(final ComplexBuilding building) { return building.topHeight().isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/command/buildings/TinyBuildingsSearchSubCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.command.buildings; import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Consumer; import java.util.stream.StreamSupport; import org.apache.commons.compress.utils.IOUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.command.AbstractAtlasSubCommand; import org.openstreetmap.atlas.geography.atlas.command.AtlasCommandConstants; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuilding; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuildingFinder; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.scalars.Surface; /** * Outputs information about buildings that have a surface area of less than 1 square meter * * @author cstaylor */ public class TinyBuildingsSearchSubCommand extends AbstractAtlasSubCommand { /** * Logs some basic information about each building that fails the 1 sq meter test * * @author cstaylor */ private static final class TinyBuildingLogger implements Consumer, Closeable { private final PrintStream output; private TinyBuildingLogger(final PrintStream output) { this.output = output; } @Override public void accept(final ComplexBuilding tinyBuilding) { final String url = String.format("http://www.openstreetmap.org/%s/%d", tinyBuilding.getSource().getType() == ItemType.AREA ? "way" : "relation", tinyBuilding.getOsmIdentifier()); final Optional outline = tinyBuilding.getOutline(); if (outline.isPresent()) { this.output.printf("%s,%d,%d,%s,%.2f\n", tinyBuilding.getTag(ISOCountryTag.class, Optional.empty()).orElse("UNK"), tinyBuilding.getIdentifier(), tinyBuilding.getOsmIdentifier(), url, outline.get().surface().asMeterSquared()); } } @Override public void close() throws IOException { IOUtils.closeQuietly(this.output); } } private static final Switch MINIMUM_BUILDING_SIZE_PARAMETER = new Switch<>("minimum", "The minimum area permitted for a building", value -> Surface.UNIT_METER_SQUARED_ON_EARTH_SURFACE.scaleBy(Double.valueOf(value)), Optionality.REQUIRED); private static final Switch OUTPUT_FILE_PARAMETER = new Switch<>("output", "File containing the CSV information about each tiny building", Paths::get, Optionality.REQUIRED); private Surface minimumSurface = Surface.UNIT_METER_SQUARED_ON_EARTH_SURFACE; private TinyBuildingLogger counter; public TinyBuildingsSearchSubCommand() { super("buildings-for-ants", "Lists all of the buildings with areas smaller than a given size"); } @Override public SwitchList switches() { return super.switches().with(MINIMUM_BUILDING_SIZE_PARAMETER, OUTPUT_FILE_PARAMETER); } public boolean tooSmall(final ComplexBuilding building) { final Optional outline = building.getOutline(); if (outline.isPresent()) { return outline.get().surface().isLessThanOrEqualTo(this.minimumSurface); } else { return false; } } @Override public void usage(final PrintStream writer) { writer.printf(AtlasCommandConstants.INPUT_PARAMETER_DESCRIPTION); writer.printf("-minimum=[scale factor of square meters]\n"); writer.printf("-output=/path/to/output/file\n"); } @Override protected int finish(final CommandMap command) { try { this.counter.close(); } catch (final IOException oops) { throw new CoreException("Failure to close", oops); } return 0; } @Override protected void handle(final Atlas atlas, final CommandMap command) { StreamSupport.stream(new ComplexBuildingFinder().find(atlas).spliterator(), false) .map(ComplexBuilding.class::cast).filter(this::tooSmall).forEach(this.counter); } @Override protected void start(final CommandMap command) { super.start(command); this.minimumSurface = (Surface) command.get(MINIMUM_BUILDING_SIZE_PARAMETER); this.counter = new TinyBuildingLogger(createStream(command)); } private PrintStream createStream(final CommandMap command) { try { final Path output = (Path) command.get(OUTPUT_FILE_PARAMETER); try { Files.createDirectories(output.getParent()); } catch (final IOException oops) { throw new CoreException("Error when creating output directory", oops); } return new PrintStream(new BufferedOutputStream(new FileOutputStream(output.toFile()))); } catch (final IOException oops) { throw new CoreException("Failure to open output", oops); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Independent {@link Area} that contains its own data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompleteArea extends Area implements CompleteEntity { private static final long serialVersionUID = 309534717673911086L; private Rectangle bounds; private long identifier; private Polygon polygon; private Map tags; private Set relationIdentifiers; private Set geometricRelationIdentifiers; private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); /** * Create a {@link CompleteArea} from a given {@link Area} reference. The {@link CompleteArea}'s * fields will match the fields of the reference. The returned {@link CompleteArea} will be * full, i.e. all of its associated fields will be non-null. * * @param area * the {@link Area} to copy * @return the full {@link CompleteArea} */ public static CompleteArea from(final Area area) { if (area instanceof CompleteArea && !((CompleteArea) area).isFull()) { throw new CoreException("Area parameter was a CompleteArea but it was not full: {}", area); } return new CompleteArea(area.getIdentifier(), area.asPolygon(), area.getTags(), area.relations().stream().map(Relation::getIdentifier).collect(Collectors.toSet())) .withGeometricRelationIdentifiers( area.relations().stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet())); } /** * Create a shallow {@link CompleteArea} from a given {@link Area} reference. The * {@link CompleteArea}'s identifier will match the identifier of the reference {@link Area}. * The returned {@link CompleteArea} will be shallow, i.e. all of its associated fields will be * null except for the identifier. * * @param area * the {@link Area} to copy * @return the shallow {@link CompleteArea} */ public static CompleteArea shallowFrom(final Area area) { if (area.bounds() == null) { throw new CoreException("Area parameter bounds were null"); } return new CompleteArea(area.getIdentifier()).withBoundsExtendedBy(area.bounds()); } public CompleteArea(final Long identifier, final Polygon polygon, final Map tags, final Set relationIdentifiers) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = polygon != null ? polygon.bounds() : null; this.identifier = identifier; this.polygon = polygon; this.tags = tags; this.relationIdentifiers = relationIdentifiers; } protected CompleteArea(final long identifier) { this(identifier, null, null, null); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public Polygon asPolygon() { return this.polygon; } @Override public Rectangle bounds() { return this.bounds; } @Override public CompleteItemType completeItemType() { return CompleteItemType.AREA; } public CompleteArea copy() { return new CompleteArea(this.identifier, this.polygon, this.tags, this.relationIdentifiers); } @Override public boolean equals(final Object other) { if (other instanceof CompleteArea) { final CompleteArea that = (CompleteArea) other; return CompleteEntity.basicEqual(this, that) && Objects.equals(this.asPolygon(), that.asPolygon()); } return false; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } public Set geometricRelationIdentifiers() { return this.geometricRelationIdentifiers; } @Override public Iterable getGeometry() { if (this.polygon != null) { return new ArrayList<>(this.polygon); } return null; } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } @Override public boolean isFull() { return this.bounds != null && this.polygon != null && this.tags != null && this.relationIdentifiers != null; } @Override public boolean isShallow() { return this.polygon == null && this.tags == null && this.relationIdentifiers == null; } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.polygon != null) { if (truncate) { builder.append("geometry: " + truncate(this.polygon.toString()) + ", "); } else { builder.append("geometry: " + this.polygon.toString() + ", "); } builder.append(separator); } if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers) + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", polygon=" + this.polygon + ", tags=" + this.tags + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.polygon == null) { return null; } return this.polygon.toWkt(); } @Override public CompleteArea withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompleteArea withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } public CompleteArea withGeometricRelationIdentifiers( final Set geometricRelationIdentifiers) { this.geometricRelationIdentifiers = geometricRelationIdentifiers; return this; } @Override public CompleteEntity withGeometry(final Iterable locations) { return this.withPolygon(new Polygon(locations)); } @Override public CompleteArea withIdentifier(final long identifier) { this.identifier = identifier; return this; } public CompleteArea withPolygon(final Polygon polygon) { this.polygon = polygon; if (this.polygon != null) { this.bounds = polygon.bounds(); } return this; } @Override public CompleteArea withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompleteArea withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = relations.stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet()); return this; } @Override public CompleteArea withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = this.geometricRelationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.gson.JsonObject; /** * Independent {@link Edge} that contains its own data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompleteEdge extends Edge implements CompleteLineItem { private static final long serialVersionUID = 309534717673911086L; private Rectangle bounds; private long identifier; private PolyLine polyLine; private Map tags; private Long startNodeIdentifier; private Long endNodeIdentifier; private Set relationIdentifiers; private Set geometricRelationIdentifiers; private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); /** * Create a {@link CompleteEdge} from a given {@link Edge} reference. The {@link CompleteEdge}'s * fields will match the fields of the reference. The returned {@link CompleteEdge} will be * full, i.e. all of its associated fields will be non-null. * * @param edge * the {@link Edge} to copy * @return the full {@link CompleteEdge} */ public static CompleteEdge from(final Edge edge) { if (edge instanceof CompleteEdge && !((CompleteEdge) edge).isFull()) { throw new CoreException("Edge parameter was a CompleteEdge but it was not full: {}", edge); } return new CompleteEdge(edge.getIdentifier(), edge.asPolyLine(), edge.getTags(), edge.start().getIdentifier(), edge.end().getIdentifier(), edge.relations().stream().map(Relation::getIdentifier).collect(Collectors.toSet())) .withGeometricRelationIdentifiers( edge.relations().stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet())); } /** * Create a shallow {@link CompleteEdge} from a given {@link Edge} reference. The * {@link CompleteEdge}'s identifier will match the identifier of the reference {@link Edge}. * The returned {@link CompleteEdge} will be shallow, i.e. all of its associated fields will be * null except for the identifier. * * @param edge * the {@link Edge} to copy * @return the shallow {@link CompleteEdge} */ public static CompleteEdge shallowFrom(final Edge edge) { if (edge.bounds() == null) { throw new CoreException("Edge parameter bounds were null"); } return new CompleteEdge(edge.getIdentifier()).withBoundsExtendedBy(edge.bounds()); } public CompleteEdge(final Long identifier, final PolyLine polyLine, final Map tags, final Long startNodeIdentifier, final Long endNodeIdentifier, final Set relationIdentifiers) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = polyLine != null ? polyLine.bounds() : null; this.identifier = identifier; this.polyLine = polyLine; this.tags = tags; this.startNodeIdentifier = startNodeIdentifier; this.endNodeIdentifier = endNodeIdentifier; this.relationIdentifiers = relationIdentifiers; } protected CompleteEdge(final long identifier) { this(identifier, null, null, null, null, null); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public PolyLine asPolyLine() { return this.polyLine; } @Override public Rectangle bounds() { return this.bounds; } @Override public CompleteItemType completeItemType() { return CompleteItemType.EDGE; } public CompleteEdge copy() { return new CompleteEdge(this.identifier, this.polyLine, this.tags, this.startNodeIdentifier, this.endNodeIdentifier, this.relationIdentifiers); } @Override public Node end() { /* * Note that the Node returned by this method will technically break the Located contract, * since it has null bounds. */ return this.endNodeIdentifier == null ? null : new CompleteNode(this.endNodeIdentifier); } public Long endNodeIdentifier() { return this.endNodeIdentifier; } @Override public boolean equals(final Object other) { if (other instanceof CompleteEdge) { final CompleteEdge that = (CompleteEdge) other; return CompleteEntity.basicEqual(this, that) && Objects.equals(this.asPolyLine(), that.asPolyLine()) && CompleteEntity.equalThroughGet(this.start(), that.start(), Node::getIdentifier) && CompleteEntity.equalThroughGet(this.end(), that.end(), Node::getIdentifier); } return false; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } public Set geometricRelationIdentifiers() { return this.geometricRelationIdentifiers; } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } @Override public boolean isFull() { return this.bounds != null && this.polyLine != null && this.tags != null && this.startNodeIdentifier != null && this.endNodeIdentifier != null && this.relationIdentifiers != null; } @Override public boolean isShallow() { return this.polyLine == null && this.startNodeIdentifier == null && this.endNodeIdentifier == null && this.tags == null && this.relationIdentifiers == null; } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.polyLine != null) { if (truncate) { builder.append("geometry: " + truncate(this.polyLine.toString()) + ", "); } else { builder.append("geometry: " + this.polyLine.toString() + ", "); } builder.append(separator); } if (this.startNodeIdentifier != null) { builder.append("startNode: " + this.startNodeIdentifier + ", "); builder.append(separator); } if (this.endNodeIdentifier != null) { builder.append("endNode: " + this.endNodeIdentifier + ", "); builder.append(separator); } if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers) + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public Node start() { /* * Note that the Node returned by this method will technically break the Located contract, * since it has null bounds. */ return this.startNodeIdentifier == null ? null : new CompleteNode(this.startNodeIdentifier); } public Long startNodeIdentifier() { return this.startNodeIdentifier; } @Override public JsonObject toJson() { final JsonObject edgeObject = super.toJson(); edgeObject.addProperty("startNode", this.startNodeIdentifier); edgeObject.addProperty("endNode", this.endNodeIdentifier); return edgeObject; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", startNodeIdentifier=" + this.startNodeIdentifier + ", endNodeIdentifier=" + this.endNodeIdentifier + ", polyLine=" + this.polyLine + ", tags=" + this.tags + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.polyLine == null) { return null; } return this.polyLine.toWkt(); } @Override public CompleteEdge withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompleteEdge withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } public CompleteEdge withEndNodeIdentifier(final Long endNodeIdentifier) { this.endNodeIdentifier = endNodeIdentifier; return this; } public CompleteEdge withGeometricRelationIdentifiers( final Set geometricRelationIdentifiers) { this.geometricRelationIdentifiers = geometricRelationIdentifiers; return this; } @Override public CompleteEntity withGeometry(final Iterable locations) { return this.withPolyLine(new PolyLine(locations)); } @Override public CompleteEdge withIdentifier(final long identifier) { this.identifier = identifier; return this; } @Override public CompleteEdge withPolyLine(final PolyLine polyLine) { this.polyLine = polyLine; if (this.polyLine != null) { this.bounds = polyLine.bounds(); } return this; } @Override public CompleteEdge withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompleteEdge withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = relations.stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet()); return this; } @Override public CompleteEdge withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = this.geometricRelationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } public CompleteEdge withStartNodeIdentifier(final Long startNodeIdentifier) { this.startNodeIdentifier = startNodeIdentifier; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listenable.TagChangeListenable; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.gson.JsonObject; /** * Simple interface for all the Complete entities. As each one extends its parent class already * (Node, Edge, Area, ...) this cannot be an abstract class. * * @param * the {@link CompleteEntity} implementation * @author matthieun * @author Yazad Khambata */ public interface CompleteEntity> extends TagChangeListenable { static Map addNewTag(final Map tags, final String key, final String value) { Map result = new HashMap<>(); if (tags != null) { result = new HashMap<>(tags); } result.put(key, value); return result; } /** * A simple equality check that only looks at identifiers, tags, and parent relations. * * @param left * the left entity * @param right * the right entity * @return if the left and right entities are related through a simple equality */ static boolean basicEqual(final AtlasEntity left, final AtlasEntity right) { return left.getIdentifier() == right.getIdentifier() && Objects.equals(left.getTags(), right.getTags()) && Objects.equals(left.relations(), right.relations()); } static boolean equalThroughGet(final M left, final M right, final Function getter) { if (left == null && right == null) { return true; } else if (left == null || right == null) { return false; } else { return Objects.equals(getter.apply(left), getter.apply(right)); } } /** * Create a {@link CompleteEntity} from a given {@link AtlasEntity} reference. The * {@link CompleteEntity}'s fields will match the fields of the reference. The returned * {@link CompleteEntity} will be full, i.e. all of its associated fields will be non-null. * * @param reference * the reference to copy * @return the full entity */ static AtlasEntity from(final AtlasEntity reference) { final ItemType type = reference.getType(); switch (type) { case NODE: return CompleteNode.from((Node) reference); case EDGE: return CompleteEdge.from((Edge) reference); case AREA: return CompleteArea.from((Area) reference); case LINE: return CompleteLine.from((Line) reference); case POINT: return CompletePoint.from((Point) reference); case RELATION: return CompleteRelation.from((Relation) reference); default: throw new CoreException("Unknown ItemType {}", type); } } static Map removeTag(final Map tags, final String key) { Map result = new HashMap<>(); if (tags != null) { result = new HashMap<>(tags); } result.remove(key); return result; } /** * Create a shallow {@link CompleteEntity} from a given {@link AtlasEntity} reference. The * {@link CompleteEntity}'s identifier will match the identifier of the reference. The returned * {@link CompleteEntity} will be shallow, i.e. all of its associated fields will be null except * for the identifier. * * @param reference * the reference to copy * @return the shallow entity */ static AtlasEntity shallowFrom(final AtlasEntity reference) { final ItemType type = reference.getType(); switch (type) { case NODE: return CompleteNode.shallowFrom((Node) reference); case EDGE: return CompleteEdge.shallowFrom((Edge) reference); case AREA: return CompleteArea.shallowFrom((Area) reference); case LINE: return CompleteLine.shallowFrom((Line) reference); case POINT: return CompletePoint.shallowFrom((Point) reference); case RELATION: return CompleteRelation.shallowFrom((Relation) reference); default: throw new CoreException("Unknown ItemType {}", type); } } /** * Create a shallow {@link CompleteEntity} from a given {@link ItemType} and identifier. * * @param type * the {@link ItemType} * @param identifier * the identifier * @return a shallow {@link CompleteEntity} that matches the requested parameters */ static AtlasEntity shallowFrom(final ItemType type, final Long identifier) { switch (type) { case NODE: return new CompleteNode(identifier); case EDGE: return new CompleteEdge(identifier); case AREA: return new CompleteArea(identifier); case LINE: return new CompleteLine(identifier); case POINT: return new CompletePoint(identifier); case RELATION: return new CompleteRelation(identifier); default: throw new CoreException("Unknown ItemType {}", type); } } static > C withAddedTag(final C completeEntity, final String key, final String value, final boolean suppressFiringEvent) { CompleteEntity.withTags(completeEntity, CompleteEntity.addNewTag(completeEntity.getTags(), key, value), true); if (!suppressFiringEvent) { completeEntity .fireTagChangeEvent(TagChangeEvent.added(completeEntity.completeItemType(), completeEntity.getIdentifier(), Pair.of(key, value))); } return completeEntity; } static > C withRemovedTag(final C completeEntity, final String key, final boolean suppressFiringEvent) { CompleteEntity.withTags(completeEntity, CompleteEntity.removeTag(completeEntity.getTags(), key), true); if (!suppressFiringEvent) { completeEntity.fireTagChangeEvent(TagChangeEvent.remove( completeEntity.completeItemType(), completeEntity.getIdentifier(), key)); } return completeEntity; } static > C withReplacedTag(final C completeEntity, final String oldKey, final String newKey, final String newValue, final boolean suppressFiringEvent) { CompleteEntity.withRemovedTag(completeEntity, oldKey, true); CompleteEntity.withAddedTag(completeEntity, newKey, newValue, true); if (!suppressFiringEvent) { completeEntity .fireTagChangeEvent(TagChangeEvent.replaced(completeEntity.completeItemType(), completeEntity.getIdentifier(), Triple.of(oldKey, newKey, newValue))); } return completeEntity; } static > C withTags(final C completeEntity, final Map tags, final boolean suppressFiringEvent) { completeEntity.setTags(tags); if (!suppressFiringEvent) { completeEntity.fireTagChangeEvent(TagChangeEvent.overwrite( completeEntity.completeItemType(), completeEntity.getIdentifier(), tags)); } return completeEntity; } CompleteItemType completeItemType(); Iterable getGeometry(); long getIdentifier(); Map getTags(); ItemType getType(); /** * A full {@link CompleteEntity} is one one that contains a non-null value for all its fields. * * @return if this entity is full */ boolean isFull(); /** * A shallow {@link CompleteEntity} is one that contains only its identifier as effective data. * * @return if this entity is shallow */ boolean isShallow(); /** * Transform this {@link CompleteEntity} into a pretty string. The pretty string for a * {@link CompleteEntity} can be customized using different available formats. * * @param format * the format type for the pretty string * @param truncate * if the string should be truncated * @return the pretty string */ String prettify(PrettifyStringFormat format, boolean truncate); /** * Transform this {@link CompleteEntity} into a pretty string. The pretty string for a * {@link CompleteEntity} can be customized using different available formats. * * @param format * the format type for the pretty string * @return the pretty string */ default String prettify(final PrettifyStringFormat format) { return prettify(PrettifyStringFormat.MINIMAL_SINGLE_LINE, true); } /** * Transform this {@link CompleteEntity} into a pretty string. This method uses the default * format {@link PrettifyStringFormat#MINIMAL_SINGLE_LINE}. * * @return the pretty string */ default String prettify() { return prettify(PrettifyStringFormat.MINIMAL_SINGLE_LINE); } Set relationIdentifiers(); void setTags(Map tags); JsonObject toJson(); /** * Get the WKT for this entity's geometry. * * @return the WKT of this entity's geometry, null if the geometry is null */ String toWkt(); default String truncate(final String input) { return input.substring(0, Math.min(input.length(), PrettifyStringFormat.TRUNCATE_LENGTH)) + PrettifyStringFormat.TRUNCATE_ELLIPSES; } CompleteEntity withAddedRelationIdentifier(Long relationIdentifier); default C withAddedTag(final String key, final String value) { return CompleteEntity.withAddedTag((C) this, key, value, false); } CompleteEntity withGeometry(Iterable locations); CompleteEntity withIdentifier(long identifier); CompleteEntity withRelationIdentifiers(Set relationIdentifiers); CompleteEntity withRelations(Set relations); CompleteEntity withRemovedRelationIdentifier(Long relationIdentifier); default C withRemovedTag(final String key) { return CompleteEntity.withRemovedTag((C) this, key, false); } default C withReplacedTag(final String oldKey, final String newKey, final String newValue) { return CompleteEntity.withReplacedTag((C) this, oldKey, newKey, newValue, false); } default C withTags(final Map tags) { return CompleteEntity.withTags((C) this, tags, false); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteItemType.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.Arrays; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; /** * A mapping of {@link CompleteEntity}-ies to {@link ItemType}. * * @author Yazad Khambata */ public enum CompleteItemType { NODE(CompleteNode.class, ItemType.NODE), EDGE(CompleteEdge.class, ItemType.EDGE), AREA(CompleteArea.class, ItemType.AREA), LINE(CompleteLine.class, ItemType.LINE), POINT(CompletePoint.class, ItemType.POINT), RELATION(CompleteRelation.class, ItemType.RELATION); private final Class completeEntityClass; private final ItemType itemType; public static CompleteItemType from(final ItemType itemType) { return Arrays.stream(CompleteItemType.values()) .filter(completeItemType -> completeItemType.itemType == itemType).findFirst() .orElseThrow(IllegalArgumentException::new); } public static C shallowFrom(final AtlasEntity reference) { final ItemType itemType = reference.getType(); final CompleteItemType completeItemType = CompleteItemType.from(itemType); final C completeEntity = completeItemType.completeEntityShallowFrom(reference); return completeEntity; } CompleteItemType(final Class completeEntityClass, final ItemType itemType) { this.completeEntityClass = completeEntityClass; this.itemType = itemType; } public C completeEntityFrom(final AtlasEntity reference) { validate(reference); return (C) CompleteEntity.from(reference); } public C completeEntityShallowFrom(final AtlasEntity reference) { validate(reference); return (C) CompleteEntity.shallowFrom(reference); } public Class getCompleteEntityClass() { return this.completeEntityClass; } public ItemType getItemType() { return this.itemType; } private void validate(final AtlasEntity reference) { Validate.isTrue(getItemType().getMemberClass().isAssignableFrom(reference.getClass()), "reference: " + reference + "; cannot be converted to completed entity " + this + "."); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Independent {@link Line} that contains its own data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompleteLine extends Line implements CompleteLineItem { private static final long serialVersionUID = 309534717673911086L; private Rectangle bounds; private long identifier; private PolyLine polyLine; private Map tags; private Set geometricRelationIdentifiers; private Set relationIdentifiers; private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); /** * Create a {@link CompleteLine} from a given {@link Line} reference. The {@link CompleteLine}'s * fields will match the fields of the reference. The returned {@link CompleteLine} will be * full, i.e. all of its associated fields will be non-null. * * @param line * the {@link Line} to copy * @return the full {@link CompleteLine} */ public static CompleteLine from(final Line line) { if (line instanceof CompleteLine && !((CompleteLine) line).isFull()) { throw new CoreException("Line parameter was a CompleteLine but it was not full: {}", line); } return new CompleteLine(line.getIdentifier(), line.asPolyLine(), line.getTags(), line.relations().stream().map(Relation::getIdentifier).collect(Collectors.toSet())) .withGeometricRelationIdentifiers( line.relations().stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet())); } /** * Create a shallow {@link CompleteLine} from a given {@link Line} reference. The * {@link CompleteLine}'s identifier will match the identifier of the reference {@link Line}. * The returned {@link CompleteLine} will be shallow, i.e. all of its associated fields will be * null except for the identifier. * * @param line * the {@link Line} to copy * @return the shallow {@link CompleteLine} */ public static CompleteLine shallowFrom(final Line line) { if (line.bounds() == null) { throw new CoreException("Line parameter bounds were null"); } return new CompleteLine(line.getIdentifier()).withBoundsExtendedBy(line.bounds()); } public CompleteLine(final Long identifier, final PolyLine polyLine, final Map tags, final Set relationIdentifiers) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = polyLine != null ? polyLine.bounds() : null; this.identifier = identifier; this.polyLine = polyLine; this.tags = tags; this.relationIdentifiers = relationIdentifiers; this.geometricRelationIdentifiers = new HashSet<>(); } protected CompleteLine(final long identifier) { this(identifier, null, null, null); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public PolyLine asPolyLine() { return this.polyLine; } @Override public Rectangle bounds() { return this.bounds; } @Override public CompleteItemType completeItemType() { return CompleteItemType.LINE; } public CompleteLine copy() { return new CompleteLine(this.identifier, this.polyLine, this.tags, this.relationIdentifiers); } @Override public boolean equals(final Object other) { if (other instanceof CompleteLine) { final CompleteLine that = (CompleteLine) other; return CompleteEntity.basicEqual(this, that) && Objects.equals(this.asPolyLine(), that.asPolyLine()); } return false; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } public Set geometricRelationIdentifiers() { return this.geometricRelationIdentifiers; } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } @Override public boolean isFull() { return this.bounds != null && this.polyLine != null && this.tags != null && this.relationIdentifiers != null; } @Override public boolean isShallow() { return this.polyLine == null && this.tags == null && this.relationIdentifiers == null; } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.polyLine != null) { if (truncate) { builder.append("polyLine: " + truncate(this.polyLine.toString()) + ", "); } else { builder.append("polyLine: " + this.polyLine.toString() + ", "); } builder.append(separator); } if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers) + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", polyLine=" + this.polyLine + ", tags=" + this.tags + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.polyLine == null) { return null; } return this.polyLine.toWkt(); } @Override public CompleteLine withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompleteLine withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } public CompleteLine withGeometricRelationIdentifiers( final Set geometricRelationIdentifiers) { this.geometricRelationIdentifiers = geometricRelationIdentifiers; return this; } @Override public CompleteEntity withGeometry(final Iterable locations) { return this.withPolyLine(new PolyLine(locations)); } @Override public CompleteLine withIdentifier(final long identifier) { this.identifier = identifier; return this; } @Override public CompleteLine withPolyLine(final PolyLine polyLine) { this.polyLine = polyLine; if (this.polyLine != null) { this.bounds = polyLine.bounds(); } return this; } @Override public CompleteLine withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompleteLine withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = relations.stream().filter(Relation::isGeometric) .filter(relation -> relation.asMultiPolygon().isPresent() && !relation.asMultiPolygon().get().isEmpty()) .map(Relation::getIdentifier).collect(Collectors.toSet()); return this; } @Override public CompleteLine withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); this.geometricRelationIdentifiers = this.geometricRelationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteLineItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.ArrayList; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; /** * Similar to a {@link org.openstreetmap.atlas.geography.atlas.items.LineItem} but for * {@link CompleteEntity}-ies. * * @param * the {@link CompleteEntity} being worked on. * @author Yazad Khambata */ public interface CompleteLineItem> extends CompleteEntity { PolyLine asPolyLine(); @Override default Iterable getGeometry() { if (asPolyLine() != null) { return new ArrayList<>(asPolyLine()); } return null; } CompleteLineItem withPolyLine(PolyLine polyLine); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteLocationItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Location; /** * Similar to a {@link org.openstreetmap.atlas.geography.atlas.items.LocationItem} but for * {@link CompleteEntity}-ies. * * @param * - the {@link CompleteEntity} bveing worked on. * @author Yazad Khambata */ public interface CompleteLocationItem> extends CompleteEntity { @Override default Iterable getGeometry() { if (getLocation() != null) { final List geometry = new ArrayList<>(); geometry.add(getLocation()); return geometry; } return null; } Location getLocation(); CompleteLocationItem withLocation(Location location); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * Independent {@link Node} that may contain its own altered data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompleteNode extends Node implements CompleteLocationItem { private static final long serialVersionUID = -8229589987121555419L; private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); private Rectangle bounds; private long identifier; private Location location; private Map tags; private SortedSet inEdgeIdentifiers; private SortedSet outEdgeIdentifiers; private Set explicitlyExcludedInEdgeIdentifiers; private Set explicitlyExcludedOutEdgeIdentifiers; private Set relationIdentifiers; /** * Create a {@link CompleteNode} from a given {@link Node} reference. The {@link CompleteNode}'s * fields will match the fields of the reference. The returned {@link CompleteNode} will be * full, i.e. all of its associated fields will be non-null. * * @param node * the {@link Node} to copy * @return the full {@link CompleteNode} */ public static CompleteNode from(final Node node) { if (node instanceof CompleteNode && !((CompleteNode) node).isFull()) { throw new CoreException("Node parameter was a CompleteNode but it was not full: {}", node); } return new CompleteNode(node.getIdentifier(), node.getLocation(), node.getTags(), node.inEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)), node.outEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)), node.relations().stream().map(Relation::getIdentifier).collect(Collectors.toSet())); } /** * Create a shallow {@link CompleteNode} from a given {@link Node} reference. The * {@link CompleteNode}'s identifier will match the identifier of the reference {@link Node}. * The returned {@link CompleteNode} will be shallow, i.e. all of its associated fields will be * null except for the identifier. * * @param node * the {@link Node} to copy * @return the shallow {@link CompleteNode} */ public static CompleteNode shallowFrom(final Node node) { if (node.bounds() == null) { throw new CoreException("Node parameter bounds were null"); } return new CompleteNode(node.getIdentifier()).withBoundsExtendedBy(node.bounds()); } public CompleteNode(final Long identifier, final Location location, final Map tags, final SortedSet inEdgeIdentifiers, final SortedSet outEdgeIdentifiers, final Set relationIdentifiers) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = location != null ? location.bounds() : null; this.identifier = identifier; this.location = location; this.tags = tags; this.inEdgeIdentifiers = inEdgeIdentifiers; this.outEdgeIdentifiers = outEdgeIdentifiers; this.explicitlyExcludedInEdgeIdentifiers = new HashSet<>(); this.explicitlyExcludedOutEdgeIdentifiers = new HashSet<>(); this.relationIdentifiers = relationIdentifiers; } CompleteNode(final long identifier) { this(identifier, null, null, null, null, null); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public Rectangle bounds() { return this.bounds; } @Override public CompleteItemType completeItemType() { return CompleteItemType.NODE; } public CompleteNode copy() { return new CompleteNode(this.identifier, this.location, this.tags, this.inEdgeIdentifiers, this.outEdgeIdentifiers, this.relationIdentifiers); } @Override public boolean equals(final Object other) { if (other instanceof CompleteNode) { final CompleteNode that = (CompleteNode) other; return CompleteEntity.basicEqual(this, that) && Objects.equals(this.getLocation(), that.getLocation()) && Objects.equals(this.inEdges(), that.inEdges()) && Objects.equals(this.outEdges(), that.outEdges()); } return false; } public Set explicitlyExcludedInEdgeIdentifiers() { return this.explicitlyExcludedInEdgeIdentifiers; } public Set explicitlyExcludedOutEdgeIdentifiers() { return this.explicitlyExcludedOutEdgeIdentifiers; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return this.location; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } public Set inEdgeIdentifiers() { return this.inEdgeIdentifiers; } @Override public SortedSet inEdges() { /* * Note that the Edges returned by this method will technically break the Located contract, * since they have null bounds. */ return this.inEdgeIdentifiers == null ? null : this.inEdgeIdentifiers.stream().map(CompleteEdge::new) .collect(Collectors.toCollection(TreeSet::new)); } @Override public boolean isFull() { return this.bounds != null && this.location != null && this.tags != null && this.inEdgeIdentifiers != null && this.outEdgeIdentifiers != null && this.relationIdentifiers != null; } @Override public boolean isShallow() { return this.location == null && this.inEdgeIdentifiers == null && this.outEdgeIdentifiers == null && this.tags == null && this.relationIdentifiers == null; } public Set outEdgeIdentifiers() { return this.outEdgeIdentifiers; } @Override public SortedSet outEdges() { /* * Note that the Edges returned by this method will technically break the Located contract, * since they have null bounds. */ return this.outEdgeIdentifiers == null ? null : this.outEdgeIdentifiers.stream().map(CompleteEdge::new) .collect(Collectors.toCollection(TreeSet::new)); } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.location != null) { builder.append("geometry: " + this.location + ", "); builder.append(separator); } if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.inEdgeIdentifiers != null) { builder.append("inEdges: " + this.inEdgeIdentifiers + ", "); builder.append(separator); } if (this.explicitlyExcludedInEdgeIdentifiers != null && !this.explicitlyExcludedInEdgeIdentifiers.isEmpty()) { builder.append("explicitlyExcludedInEdges: " + new TreeSet<>(this.explicitlyExcludedInEdgeIdentifiers) + ", "); builder.append(separator); } if (this.outEdgeIdentifiers != null) { builder.append("outEdges: " + this.outEdgeIdentifiers + ", "); builder.append(separator); } if (this.explicitlyExcludedOutEdgeIdentifiers != null && !this.explicitlyExcludedOutEdgeIdentifiers.isEmpty()) { builder.append("explicitlyExcludedOutEdges: " + new TreeSet<>(this.explicitlyExcludedOutEdgeIdentifiers) + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers) + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } public void setExplicitlyExcludedInEdgeIdentifiers(final Set edges) { this.explicitlyExcludedInEdgeIdentifiers = edges; } public void setExplicitlyExcludedOutEdgeIdentifiers(final Set edges) { this.explicitlyExcludedOutEdgeIdentifiers = edges; } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public JsonObject toJson() { final JsonObject nodeObject = super.toJson(); final JsonArray inEdgeIdentifiersArray = new JsonArray(); for (final Long inEdgeIdentifier : new TreeSet<>(this.inEdgeIdentifiers)) { inEdgeIdentifiersArray.add(new JsonPrimitive(inEdgeIdentifier)); } final JsonArray outEdgeIdentifiersArray = new JsonArray(); for (final Long outEdgeIdentifier : new TreeSet<>(this.outEdgeIdentifiers)) { outEdgeIdentifiersArray.add(new JsonPrimitive(outEdgeIdentifier)); } nodeObject.add("inEdges", inEdgeIdentifiersArray); nodeObject.add("outEdges", outEdgeIdentifiersArray); return nodeObject; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", inEdgeIdentifiers=" + this.inEdgeIdentifiers + ", outEdgeIdentifiers=" + this.outEdgeIdentifiers + ", location=" + this.location + ", tags=" + this.tags + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.location == null) { return null; } return this.location.toWkt(); } public CompleteNode withAddedInEdgeIdentifier(final Long inEdgeIdentifier) { this.inEdgeIdentifiers.add(inEdgeIdentifier); return this; } public CompleteNode withAddedOutEdgeIdentifier(final Long inEdgeIdentifier) { this.outEdgeIdentifiers.add(inEdgeIdentifier); return this; } @Override public CompleteNode withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompleteNode withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } @Override public CompleteEntity withGeometry(final Iterable locations) { if (!locations.iterator().hasNext()) { throw new CoreException("Cannot interpret empty Iterable as a Location"); } return this.withLocation(locations.iterator().next()); } @Override public CompleteNode withIdentifier(final long identifier) { this.identifier = identifier; return this; } public CompleteNode withInEdgeIdentifiers(final SortedSet inEdgeIdentifiers) { this.inEdgeIdentifiers = inEdgeIdentifiers; return this; } public CompleteNode withInEdgeIdentifiersAndSource(final SortedSet inEdgeIdentifiers, final Node source) { final Set sourceIdentifiers = source.inEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toSet()); final Set excludedBasedOnSource = com.google.common.collect.Sets .difference(sourceIdentifiers, inEdgeIdentifiers); this.inEdgeIdentifiers = inEdgeIdentifiers; this.explicitlyExcludedInEdgeIdentifiers.addAll(excludedBasedOnSource); return this; } public CompleteNode withInEdges(final Set inEdges) { this.inEdgeIdentifiers = inEdges.stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)); return this; } public CompleteNode withInEdgesAndSource(final Set inEdges, final Node source) { return withInEdgeIdentifiersAndSource(inEdges.stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)), source); } @Override public CompleteNode withLocation(final Location location) { this.location = location; if (this.location != null) { this.bounds = location.bounds(); } return this; } public CompleteNode withOutEdgeIdentifiers(final SortedSet outEdgeIdentifiers) { this.outEdgeIdentifiers = outEdgeIdentifiers; return this; } public CompleteNode withOutEdgeIdentifiersAndSource(final SortedSet outEdgeIdentifiers, final Node source) { final Set sourceIdentifiers = source.outEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toSet()); final Set excludedBasedOnSource = com.google.common.collect.Sets .difference(sourceIdentifiers, outEdgeIdentifiers); this.outEdgeIdentifiers = outEdgeIdentifiers; this.explicitlyExcludedOutEdgeIdentifiers.addAll(excludedBasedOnSource); return this; } public CompleteNode withOutEdges(final Set outEdges) { this.outEdgeIdentifiers = outEdges.stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)); return this; } public CompleteNode withOutEdgesAndSource(final Set outEdges, final Node source) { return withOutEdgeIdentifiersAndSource(outEdges.stream().map(Edge::getIdentifier) .collect(Collectors.toCollection(TreeSet::new)), source); } @Override public CompleteNode withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompleteNode withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); return this; } public CompleteNode withRemovedInEdgeIdentifier(final Long inEdgeIdentifier) { this.inEdgeIdentifiers.remove(inEdgeIdentifier); this.explicitlyExcludedInEdgeIdentifiers.add(inEdgeIdentifier); return this; } public CompleteNode withRemovedOutEdgeIdentifier(final Long outEdgeIdentifier) { this.outEdgeIdentifiers.remove(outEdgeIdentifier); this.explicitlyExcludedOutEdgeIdentifiers.add(outEdgeIdentifier); return this; } @Override public CompleteNode withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } public CompleteNode withReplacedInEdgeIdentifier(final Long beforeInEdgeIdentifier, final Long afterInEdgeIdentifier) { return this.withRemovedInEdgeIdentifier(beforeInEdgeIdentifier) .withAddedInEdgeIdentifier(afterInEdgeIdentifier); } public CompleteNode withReplacedOutEdgeIdentifier(final Long beforeOutEdgeIdentifier, final Long afterOutEdgeIdentifier) { return this.withRemovedOutEdgeIdentifier(beforeOutEdgeIdentifier) .withAddedOutEdgeIdentifier(afterOutEdgeIdentifier); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompletePoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Independent {@link Point} that contains its own data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompletePoint extends Point implements CompleteLocationItem { private static final long serialVersionUID = 309534717673911086L; private Rectangle bounds; private long identifier; private Location location; private Map tags; private Set relationIdentifiers; private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); /** * Create a {@link CompletePoint} from a given {@link Point} reference. The * {@link CompletePoint}'s fields will match the fields of the reference. The returned * {@link CompletePoint} will be full, i.e. all of its associated fields will be non-null. * * @param point * the {@link Point} to copy * @return the full {@link CompletePoint} */ public static CompletePoint from(final Point point) { if (point instanceof CompletePoint && !((CompletePoint) point).isFull()) { throw new CoreException("Point parameter was a CompletePoint but it was not full: {}", point); } return new CompletePoint(point.getIdentifier(), point.getLocation(), point.getTags(), point .relations().stream().map(Relation::getIdentifier).collect(Collectors.toSet())); } /** * Create a shallow {@link CompletePoint} from a given {@link Point} reference. The * {@link CompletePoint}'s identifier will match the identifier of the reference {@link Point}. * The returned {@link CompletePoint} will be shallow, i.e. all of its associated fields will be * null except for the identifier. * * @param point * the {@link Point} to copy * @return the shallow {@link CompletePoint} */ public static CompletePoint shallowFrom(final Point point) { if (point.bounds() == null) { throw new CoreException("Point parameter bounds were null"); } return new CompletePoint(point.getIdentifier()).withBoundsExtendedBy(point.bounds()); } public CompletePoint(final Long identifier, final Location location, final Map tags, final Set relationIdentifiers) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = location != null ? location.bounds() : null; this.identifier = identifier; this.location = location; this.tags = tags; this.relationIdentifiers = relationIdentifiers; } CompletePoint(final long identifier) { this(identifier, null, null, null); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public Rectangle bounds() { return this.bounds; } @Override public CompleteItemType completeItemType() { return CompleteItemType.POINT; } public CompletePoint copy() { return new CompletePoint(this.identifier, this.location, this.tags, this.relationIdentifiers); } @Override public boolean equals(final Object other) { if (other instanceof CompletePoint) { final CompletePoint that = (CompletePoint) other; return CompleteEntity.basicEqual(this, that) && Objects.equals(this.getLocation(), that.getLocation()); } return false; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return this.location; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } @Override public boolean isFull() { return this.bounds != null && this.location != null && this.tags != null && this.relationIdentifiers != null; } @Override public boolean isShallow() { return this.location == null && this.tags == null && this.relationIdentifiers == null; } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.location != null) { builder.append("geometry: " + this.location + ", "); builder.append(separator); } if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers).toString() + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", location=" + this.location + ", tags=" + this.tags + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.location == null) { return null; } return this.location.toWkt(); } @Override public CompletePoint withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompletePoint withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } @Override public CompletePoint withGeometry(final Iterable locations) { if (!locations.iterator().hasNext()) { throw new CoreException("Cannot interpret empty Iterable as a Location"); } return this.withLocation(locations.iterator().next()); } @Override public CompletePoint withIdentifier(final long identifier) { this.identifier = identifier; return this; } @Override public CompletePoint withLocation(final Location location) { this.location = location; if (this.location != null) { this.bounds = location.bounds(); } return this; } @Override public CompletePoint withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompletePoint withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); return this; } @Override public CompletePoint withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/CompleteRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonObject; /** * Independent {@link Relation} that contains its own data. At scale, use at your own risk. * * @author matthieun * @author Yazad Khambata */ public class CompleteRelation extends Relation implements CompleteEntity { private static final long serialVersionUID = -8295865049110084558L; private static final Logger logger = LoggerFactory.getLogger(CompleteRelation.class); private Rectangle bounds; private long identifier; private Map tags; private RelationBean members; private List allRelationsWithSameOsmIdentifier; private RelationBean allKnownOsmMembers; private Long osmRelationIdentifier; private Set relationIdentifiers; private MultiPolygon storedGeometry; private boolean overrideGeometry = false; private final Set addedGeometry = new HashSet<>(); private final Set removedGeometry = new HashSet<>(); private final TagChangeDelegate tagChangeDelegate = TagChangeDelegate.newTagChangeDelegate(); /** * Create a {@link CompleteRelation} from a given {@link Relation} reference. The * {@link CompleteRelation}'s fields will match the fields of the reference. The returned * {@link CompleteRelation} will be full, i.e. all of its associated fields will be non-null. * * @param relation * the {@link Relation} to copy * @return the full {@link CompleteRelation} */ public static CompleteRelation from(final Relation relation) { if (relation instanceof CompleteRelation && !((CompleteRelation) relation).isFull()) { throw new CoreException( "Relation parameter was a CompleteRelation but it was not full: {}", relation); } return new CompleteRelation(relation.getIdentifier(), relation.getTags(), relation.bounds(), relation.members().asBean(), relation.allRelationsWithSameOsmIdentifier().stream().map(Relation::getIdentifier) .collect(Collectors.toList()), relation.allKnownOsmMembers().asBean(), relation.osmRelationIdentifier(), relation.relations().stream().map(Relation::getIdentifier) .collect(Collectors.toSet()), relation.asMultiPolygon().orElse(null)); } /** * Create a shallow {@link CompleteRelation} from a given {@link Relation} reference. The * {@link CompleteRelation}'s identifier will match the identifier of the reference * {@link Relation}. The returned {@link CompleteRelation} will be shallow, i.e. all of its * associated fields will be null except for the identifier. * * @param relation * the {@link Relation} to copy * @return the shallow {@link CompleteRelation} */ public static CompleteRelation shallowFrom(final Relation relation) { if (relation.bounds() == null) { throw new CoreException("Relation parameter bounds were null"); } return new CompleteRelation(relation.getIdentifier()) .withBoundsExtendedBy(relation.bounds()); } public CompleteRelation(final long identifier) { this(identifier, null, null, null, null, null, null, null); } public CompleteRelation(final Long identifier, final Map tags, // NOSONAR final Rectangle bounds, final RelationBean members, final List allRelationsWithSameOsmIdentifier, final RelationBean allKnownOsmMembers, final Long osmRelationIdentifier, final Set relationIdentifiers) { this(identifier, tags, bounds, members, allRelationsWithSameOsmIdentifier, allKnownOsmMembers, osmRelationIdentifier, relationIdentifiers, null); } public CompleteRelation(final Long identifier, final Map tags, // NOSONAR final Rectangle bounds, final RelationBean members, final List allRelationsWithSameOsmIdentifier, final RelationBean allKnownOsmMembers, final Long osmRelationIdentifier, final Set relationIdentifiers, final MultiPolygon jtsGeometry) { super(new EmptyAtlas()); if (identifier == null) { throw new CoreException("Identifier can never be null."); } this.bounds = bounds != null ? bounds : null; this.identifier = identifier; this.tags = tags; this.members = members; this.allRelationsWithSameOsmIdentifier = allRelationsWithSameOsmIdentifier; this.allKnownOsmMembers = allKnownOsmMembers; this.osmRelationIdentifier = osmRelationIdentifier; this.relationIdentifiers = relationIdentifiers; this.storedGeometry = jtsGeometry; } protected CompleteRelation(final Atlas atlas) { super(atlas); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { this.tagChangeDelegate.addTagChangeListener(tagChangeListener); } @Override public RelationMemberList allKnownOsmMembers() { return membersFor(this.allKnownOsmMembers); } @Override public List allRelationsWithSameOsmIdentifier() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.allRelationsWithSameOsmIdentifier == null ? null : this.allRelationsWithSameOsmIdentifier.stream().map(CompleteRelation::new) .collect(Collectors.toList()); } @Override public Optional asMultiPolygon() { return Optional.ofNullable(this.storedGeometry); } @Override public Rectangle bounds() { return this.bounds; } public CompleteRelation changeMemberRole(final AtlasEntity member, final String role) { final Optional oldItem = this.members.getItemFor(member.getIdentifier(), member.getType()); if (oldItem.isPresent()) { if (isGeometric() && this.asMultiPolygon().isPresent()) { throw new CoreException( "Cannot modify roles directly for relations with existing geometry! Please remove and add members directly instead"); } else { final RelationBeanItem old = oldItem.get(); this.members.removeItem(old); final RelationBeanItem newItem = new RelationBeanItem(member.getIdentifier(), role, member.getType()); this.members.addItem(newItem); this.members.addItemExplicitlyExcluded(member.getIdentifier(), old.getRole(), member.getType()); } } return this; } @Override public CompleteItemType completeItemType() { return CompleteItemType.RELATION; } public CompleteRelation copy() { return new CompleteRelation(this.identifier, this.tags, this.bounds, this.members, this.allRelationsWithSameOsmIdentifier, this.allKnownOsmMembers, this.osmRelationIdentifier, this.relationIdentifiers, this.storedGeometry); } @Override public boolean equals(final Object other) { if (other instanceof CompleteRelation) { final CompleteRelation that = (CompleteRelation) other; return CompleteEntity.basicEqual(this, that) && CompleteEntity.equalThroughGet(this.members(), that.members(), RelationMemberList::asBean) && Objects.equals(this.allRelationsWithSameOsmIdentifier(), that.allRelationsWithSameOsmIdentifier()) && CompleteEntity.equalThroughGet(this.allKnownOsmMembers(), that.allKnownOsmMembers(), RelationMemberList::asBean) && Objects.equals(this.osmRelationIdentifier(), that.osmRelationIdentifier()) && Objects.equals(this.asMultiPolygon(), that.asMultiPolygon()) && Objects.equals(this.getAddedGeometry(), that.getAddedGeometry()) && Objects.equals(this.getRemovedGeometry(), that.getRemovedGeometry()); } return false; } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { this.tagChangeDelegate.fireTagChangeEvent(tagChangeEvent); } public Set getAddedGeometry() { if (this.overrideGeometry) { return new HashSet<>(); } return this.addedGeometry; } @Override public Iterable getGeometry() { // TO DO enable for geometric? throw new UnsupportedOperationException("Relations do not have an explicit geometry." + " Please instead use bounds to check the apparent geometry."); } @Override public long getIdentifier() { return this.identifier; } public Set getRemovedGeometry() { if (this.overrideGeometry) { return new HashSet<>(); } return this.removedGeometry; } @Override public Map getTags() { return this.tags; } @Override public int hashCode() { return super.hashCode(); } @Override public boolean isFull() { return this.bounds != null && this.tags != null && this.members != null && this.allRelationsWithSameOsmIdentifier != null && this.allKnownOsmMembers != null && this.osmRelationIdentifier != null && this.relationIdentifiers != null; } @Override public boolean isGeometric() { return this.getTags() != null && super.isGeometric(); } public boolean isOverrideGeometry() { return this.overrideGeometry; } @Override public boolean isShallow() { return this.bounds == null && this.members == null && this.allRelationsWithSameOsmIdentifier == null && this.allKnownOsmMembers == null && this.osmRelationIdentifier == null && this.tags == null && this.relationIdentifiers == null && this.storedGeometry == null; } @Override public RelationMemberList members() { return membersFor(this.members); } @Override public Long osmRelationIdentifier() { return this.osmRelationIdentifier; } @Override public String prettify(final PrettifyStringFormat format, final boolean truncate) { String separator = ""; if (format == PrettifyStringFormat.MINIMAL_SINGLE_LINE) { separator = ""; } else if (format == PrettifyStringFormat.MINIMAL_MULTI_LINE) { separator = "\n"; } final StringBuilder builder = new StringBuilder(); builder.append(this.getClass().getSimpleName() + " "); builder.append("["); builder.append(separator); builder.append("identifier: " + this.identifier + ", "); builder.append(separator); if (this.tags != null) { builder.append("tags: " + new TreeMap<>(this.tags) + ", "); builder.append(separator); } if (this.members != null && !this.members.isEmpty()) { builder.append("members: " + this.members + ", "); builder.append(separator); } if (this.relationIdentifiers != null) { builder.append("parentRelations: " + new TreeSet<>(this.relationIdentifiers) + ", "); builder.append(separator); } if (this.bounds != null) { builder.append("bounds: " + this.bounds.toWkt() + ", "); builder.append(separator); } if (this.isGeometric()) { builder.append("multiPolygonGeometry: " + this.asMultiPolygon().map(Geometry::toText).orElse("null")); } builder.append(separator); builder.append("]"); return builder.toString(); } @Override public Set relationIdentifiers() { return this.relationIdentifiers; } @Override public Set relations() { /* * Note that the Relations returned by this method will technically break the Located * contract, since they have null bounds. */ return this.relationIdentifiers == null ? null : this.relationIdentifiers.stream().map(CompleteRelation::new) .collect(Collectors.toSet()); } @Override public void removeTagChangeListeners() { this.tagChangeDelegate.removeTagChangeListeners(); } @Override public void setTags(final Map tags) { this.tags = tags != null ? new HashMap<>(tags) : null; } @Override public JsonObject toJson() { final JsonObject relationObject = super.toJson(); final JsonArray membersArray = new JsonArray(); for (final RelationBeanItem item : this.members) { final JsonObject memberObject = new JsonObject(); memberObject.addProperty("type", item.getType().toString()); memberObject.addProperty("identifier", item.getIdentifier()); memberObject.addProperty("role", item.getRole()); membersArray.add(memberObject); } relationObject.add("members", membersArray); relationObject.addProperty("multiPolygonGeometry", this.asMultiPolygon().map(Geometry::toText).orElse("null")); return relationObject; } @Override public String toString() { return this.getClass().getSimpleName() + " [identifier=" + this.identifier + ", tags=" + this.tags + ", members=" + this.members + ", relationIdentifiers=" + this.relationIdentifiers + "]"; } @Override public String toWkt() { if (this.isGeometric() && this.storedGeometry != null) { return this.storedGeometry.toText(); } if (this.bounds == null) { return null; } return this.bounds.toWkt(); } public void updateGeometry() { try { this.storedGeometry = new JtsMultiPolygonToMultiPolygonConverter() .backwardConvert(new RelationOrAreaToMultiPolygonConverter().convert(this)); } catch (final Exception exc) { logger.error("Couldn't reconstruct geometry for relation {}", this, exc); this.storedGeometry = null; } } public CompleteRelation withAddedMember(final AtlasEntity newMember, final AtlasEntity memberFromWhichToCopyRole) { // TO DO support geometry final Relation parentRelation = Iterables.stream(memberFromWhichToCopyRole.relations()) .firstMatching(relation -> relation.getIdentifier() == this.getIdentifier()) .orElseThrow(() -> new CoreException( "Cannot copy role from {} {} as it does not have relation {} as parent", memberFromWhichToCopyRole.getType(), memberFromWhichToCopyRole.getIdentifier(), this.getIdentifier())); final String role = parentRelation.members().asBean() .getItemFor(memberFromWhichToCopyRole.getIdentifier(), memberFromWhichToCopyRole.getType()) .orElseThrow(() -> new CoreException( "Cannot copy role from {} {} as it is not a member of {} {}", memberFromWhichToCopyRole.getType(), memberFromWhichToCopyRole.getIdentifier(), this.getClass().getSimpleName(), this)) .getRole(); return withAddedMember(newMember, role); } public CompleteRelation withAddedMember(final AtlasEntity newMember, final String role) { if (this.members == null) { final Collection newMembers = new ArrayList<>(); newMembers.add(new RelationMember(role, newMember, this.getIdentifier())); final RelationMemberList memberList = new RelationMemberList(newMembers); return this.withMembers(memberList); } this.members.addItem( new RelationBeanItem(newMember.getIdentifier(), role, newMember.getType())); if (this.isGeometric() && this.asMultiPolygon().isPresent() && (role.equalsIgnoreCase(Ring.INNER.toString()) || role.equalsIgnoreCase(Ring.OUTER.toString()))) { switch (newMember.getType()) { case LINE: this.addedGeometry.add( new JtsPolyLineConverter().convert(((Line) newMember).asPolyLine())); break; case AREA: this.addedGeometry.add( new JtsPolyLineConverter().convert(((Area) newMember).asPolygon())); break; case EDGE: this.addedGeometry.add( new JtsPolyLineConverter().convert(((Edge) newMember).asPolyLine())); break; case NODE: case POINT: case RELATION: default: break; } } this.withBoundsExtendedBy(newMember.bounds()); return this; } @Override public CompleteRelation withAddedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers.add(relationIdentifier); return this; } public CompleteRelation withAllKnownOsmMembers(final RelationBean allKnownOsmMembers) { this.allKnownOsmMembers = allKnownOsmMembers; return this; } public CompleteRelation withAllRelationsWithSameOsmIdentifier( final List allRelationsWithSameOsmIdentifier) { this.allRelationsWithSameOsmIdentifier = allRelationsWithSameOsmIdentifier; return this; } public CompleteRelation withBounds(final Rectangle bounds) { this.bounds = bounds; return this; } public CompleteRelation withBoundsExtendedBy(final Rectangle bounds) { if (this.bounds == null) { this.bounds = bounds; return this; } this.bounds = Rectangle.forLocated(this.bounds, bounds); return this; } @Override public CompleteEntity withGeometry(final Iterable locations) { throw new UnsupportedOperationException("Relations cannot have an explicit geometry." + " Please instead use withBounds or withBoundsExtendedBy to adjust the bounds."); } @Override public CompleteRelation withIdentifier(final long identifier) { this.identifier = identifier; return this; } /** * Assign this {@link CompleteRelation} with members. *

* In case this {@link CompleteRelation} is created from an existing relation, and the new * member list has had some existing members removed, use * {@link #withMembersAndSource(RelationBean, Relation, Rectangle)} * * @param members * The members of the relation * @param bounds * The bounds of all the members of the relation. * @return This */ public CompleteRelation withMembers(final RelationBean members, final Rectangle bounds) { this.members = members; updateBounds(bounds); return this; } /** * Assign this {@link CompleteRelation} with members. *

* In case this {@link CompleteRelation} is created from an existing relation, and the new * member list has had some existing members removed, use * {@link #withMembersAndSource(RelationMemberList, Relation)} * * @param members * The full members of the Relation * @return This */ public CompleteRelation withMembers(final RelationMemberList members) { return withMembers(members.asBean(), members.bounds()); } /** * @param members * The members of the relation * @param source * The relation that was used as a base to create that {@link CompleteRelation}, if * any. Due to the weak nature of relation membership across Atlas(es), this helps * decide what relation members are forcibly removed if any. * @param bounds * The bounds of all the members of the relation. * @return This. */ public CompleteRelation withMembersAndSource(final RelationBean members, final Relation source, final Rectangle bounds) { if (source == null) { throw new CoreException("Source relation must not be null."); } this.members = members; // This has been created from an existing relation, make sure to record the members that // have been intentionally omitted, so as not to add them back in the future when either // merging FeatureChanges or stitching MultiAtlases. for (final RelationMember member : source.members()) { if (!members.getItemFor(member.getEntity().getIdentifier(), member.getRole(), member.getEntity().getType()).isPresent()) { this.members.addItemExplicitlyExcluded(member.getEntity().getIdentifier(), member.getRole(), member.getEntity().getType()); } } updateBounds(bounds); return this; } /** * Here the members have to be full so the new bounds can be computed from them. * * @param members * The full members of the Relation * @param source * The relation that was used as a base to create that {@link CompleteRelation}, if * any. Due to the weak nature of relation membership across Atlas(es), this helps * decide what relation members are forcibly removed if any. * @return This. */ public CompleteRelation withMembersAndSource(final RelationMemberList members, final Relation source) { if (source instanceof CompleteRelation) { throw new CoreException( "This version of withMembersAndSource must use a source Relation that is tied to an atlas, instead found Relation of type {}", source.getClass().getName()); } return withMembersAndSource(members.asBean(), source, members.bounds()); } public CompleteRelation withMultiPolygonGeometry(final MultiPolygon geometry) { this.storedGeometry = geometry; this.overrideGeometry = true; if (geometry != null && geometry.getEnvelope() instanceof Polygon) { this.bounds = Rectangle.forLocated( new JtsPolygonConverter().backwardConvert((Polygon) geometry.getEnvelope())); } return this; } public CompleteRelation withOsmRelationIdentifier(final Long osmRelationIdentifier) { this.osmRelationIdentifier = osmRelationIdentifier; return this; } @Override public CompleteRelation withRelationIdentifiers(final Set relationIdentifiers) { this.relationIdentifiers = relationIdentifiers; return this; } @Override public CompleteRelation withRelations(final Set relations) { this.relationIdentifiers = relations.stream().map(Relation::getIdentifier) .collect(Collectors.toSet()); return this; } public CompleteRelation withRemovedMember(final AtlasEntity memberToRemove) { final List roles = this.members .removeAllMatchingItems(memberToRemove.getIdentifier(), memberToRemove.getType()); roles.forEach(role -> this.members.addItemExplicitlyExcluded(memberToRemove.getIdentifier(), role, memberToRemove.getType())); if (this.isGeometric() && this.asMultiPolygon().isPresent() && (roles.contains(Ring.INNER.toString()) || roles.contains(Ring.OUTER.toString()) || roles.contains(Ring.INNER.toString().toLowerCase()) || roles.contains(Ring.OUTER.toString().toLowerCase()))) { switch (memberToRemove.getType()) { case LINE: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Line) memberToRemove).asPolyLine())); break; case AREA: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Area) memberToRemove).asPolygon())); break; case EDGE: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Edge) memberToRemove).asPolyLine())); break; case NODE: case POINT: case RELATION: default: break; } } return this; } public CompleteRelation withRemovedMember(final AtlasEntity memberToRemove, final String role) { final boolean success = this.members.removeItem(memberToRemove.getIdentifier(), role, memberToRemove.getType()); if (success) { this.members.addItemExplicitlyExcluded(memberToRemove.getIdentifier(), role, memberToRemove.getType()); if (this.isGeometric() && this.asMultiPolygon().isPresent() && (role.equalsIgnoreCase(Ring.INNER.toString()) || role.equalsIgnoreCase(Ring.OUTER.toString()))) { switch (memberToRemove.getType()) { case LINE: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Line) memberToRemove).asPolyLine())); break; case AREA: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Area) memberToRemove).asPolygon())); break; case EDGE: this.removedGeometry.add(new JtsPolyLineConverter() .convert(((Edge) memberToRemove).asPolyLine())); break; case NODE: case POINT: case RELATION: default: break; } } } return this; } @Override public CompleteRelation withRemovedRelationIdentifier(final Long relationIdentifier) { this.relationIdentifiers = this.relationIdentifiers.stream() .filter(keepId -> keepId != relationIdentifier.longValue()) .collect(Collectors.toSet()); return this; } private RelationMemberList membersFor(final RelationBean bean) { if (bean == null) { return null; } final List memberList = new ArrayList<>(); for (final RelationBeanItem item : bean) { memberList.add(new RelationMember(item.getRole(), getAtlas().entity(item.getIdentifier(), item.getType()), getIdentifier())); } final RelationMemberList result = new RelationMemberList(memberList); bean.getExplicitlyExcluded().forEach(result::addItemExplicitlyExcluded); return result; } private void updateBounds(final Rectangle bounds) { this.bounds = bounds; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/EmptyAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.SnappedEdge; import org.openstreetmap.atlas.geography.atlas.items.SnappedLineItem; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.gson.JsonObject; /** * Simple Atlas that supports single temporary entities. It does not do anything by design, as all * the {@link CompleteEntity} are self-contained. They just need an Atlas to refer to, so they * comply with the Edge, Node, Area etc. definitions. * * @author matthieun */ public class EmptyAtlas implements Atlas { private static final long serialVersionUID = 5265300513234306056L; @Override public Area area(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable areas() { throw new UnsupportedOperationException(); } @Override public Iterable areas(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable areasCovering(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable areasCovering(final Location location, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable areasIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable areasIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable areasWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public JsonObject asGeoJson() { throw new UnsupportedOperationException(); } @Override public JsonObject asGeoJson(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Rectangle bounds() { throw new UnsupportedOperationException(); } @Override public Edge edge(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable edges() { throw new UnsupportedOperationException(); } @Override public Iterable edges(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable edgesContaining(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable edgesContaining(final Location location, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable edgesIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable edgesIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable edgesWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable entities() { throw new UnsupportedOperationException(); } @Override public Iterable entities(final ItemType type, final Class memberClass) { throw new UnsupportedOperationException(); } @Override public Iterable entities(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable entitiesIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable entitiesIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable entitiesWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable entitiesWithin(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } /** * Note that the {@link AtlasEntity}s returned by this method will technically break the * {@link Located} contract, since they have null bounds. * * @param identifier * the entity identifier * @param type * the entity type * @return the matching {@link AtlasEntity} */ @Override public AtlasEntity entity(final long identifier, final ItemType type) { switch (type) { case NODE: return new CompleteNode(identifier); case EDGE: return new CompleteEdge(identifier); case AREA: return new CompleteArea(identifier); case LINE: return new CompleteLine(identifier); case POINT: return new CompletePoint(identifier); case RELATION: return new CompleteRelation(identifier); default: throw new CoreException("Unknown type {}", type); } } @Override public Iterable getGeoJsonObjects() { throw new UnsupportedOperationException(); } @Override public JsonObject getGeoJsonProperties() { throw new UnsupportedOperationException(); } @Override public UUID getIdentifier() { throw new UnsupportedOperationException(); } @Override public String getName() { return "EmptyAtlas"; } @Override public Iterable items() { throw new UnsupportedOperationException(); } @Override public Iterable items(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable itemsContaining(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable itemsContaining(final Location location, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable itemsIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable itemsIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable itemsWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterator iterator() { throw new UnsupportedOperationException(); } @Override public Line line(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable lineItems() { throw new UnsupportedOperationException(); } @Override public Iterable lineItems(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable lineItemsContaining(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable lineItemsContaining(final Location location, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable lineItemsIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable lineItemsIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable lineItemsWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable lines() { throw new UnsupportedOperationException(); } @Override public Iterable lines(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable linesContaining(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable linesContaining(final Location location, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable linesIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable linesIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable linesWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable locationItems() { throw new UnsupportedOperationException(); } @Override public Iterable locationItems(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable locationItemsWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable locationItemsWithin(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public AtlasMetaData metaData() { throw new UnsupportedOperationException(); } @Override public Node node(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable nodes() { throw new UnsupportedOperationException(); } @Override public Iterable nodes(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable nodesAt(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable nodesWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable nodesWithin(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public long numberOfAreas() { throw new UnsupportedOperationException(); } @Override public long numberOfEdges() { throw new UnsupportedOperationException(); } @Override public long numberOfLines() { throw new UnsupportedOperationException(); } @Override public long numberOfNodes() { throw new UnsupportedOperationException(); } @Override public long numberOfPoints() { throw new UnsupportedOperationException(); } @Override public long numberOfRelations() { throw new UnsupportedOperationException(); } @Override public Point point(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable points() { throw new UnsupportedOperationException(); } @Override public Iterable points(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable pointsAt(final Location location) { throw new UnsupportedOperationException(); } @Override public Iterable pointsWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable pointsWithin(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Relation relation(final long identifier) { throw new UnsupportedOperationException(); } @Override public Iterable relations() { throw new UnsupportedOperationException(); } @Override public Iterable relations(final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable relationsLowerOrderFirst() { throw new UnsupportedOperationException(); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public Iterable relationsWithEntitiesWithin(final GeometricSurface surface) { throw new UnsupportedOperationException(); } @Override public void save(final WritableResource writableResource) { throw new UnsupportedOperationException(); } @Override public void saveAsGeoJson(final WritableResource resource) { throw new UnsupportedOperationException(); } @Override public void saveAsGeoJson(final WritableResource resource, final Predicate matcher) { throw new UnsupportedOperationException(); } @Override public void saveAsLineDelimitedGeoJsonFeatures(final WritableResource resource, final BiConsumer jsonMutator) { throw new UnsupportedOperationException(); } @Override public void saveAsLineDelimitedGeoJsonFeatures(final WritableResource resource, final Predicate matcher, final BiConsumer jsonMutator) { throw new UnsupportedOperationException(); } @Override public void saveAsList(final WritableResource resource) { throw new UnsupportedOperationException(); } @Override public void saveAsProto(final WritableResource resource) { throw new UnsupportedOperationException(); } @Override public void saveAsText(final WritableResource resource) { throw new UnsupportedOperationException(); } @Override public SnappedEdge snapped(final Location point, final Distance threshold) { throw new UnsupportedOperationException(); } @Override public List snaps(final Location point, final Distance threshold) { throw new UnsupportedOperationException(); } @Override public List snapsLineItem(final Location point, final Distance threshold) { throw new UnsupportedOperationException(); } @Override public Optional subAtlas(final GeometricSurface boundary, final AtlasCutType cutType) { throw new UnsupportedOperationException(); } @Override public Optional subAtlas(final Predicate matcher, final AtlasCutType cutType) { throw new UnsupportedOperationException(); } @Override public String summary() { throw new UnsupportedOperationException(); } @Override public String toStringDetailed() { throw new UnsupportedOperationException(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/PrettifyStringFormat.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; /** * @author lcram */ public enum PrettifyStringFormat { MINIMAL_SINGLE_LINE, MINIMAL_MULTI_LINE; public static final int TRUNCATE_LENGTH = 2000; public static final String TRUNCATE_ELLIPSES = "..."; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/complete/TagChangeDelegate.java ================================================ package org.openstreetmap.atlas.geography.atlas.complete; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.event.TagChangeEvent; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listenable.TagChangeListenable; import org.openstreetmap.atlas.geography.atlas.change.eventhandling.listener.TagChangeListener; /** * Consolidates redundant state and behavior that would have otherwise be repeated across all * {@link CompleteEntity} implementations. * * @author Yazad Khambata */ class TagChangeDelegate implements TagChangeListenable, Serializable { private static final long serialVersionUID = -7015756232511317683L; private final List tagChangeListeners = new ArrayList<>(); static TagChangeDelegate newTagChangeDelegate() { return new TagChangeDelegate(); } protected TagChangeDelegate() { super(); } @Override public void addTagChangeListener(final TagChangeListener tagChangeListener) { Validate.notNull(tagChangeListener, "tagChangeListener is NULL."); synchronized (this.tagChangeListeners) { this.tagChangeListeners.add(tagChangeListener); } } @Override public void fireTagChangeEvent(final TagChangeEvent tagChangeEvent) { Validate.notNull(tagChangeEvent, "tagChangeEvent is EMPTY!"); if (!this.tagChangeListeners.isEmpty()) { synchronized (this.tagChangeListeners) { this.tagChangeListeners.stream().forEach( tagChangeListener -> tagChangeListener.entityChanged(tagChangeEvent)); } } } @Override public void removeTagChangeListeners() { synchronized (this.tagChangeListeners) { this.tagChangeListeners.removeIf(tagChangeListener -> true); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/converters/AtlasDebugTool.java ================================================ package org.openstreetmap.atlas.geography.atlas.converters; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasSlicer; import org.openstreetmap.atlas.geography.atlas.routing.AStarRouter; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.boundary.converters.CountryListTwoWayStringConverter; import org.openstreetmap.atlas.geography.converters.MultiPolygonStringConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Convert an Atlas to GeoJson * * @author matthieun */ public class AtlasDebugTool extends Command { private static final Logger logger = LoggerFactory.getLogger(AtlasDebugTool.class); private static final Switch PBF = new Switch<>("pbf", "The protobuf file", path -> new File(path), Optionality.OPTIONAL); private static final Switch ATLAS = new Switch<>("atlas", "The atlas file", path -> new File(path), Optionality.REQUIRED); private static final Switch GEOJSON = new Switch<>("geojson", "The geojson file", path -> new File(path), Optionality.OPTIONAL); private static final Switch TEXT = new Switch<>("text", "The text file", path -> new File(path), Optionality.OPTIONAL); private static final Switch BOUNDARY = new Switch<>("boundary", "The country boundary file", path -> new java.io.File(path), Optionality.OPTIONAL); private static final Switch COUNTRY = new Switch<>("country", "The country name which will be loaded", name -> name, Optionality.OPTIONAL); private static final Switch> ROUTE = new Switch<>("route", "The lat,lon:lat,lon representing a start and end points to get a route", value -> { final StringList split = StringList.split(value, ":"); final List result = new ArrayList<>(); result.add(Location.forString(split.get(0))); result.add(Location.forString(split.get(1))); return result; }, Optionality.OPTIONAL); private static final Switch BOUND = new Switch<>("bound", "Data will be loaded only in this bounding box", value -> Rectangle.forString(value)); private static final Switch MULTIPOLYGON = new Switch<>("multipolygon", "Data will be loaded only in this multipolygon", value -> new MultiPolygonStringConverter().convert(value)); public static void main(final String[] args) { new AtlasDebugTool().run(args); } @Override protected int onRun(final CommandMap command) { final File pbf = (File) command.get(PBF); final File atlasFile = (File) command.get(ATLAS); final File geojson = (File) command.get(GEOJSON); final File text = (File) command.get(TEXT); final java.io.File boundaryFile = (java.io.File) command.get(BOUNDARY); final String country = (String) command.get(COUNTRY); final Rectangle bound = (Rectangle) command.get(BOUND); final MultiPolygon inputMultipolygon = (MultiPolygon) command.get(MULTIPOLYGON); @SuppressWarnings("unchecked") final List startEndRoute = (List) command.get(ROUTE); Atlas atlas; if (pbf != null && pbf.exists()) { final AtlasLoadingOption option; MultiPolygon multiPolygon = MultiPolygon.forPolygon(Rectangle.MAXIMUM); if (boundaryFile != null) { final CountryBoundaryMap boundaryMap = CountryBoundaryMap .fromShapeFile(boundaryFile); option = AtlasLoadingOption.createOptionWithAllEnabled(boundaryMap); if (country != null) { if (new CountryListTwoWayStringConverter().convert(country).size() == 1) { multiPolygon = new JtsPolygonToMultiPolygonConverter() .convert(boundaryMap.countryBoundary(country).get(0)); } option.setCountryCode(country); } } else { option = AtlasLoadingOption.createOptionWithNoSlicing(); } if (bound != null) { multiPolygon = MultiPolygon.forPolygon(bound); } if (inputMultipolygon != null) { multiPolygon = inputMultipolygon; } atlas = new RawAtlasGenerator(pbf, option, multiPolygon).build(); if (option.isCountrySlicing()) { atlas = new RawAtlasSlicer(option, atlas).slice(); } atlas = new AtlasSectionProcessor(atlas, option).run(); atlas.save(atlasFile); } else if (atlasFile != null && atlasFile.exists()) { atlas = new AtlasResourceLoader().load(atlasFile); } else { logger.error("Must have at least one source, -pbf or -atlas"); atlas = null; System.exit(1); } logger.info("Loaded {}", atlas.summary()); if (geojson != null) { atlas.saveAsGeoJson(geojson); } if (text != null) { atlas.saveAsText(text); } if (startEndRoute != null) { logger.info("Route between {} and {} = {}", startEndRoute.get(0), startEndRoute.get(1), AStarRouter.dijkstra(atlas, Distance.TEN_MILES).route(startEndRoute.get(0), startEndRoute.get(1))); } return 0; } @Override protected SwitchList switches() { return new SwitchList().with(ATLAS, GEOJSON, TEXT, PBF, BOUNDARY, COUNTRY, ROUTE, BOUND, MULTIPOLYGON); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/delta/AtlasDelta.java ================================================ package org.openstreetmap.atlas.geography.atlas.delta; import java.io.Serializable; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.delta.Diff.DiffReason; import org.openstreetmap.atlas.geography.atlas.delta.Diff.DiffType; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.matching.PolyLineRoute; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.statistic.storeless.CounterWithStatistic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Difference between two {@link Atlas}. * * @author matthieun */ public class AtlasDelta implements Serializable { private static final long serialVersionUID = 1189641317938152158L; private static final Logger logger = LoggerFactory.getLogger(AtlasDelta.class); private static final int COUNTER_REPORT = 100_000; private final Atlas before; private final Atlas after; private final SortedSet differences; private final boolean withGeometryMatching; private final transient CounterWithStatistic counter; public AtlasDelta(final Atlas before, final Atlas after) { this(before, after, false); } public AtlasDelta(final Atlas before, final Atlas after, final boolean withGeometryMatching) { this.before = before; this.after = after; this.differences = new TreeSet<>(); this.counter = new CounterWithStatistic(logger, COUNTER_REPORT, "Processed"); this.withGeometryMatching = withGeometryMatching; } public AtlasDelta generate() { // Check for removed identifiers logger.info("Looking for removed items."); for (final AtlasEntity entity : this.before) { this.counter.increment(); if (entity.getType().entityForIdentifier(this.after, entity.getIdentifier()) == null && (!(entity instanceof Edge) || !hasGoodMatch((Edge) entity, this.after))) { this.differences.add(new Diff(entity.getType(), DiffType.REMOVED, DiffReason.REMOVED, this.before, this.after, entity.getIdentifier())); } } // Check for added identifiers logger.info("Looking for added items."); for (final AtlasEntity entity : this.after) { this.counter.increment(); if (entity.getType().entityForIdentifier(this.before, entity.getIdentifier()) == null && (!(entity instanceof Edge) || !hasGoodMatch((Edge) entity, this.before))) { this.differences.add(new Diff(entity.getType(), DiffType.ADDED, DiffReason.ADDED, this.before, this.after, entity.getIdentifier())); } } logger.info("Looking for changed items."); for (final AtlasEntity baseEntity : this.before) { this.counter.increment(); final long identifier = baseEntity.getIdentifier(); final AtlasEntity alterEntity = baseEntity.getType().entityForIdentifier(this.after, baseEntity.getIdentifier()); // Look only at entities that are in both Atlas. if (alterEntity != null) { // Entity Tags & Entity's Relations first if (!baseEntity.getTags().equals(alterEntity.getTags())) { this.differences.add(new Diff(baseEntity.getType(), DiffType.CHANGED, DiffReason.TAGS, this.before, this.after, identifier)); } else if (differentInRelation(baseEntity, alterEntity)) { this.differences.add(new Diff(baseEntity.getType(), DiffType.CHANGED, DiffReason.RELATION_MEMBER, this.before, this.after, identifier)); } else if (baseEntity instanceof Node) { if (differentNodes((Node) baseEntity, (Node) alterEntity)) { this.differences.add(new Diff(ItemType.NODE, DiffType.CHANGED, DiffReason.GEOMETRY_OR_TOPOLOGY, this.before, this.after, identifier)); } } else if (baseEntity instanceof Edge) { if (differentEdges((Edge) baseEntity, (Edge) alterEntity)) { this.differences.add(new Diff(ItemType.EDGE, DiffType.CHANGED, DiffReason.GEOMETRY_OR_TOPOLOGY, this.before, this.after, identifier)); } } else if (baseEntity instanceof Area) { if (differentAreas((Area) baseEntity, (Area) alterEntity)) { this.differences.add(new Diff(ItemType.AREA, DiffType.CHANGED, DiffReason.GEOMETRY_OR_TOPOLOGY, this.before, this.after, identifier)); } } else if (baseEntity instanceof Line) { if (differentLines((Line) baseEntity, (Line) alterEntity)) { this.differences.add(new Diff(ItemType.LINE, DiffType.CHANGED, DiffReason.GEOMETRY_OR_TOPOLOGY, this.before, this.after, identifier)); } } else if (baseEntity instanceof Point) { if (differentPoints((Point) baseEntity, (Point) alterEntity)) { this.differences.add(new Diff(ItemType.POINT, DiffType.CHANGED, DiffReason.GEOMETRY_OR_TOPOLOGY, this.before, this.after, identifier)); } } else if (baseEntity instanceof Relation) { if (differentRelations((Relation) baseEntity, (Relation) alterEntity)) { this.differences.add(new Diff(ItemType.RELATION, DiffType.CHANGED, DiffReason.RELATION_TOPOLOGY, this.before, this.after, identifier)); } } } } this.counter.summary(); return this; } public Atlas getAfter() { return this.after; } public Atlas getBefore() { return this.before; } public SortedSet getDifferences() { return this.differences; } /** * Similar to the regular {@link AtlasDelta#toString}, but attempts to make the string more * friendly to diff viewing. * * @return the diff string */ public String toDiffViewFriendlyString() { return Diff.toDiffViewFriendlyString(this.differences); } public String toGeoJson() { return Diff.toGeoJson(this.differences); } public String toGeoJson(final Predicate filter) { return Diff.toGeoJson(this.differences, filter); } public String toRelationsGeoJson() { return Diff.toRelationsGeoJson(this.differences); } public String toRelationsGeoJson(final Predicate filter) { return Diff.toRelationsGeoJson(this.differences, filter); } @Override public String toString() { return Diff.toString(this.differences); } private boolean differentAreas(final Area baseArea, final Area alterArea) { try { if (!baseArea.asPolygon().equals(alterArea.asPolygon())) { return true; } return false; } catch (final Exception e) { throw new CoreException("Unable to compare areas {} and {}", baseArea, alterArea, e); } } private boolean differentEdgeSet(final SortedSet baseEdges, final SortedSet alterEdges) { final boolean differentEdgeSetBasic = differentEdgeSetBasic(baseEdges, alterEdges); final boolean differentEdgeSetWithMatch = differentEdgeSetWithMatch(baseEdges, alterEdges); return differentEdgeSetBasic && differentEdgeSetWithMatch; } private boolean differentEdgeSetBasic(final SortedSet baseEdges, final SortedSet alterEdges) { if (baseEdges.size() != alterEdges.size()) { return true; } final Iterator baseInEdgeIterator = baseEdges.iterator(); final Iterator alterInEdgeIterator = alterEdges.iterator(); for (int i = 0; i < baseEdges.size(); i++) { final Edge baseInEdge = baseInEdgeIterator.next(); final Edge alterInEdge = alterInEdgeIterator.next(); if (baseInEdge.getIdentifier() != alterInEdge.getIdentifier()) { return true; } } return false; } private boolean differentEdgeSetWithMatch(final Set baseEdges, final Set alterEdges) { if (baseEdges.isEmpty() && alterEdges.isEmpty()) { return false; } boolean baseToAlterResult = baseEdges.isEmpty(); for (final Edge edge : baseEdges) { if (alterEdges.isEmpty() || !hasPerfectMatch(edge, alterEdges)) { baseToAlterResult = true; break; } } boolean alterToBaseResult = alterEdges.isEmpty(); for (final Edge edge : alterEdges) { if (baseEdges.isEmpty() || !hasPerfectMatch(edge, baseEdges)) { alterToBaseResult = true; break; } } return baseToAlterResult && alterToBaseResult; } private boolean differentEdges(final Edge baseEdge, final Edge alterEdge) { try { boolean result = false; if (!baseEdge.asPolyLine().equals(alterEdge.asPolyLine())) { result = true; } if (!result && baseEdge.start().getIdentifier() != alterEdge.start().getIdentifier()) { result = true; } if (!result && baseEdge.end().getIdentifier() != alterEdge.end().getIdentifier()) { result = true; } if (result) { // Make sure that there is not way to find a match with the other polylines result = !hasGoodMatch(baseEdge, alterEdge.getAtlas()); } return result; } catch (final Exception e) { throw new CoreException("Unable to compare edges {} and {}", baseEdge, alterEdge, e); } } private boolean differentInRelation(final AtlasEntity baseEntity, final AtlasEntity alterEntity) { try { final Set baseRelations = baseEntity.relations(); final Set alterRelations = alterEntity.relations(); if (baseRelations.size() != alterRelations.size()) { return true; } for (final Relation baseRelation : baseRelations) { Relation alterRelation = null; for (final Relation alterRelationCandidate : alterRelations) { if (alterRelationCandidate.getIdentifier() == baseRelation.getIdentifier()) { alterRelation = alterRelationCandidate; break; } } if (alterRelation == null) { // The two relation sets are different return true; } // Index of the member in the Relation's member list int baseIndex = -1; int alterIndex = -1; final RelationMemberList baseMembers = baseRelation.members(); final RelationMemberList alterMembers = alterRelation.members(); for (int j = 0; j < baseMembers.size(); j++) { final RelationMember baseMember = baseMembers.get(j); if (baseMember.getEntity().getIdentifier() == baseEntity.getIdentifier()) { baseIndex = j; } } for (int j = 0; j < alterMembers.size(); j++) { final RelationMember alterMember = alterMembers.get(j); if (alterMember.getEntity().getIdentifier() == baseEntity.getIdentifier()) { alterIndex = j; } } if (baseIndex < 0 || alterIndex < 0) { throw new CoreException("Corrupted Atlas dataset."); } if (baseIndex != alterIndex) { // Order changed return true; } if (!baseMembers.get(baseIndex).getRole() .equals(alterMembers.get(alterIndex).getRole())) { // Role changed return true; } if (baseMembers.get(baseIndex).getEntity().getType() != alterMembers.get(alterIndex) .getEntity().getType()) { // Type changed return true; } } return false; } catch (final Exception e) { throw new CoreException("Unable to compare relations for {} and {}", baseEntity, alterEntity, e); } } private boolean differentLines(final Line baseLine, final Line alterLine) { try { if (!baseLine.asPolyLine().equals(alterLine.asPolyLine())) { return true; } return false; } catch (final Exception e) { throw new CoreException("Unable to compare line geometries for {} and {}", baseLine, alterLine, e); } } private boolean differentNodes(final Node baseNode, final Node alterNode) { try { if (!baseNode.getLocation().equals(alterNode.getLocation())) { return true; } if (differentEdgeSet(baseNode.inEdges(), alterNode.inEdges())) { return true; } if (differentEdgeSet(baseNode.outEdges(), alterNode.outEdges())) { return true; } return false; } catch (final Exception e) { throw new CoreException("Unable to compare nodes {} and {}", baseNode, alterNode, e); } } private boolean differentPoints(final Point basePoint, final Point alterPoint) { try { if (!basePoint.getLocation().equals(alterPoint.getLocation())) { return true; } return false; } catch (final Exception e) { throw new CoreException("Unable to compare points {} and {}", basePoint, alterPoint, e); } } private boolean differentRelationMemberListsWithMatch(final RelationMemberList baseMembers, final RelationMemberList alterMembers) { final SortedSet baseEdges = Iterables.stream(baseMembers) .map(member -> member.getEntity()).filter(entity -> entity instanceof Edge) .map(entity -> (Edge) entity).collectToSortedSet(); final SortedSet alterEdges = Iterables.stream(alterMembers) .map(member -> member.getEntity()).filter(entity -> entity instanceof Edge) .map(entity -> (Edge) entity).collectToSortedSet(); return differentEdgeSet(baseEdges, alterEdges); } private boolean differentRelations(final Relation baseRelation, final Relation alterRelation) { try { final RelationMemberList baseMembers = baseRelation.members(); final RelationMemberList alterMembers = alterRelation.members(); return !baseMembers.equals(alterMembers) && !differentRelationMemberListsWithMatch(baseMembers, alterMembers); } catch (final Exception e) { throw new CoreException("Unable to compare relations {} and {}", baseRelation, alterRelation, e); } } private boolean hasGoodMatch(final Edge edge, final Atlas other) { if (this.withGeometryMatching) { final Rectangle bounds = edge.bounds(); return hasPerfectMatch(edge, other.edgesIntersecting(bounds, otherEdge -> edge.getOsmIdentifier() == otherEdge.getOsmIdentifier())); } return false; } private boolean hasPerfectMatch(final Edge edge, final Iterable otherEdges) { if (this.withGeometryMatching) { final PolyLine source = edge.asPolyLine(); final List candidates = Iterables.stream(otherEdges).map(Edge::asPolyLine) .collectToList(); final Optional match = source.costDistanceToOneWay(candidates) .match(Distance.ZERO); if (match.isPresent() && match.get().getCost().isLessThanOrEqualTo(Distance.ZERO)) { // The edge was probably split by way sectioning without changing itself. logger.trace("Edge {} from {} has no equal member but found a match with no cost.", edge, edge.getAtlas().getName()); return true; } } return false; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/delta/AtlasDeltaGenerator.java ================================================ package org.openstreetmap.atlas.geography.atlas.delta; import static org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader.HAS_ATLAS_EXTENSION; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.threads.Pool; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This CLI will allow you to point to two atlases and generate a diff of the two. You can point to * single atlas files or directories of atlas shards. If you are diffing shards for both your input * and your output, we will process diffs for each shard individually in parallel. * * @author matthieun * @author hallahan */ public class AtlasDeltaGenerator extends Command { private static final int DEFAULT_THREADS = 8; private static final int COMMAND_LINE_USAGE_ERROR_EXIT = 64; private static final Switch BEFORE_SWITCH = new Switch<>("before", "The before atlas directory or file from which to delta.", Paths::get, Optionality.REQUIRED); private static final Switch AFTER_SWITCH = new Switch<>("after", "The after atlas directory or file that the before atlas deltas to.", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_DIRECTORY_SWITCH = new Switch<>("outputDirectory", "The path of the output directory.", Paths::get, Optionality.REQUIRED); private static final Switch THREADS_SWITCH = new Switch<>("threads", "The number of threads to work on processing atlas shards.", Integer::valueOf, Optionality.OPTIONAL, String.valueOf(DEFAULT_THREADS)); private static final AtlasResourceLoader ATLAS_RESOURCE_LOADER = new AtlasResourceLoader(); private final Logger logger; /** * The size of the thread pool for shard-by-shard parallel processing. */ private int threads = DEFAULT_THREADS; public static void main(final String[] args) { new AtlasDeltaGenerator(LoggerFactory.getLogger(AtlasDeltaGenerator.class)).run(args); } private static List fetchAtlasFilesInDirectory(final Path directory) { return new File(directory.toFile()).listFilesRecursively().stream() .filter(HAS_ATLAS_EXTENSION).collect(Collectors.toList()); } public AtlasDeltaGenerator(final Logger logger) { this.logger = logger; } @Override protected int onRun(final CommandMap command) { final Path before = (Path) command.get("before"); final Path after = (Path) command.get("after"); final Path outputDirectory = (Path) command.get("outputDirectory"); this.threads = (Integer) command.get("threads"); run(before, after, outputDirectory); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(BEFORE_SWITCH, AFTER_SWITCH, OUTPUT_DIRECTORY_SWITCH, THREADS_SWITCH); } private void compare(final Atlas beforeAtlas, final Atlas afterAtlas, final Path outputDirectory) { final String name = FilenameUtils.removeExtension(beforeAtlas.getName()); final AtlasDelta delta = new AtlasDelta(beforeAtlas, afterAtlas).generate(); final String text = delta.toDiffViewFriendlyString(); final File textFile = new File( outputDirectory.resolve(name + FileSuffix.TEXT.toString()).toFile()); textFile.writeAndClose(text); this.logger.info("Saved text file {}", textFile); final String geoJson = delta.toGeoJson(); final File geoJsonFile = new File( outputDirectory.resolve(name + FileSuffix.GEO_JSON.toString()).toFile()); geoJsonFile.writeAndClose(geoJson); this.logger.info("Saved GeoJSON file {}", geoJsonFile); final String relationsGeoJson = delta.toRelationsGeoJson(); final String relationsGeoJsonFileName = name + "_relations" + FileSuffix.GEO_JSON.toString(); final File relationsGeoJsonFile = new File( outputDirectory.resolve(relationsGeoJsonFileName).toFile()); relationsGeoJsonFile.writeAndClose(relationsGeoJson); this.logger.info("Saved Relations GeoJSON file {}", relationsGeoJsonFile); } private void compareShardByShard(final Path before, final Path after, final Path outputDirectory) { final List afterShardFiles = fetchAtlasFilesInDirectory(after); afterShardFiles.parallelStream().forEach(afterShardFile -> { final Path beforeShardPath = before.resolve(afterShardFile.getName()); final Atlas beforeAtlas = load(beforeShardPath); final Atlas afterAtlas = ATLAS_RESOURCE_LOADER.load(afterShardFile); compare(beforeAtlas, afterAtlas, outputDirectory); }); } /** * Load a multi atlas if directory, otherwise load single atlas. * * @param path * An atlas shard directory or a single atlas. * @return An atlas object. */ private Atlas load(final Path path) { return ATLAS_RESOURCE_LOADER.load(new File(path.toFile())); } private void run(final Path before, final Path after, final Path outputDirectory) { final Time time = Time.now(); this.logger.info("Comparing {} and {}", before, after); // If the after is a directory, we want to diff the individual shards in parallel. if (after.toFile().isDirectory()) { // You need to have the before dir be a dir of shards too for this to work. if (!before.toFile().isDirectory()) { this.logger.error( "Your -before parameter must point to a directory of atlas shards if " + "you want to compare shard by shard with an -after directory also of shards!"); System.exit(COMMAND_LINE_USAGE_ERROR_EXIT); } // Execute in a pool of threads so we limit how many atlases get loaded in parallel. try (Pool pool = new Pool(this.threads, "atlas-diff-worker")) { pool.queue(() -> this.compareShardByShard(before, after, outputDirectory)); } } // Otherwise, we can do a normal compare where we look at 2 atlases or input shards with a // single output. else { final Atlas beforeAtlas = load(before); final Atlas afterAtlas = load(after); compare(beforeAtlas, afterAtlas, outputDirectory); } this.logger.info("AtlasDeltaGenerator complete. Total time: {}.", time.elapsedSince()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/delta/Diff.java ================================================ package org.openstreetmap.atlas.geography.atlas.delta; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; /** * A single Atlas diff * * @author matthieun */ public class Diff implements Comparable, Serializable { /** * @author matthieun */ public enum DiffReason { ADDED, REMOVED, TAGS, GEOMETRY_OR_TOPOLOGY, RELATION_MEMBER, RELATION_TOPOLOGY } /** * @author matthieun */ public enum DiffType { ADDED, CHANGED, REMOVED } private static final long serialVersionUID = -1798331824716201841L; private final ItemType itemType; private final DiffType diffType; private final DiffReason diffReason; private final Atlas before; private final Atlas after; private final long identifier; /** * Similar to the regular toString, but attempts to make the diff string more friendly to human * readers. * * @param diffs * An {@link Iterable} of {@link Diff} * @return the human readable diff string */ public static String toDiffViewFriendlyString(final Iterable diffs) { final String newLine = System.getProperty("line.separator"); final StringBuilder builder = new StringBuilder(); builder.append("Diffset {"); final StringList list = new StringList(); for (final Diff diff : diffs) { list.add(newLine + diff.toDiffViewFriendlyString()); } builder.append(list.join(newLine)); builder.append(newLine + "}"); return builder.toString(); } /** * @param diffs * An {@link Iterable} of {@link Diff} * @return A GeoJSON String representation of all the {@link Diff} items in the {@link Iterable} */ public static String toGeoJson(final Iterable diffs) { return toGeoJson(diffs, val -> true); } /** * @param diffs * An {@link Iterable} of {@link Diff} * @param filter * The filter to apply to the diff * @return A GeoJSON String representation of all the {@link Diff} items in the {@link Iterable} * , which match the filter. */ public static String toGeoJson(final Iterable diffs, final Predicate filter) { return new GeoJsonBuilder().create( Iterables.stream(diffs).filter(diff -> diff.getItemType() != ItemType.RELATION) .filter(filter).flatMap(Diff::processDiff).collect()) .jsonObject().toString(); } /** * @param diffs * An {@link Iterable} of {@link Diff} * @return A GeoJSON String representation of all the {@link AtlasItem}s that are a * {@link RelationMember} in a {@link Diff} {@link Relation} */ public static String toRelationsGeoJson(final Iterable diffs) { return toRelationsGeoJson(diffs, val -> true); } /** * @param diffs * An {@link Iterable} of {@link Diff} * @param filter * The filter to apply to the diff * @return A GeoJSON String representation of all the {@link AtlasItem}s that are a * {@link RelationMember} in a {@link Diff} {@link Relation}, which match the filter. */ public static String toRelationsGeoJson(final Iterable diffs, final Predicate filter) { return new GeoJsonBuilder().create( Iterables.stream(diffs).filter(diff -> diff.getItemType() == ItemType.RELATION) .filter(filter).flatMap(Diff::processDiff).collect()) .jsonObject().toString(); } /** * @param diffs * An {@link Iterable} of {@link Diff} * @return A String representation of all the {@link Diff} items in the {@link Iterable} */ public static String toString(final Iterable diffs) { final StringBuilder builder = new StringBuilder(); builder.append("[Diffs: "); final StringList list = new StringList(); for (final Diff diff : diffs) { list.add("\n\t" + diff.toString()); } builder.append(list.join(", ")); builder.append("\n]"); return builder.toString(); } private static List processDiff(final Diff diff) { final List items = new ArrayList<>(); // Process the BEFORE entity. Note that it is null if it is ADDED. final AtlasEntity beforeEntity = diff.getBeforeEntity(); processDiffEntity(diff, beforeEntity, items, "BEFORE"); // Process the AFTER entity. Note that it is null if it is REMOVED. final AtlasEntity afterEntity = diff.getAfterEntity(); processDiffEntity(diff, afterEntity, items, "AFTER"); return items; } private static void processDiffEntity(final Diff diff, final AtlasEntity entity, final List items, final String diffStage) { // Depending on if it is a before or after for a given diff type, an entity may be null. if (entity != null) { final Map tags = entity.getTags(); tags.put("diff", diffStage); tags.put("diff:type", diff.getDiffType().name()); tags.put("diff:reason", diff.getDiffReason().name()); if (diff.getItemType() == ItemType.RELATION) { items.addAll(processRelationForGeoJson((Relation) entity, tags)); } else { items.add(new LocationIterableProperties(((AtlasItem) entity).getRawGeometry(), tags)); } } } private static List processRelationForGeoJson( final Relation relation, final Map parentTags) { final Map relationTags = relation.getTags(); final Map modifiedRelationTags = new HashMap<>(parentTags); for (final String key : relationTags.keySet()) { modifiedRelationTags.put("[REL_ID:" + relation.getIdentifier() + "]" + key, relationTags.get(key)); } final List result = new ArrayList<>(); for (final RelationMember member : relation.members()) { if (member.getEntity() instanceof Relation) { final Relation subRelation = (Relation) member.getEntity(); result.addAll(processRelationForGeoJson(subRelation, modifiedRelationTags)); } else { final AtlasItem item = (AtlasItem) member.getEntity(); final Map modifiedTags = item.getTags(); modifiedTags.putAll(modifiedRelationTags); result.add(new LocationIterableProperties(item.getRawGeometry(), modifiedTags)); } } return result; } /** * Construct * * @param itemType * The type of the entity that this {@link Diff} represents * @param diffType * The type of this {@link Diff}, among "ADDED", "REMOVED" and "CHANGED" * @param diffReason * The reason of this {@link Diff} * @param before * The before {@link Atlas}, i.e. the older one * @param after * The after {@link Atlas}, i.e. the newer one * @param identifier * The identifier if the entity that this {@link Diff} represents. */ public Diff(final ItemType itemType, final DiffType diffType, final DiffReason diffReason, final Atlas before, final Atlas after, final long identifier) { this.itemType = itemType; this.diffType = diffType; this.diffReason = diffReason; this.before = before; this.after = after; this.identifier = identifier; } @Override public int compareTo(final Diff other) { if (this.getDiffType() != other.getDiffType()) { return this.getDiffType().compareTo(other.getDiffType()); } if (this.getItemType() != other.getItemType()) { return this.itemType.compareTo(other.getItemType()); } final long deltaIdentifier = this.getIdentifier() - other.getIdentifier(); return deltaIdentifier > 0 ? 1 : deltaIdentifier == 0 ? 0 : -1; } /** * @return The after {@link Atlas}, i.e. the newer one */ public Atlas getAfter() { return this.after; } /** * @return The entity this {@link Diff} represents in the newer Atlas. null if this Diff is of * type "REMOVED" */ public AtlasEntity getAfterEntity() { return this.itemType.entityForIdentifier(this.after, this.identifier); } /** * @return The before {@link Atlas}, i.e. the older one */ public Atlas getBefore() { return this.before; } /** * @return The entity this {@link Diff} represents in the older Atlas. null if this Diff is of * type "ADDED" */ public AtlasEntity getBeforeEntity() { return this.itemType.entityForIdentifier(this.before, this.identifier); } /** * @return The reason for this diff */ public DiffReason getDiffReason() { return this.diffReason; } /** * @return The type of this {@link Diff}, among "ADDED", "REMOVED" and "CHANGED" */ public DiffType getDiffType() { return this.diffType; } /** * @return The identifier of the entity that this {@link Diff} represents. */ public long getIdentifier() { return this.identifier; } /** * @return The type of the entity that this {@link Diff} represents */ public ItemType getItemType() { return this.itemType; } /** * @return True if this diff is of type ADDED */ public boolean isAdded() { return DiffType.ADDED == this.getDiffType(); } /** * @return True if this diff is of type CHANGED */ public boolean isChanged() { return DiffType.CHANGED == this.getDiffType(); } /** * @return True if this diff is of type REMOVED */ public boolean isRemoved() { return DiffType.REMOVED == this.getDiffType(); } /** * Similar to the regular toString method. However, this version returns the {@link Diff} with * an attempt at being more friendly to diff readouts. * * @return the string */ public String toDiffViewFriendlyString() { final String newLine = System.getProperty("line.separator"); final StringBuilder builder = new StringBuilder(); builder.append("Diff {"); builder.append(newLine); builder.append("diffType: " + this.diffType); builder.append(newLine); builder.append("diffReason: " + this.diffReason); builder.append(newLine); builder.append("Entity = " + this.itemType); builder.append(newLine); builder.append("ID = " + this.identifier); builder.append(newLine); if (this.getBeforeEntity() != null) { builder.append(this.getBeforeEntity().toDiffViewFriendlyString()); } else { builder.append("null"); } builder.append(newLine); builder.append(" -> "); builder.append(newLine); if (this.getAfterEntity() != null) { builder.append(this.getAfterEntity().toDiffViewFriendlyString()); } else { builder.append("null"); } builder.append(newLine); builder.append("}"); return builder.toString(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[Diff: "); builder.append(this.diffType); builder.append(", Entity = {"); builder.append(this.itemType); builder.append(", ID = "); builder.append(this.identifier); builder.append(", "); builder.append(this.getBeforeEntity()); builder.append(" -> "); builder.append(this.getAfterEntity()); builder.append("}]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/delta/README.md ================================================ # Atlas Delta ## Use Cases ### All * An Entity's tag is added * An Entity's tag is removed * An Entity's tag is changed ### Node * A Node is added (There is a new identifier) * A Node is deleted (The identifier has disappeared) * A Node is moved (The Location is different for the same identifier) * A Node's in-Edge is added * A Node's in-Edge is removed * A Node's out-Edge is added * A Node's out-Edge is removed * A Node is added to a Relation * A Node is removed from a Relation * A Node's order is changed inside a Relation ### Edge * An Edge is added (There is a new identifier) * An Edge is deleted (The identifier has disappeared) * An Edge is moved (The PolyLine is different for the same identifier) * An Edge's start Node is different * An Edge's end Node is different * An Edge is added to a Relation * An Edge is removed from a Relation * An Edge's order is changed inside a Relation ### Area * An Area is added (There is a new identifier) * An Area is deleted (The identifier has disappeared) * An Area is moved (The Polygon is different for the same identifier) * An Area is added to a Relation * An Area is removed from a Relation * An Area's order is changed inside a Relation ### Line * A Line is added (There is a new identifier) * A Line is deleted (The identifier has disappeared) * A Line is moved (The PolyLine is different for the same identifier) * A Line is added to a Relation * A Line is removed from a Relation * A Line's order is changed inside a Relation ### Point * A Point is added (There is a new identifier) * A Point is deleted (The identifier has disappeared) * A Point is moved (The Location is different for the same identifier) * A Point is added to a Relation * A Point is removed from a Relation * A Point's order is changed inside a Relation ### Relation * A Relation is added (There is a new identifier) * A Relation is deleted (The identifier has disappeared) * A Relation is changed: * A Relation member is added (There is a new member identifier) * A Relation member is removed (A member identifier is changed) * A Relation member list order is different * A Relation member is changed: * The role changed (The role String is different) * A Relation is added to a Relation * A Relation is removed from a Relation * A Relation's order is changed inside a Relation ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * @author matthieun */ public class DynamicArea extends Area { private static final long serialVersionUID = 3402097623330654390L; // Not index! private final long identifier; protected DynamicArea(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public Polygon asPolygon() { return subArea().asPolygon(); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return subArea().getTags(); } @Override public Set relations() { return subArea().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private Area subArea() { final Area result = dynamicAtlas().subArea(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.BareAtlas; import org.openstreetmap.atlas.geography.atlas.dynamic.policy.DynamicAtlasPolicy; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * This is not thread safe! *

* An Atlas that is dynamically expanding by loading neighboring shards upon request of its * features. * * @author matthieun */ // NOSONAR here as the parent equals is enough public class DynamicAtlas extends BareAtlas // NOSONAR { private static final long serialVersionUID = -2858997785405677961L; // The current Atlas that will be swapped during expansion. private Atlas current; private final DynamicAtlasExpander expander; /** * @param dynamicAtlasExpansionPolicy * Expansion policy for the dynamic atlas */ public DynamicAtlas(final DynamicAtlasPolicy dynamicAtlasExpansionPolicy) { this.setName("DynamicAtlas(" + dynamicAtlasExpansionPolicy.getInitialShards().stream() .map(Shard::getName).collect(Collectors.toSet()) + ")"); this.expander = new DynamicAtlasExpander(this, dynamicAtlasExpansionPolicy); } @Override public Area area(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subArea(identifier)), this.expander::areaCovered, this::newArea) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable areas() { return this.expander.expand(() -> this.current.areas(), this.expander::areaCovered, this::newArea); } @Override public Iterable areasCovering(final Location location) { return this.expander.expand(() -> this.current.areasCovering(location), this.expander::areaCovered, this::newArea); } @Override public Iterable areasCovering(final Location location, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.areasCovering(location), this.expander::areaCovered, this::newArea), matcher); } @Override public Iterable areasIntersecting(final GeometricSurface surface) { return this.expander.expand(() -> this.current.areasIntersecting(surface), this.expander::areaCovered, this::newArea); } @Override public Iterable areasIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.areasIntersecting(surface), this.expander::areaCovered, this::newArea), matcher); } @Override public Iterable areasWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.areasWithin(surface), this.expander::areaCovered, this::newArea); } @Override public Rectangle bounds() { return this.current.bounds(); } @Override public Edge edge(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subEdge(identifier)), this.expander::lineItemCovered, this::newEdge) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable edges() { return this.expander.expand(() -> this.current.edges(), this.expander::lineItemCovered, this::newEdge); } @Override public Iterable edgesContaining(final Location location) { return this.expander.expand(() -> this.current.edgesContaining(location), this.expander::lineItemCovered, this::newEdge); } @Override public Iterable edgesContaining(final Location location, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.edgesContaining(location), this.expander::lineItemCovered, this::newEdge), matcher); } @Override public Iterable edgesIntersecting(final GeometricSurface surface) { return this.expander.expand(() -> this.current.edgesIntersecting(surface), this.expander::lineItemCovered, this::newEdge); } @Override public Iterable edgesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.edgesIntersecting(surface), this.expander::lineItemCovered, this::newEdge), matcher); } @Override public Iterable edgesWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.edgesWithin(surface), this.expander::lineItemCovered, this::newEdge); } /** * @return All the {@link Atlas}es explored by that {@link DynamicAtlas} */ public Set getAtlasesLoaded() { return this.expander.getLoadedShards().values().stream().filter(Objects::nonNull) .collect(Collectors.toSet()); } /** * @return The number of shards loaded by that {@link DynamicAtlas} at any time. */ public int getNumberOfShardsLoaded() { return getShardsLoaded().size(); } public DynamicAtlasPolicy getPolicy() { return this.expander.getPolicy(); } /** * @return A copy of the {@link Shard} to {@link Atlas} {@link Map} populated by the underlying * {@link DynamicAtlasExpander}, with any null Atlases filtered out */ public Map getShardToAtlasMap() { return new HashMap<>(this.expander.getLoadedShards().entrySet().stream() .filter(entry -> entry.getValue() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } /** * @return All the shards explored by this {@link DynamicAtlas} including the ones that yielded * no Atlas. */ public Set getShardsExplored() { return this.expander.getLoadedShards().keySet(); } /** * @return All the shards explored by that {@link DynamicAtlas} which yielded some non null * Atlas. */ public Set getShardsLoaded() { return this.expander.getLoadedShards().entrySet().stream() .filter(entry -> entry.getValue() != null).map(Entry::getKey) .collect(Collectors.toSet()); } /** * @return The number of times that {@link DynamicAtlas} has (re-)built its {@link MultiAtlas} * underneath. */ public int getTimesMultiAtlasWasBuiltUnderneath() { return this.expander.getTimesMultiAtlasWasBuiltUnderneath(); } @Override public Line line(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subLine(identifier)), this.expander::lineItemCovered, this::newLine) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable lines() { return this.expander.expand(() -> this.current.lines(), this.expander::lineItemCovered, this::newLine); } @Override public Iterable linesContaining(final Location location) { return this.expander.expand(() -> this.current.linesContaining(location), this.expander::lineItemCovered, this::newLine); } @Override public Iterable linesContaining(final Location location, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.linesContaining(location), this.expander::lineItemCovered, this::newLine), matcher); } @Override public Iterable linesIntersecting(final GeometricSurface surface) { return this.expander.expand(() -> this.current.linesIntersecting(surface), this.expander::lineItemCovered, this::newLine); } @Override public Iterable linesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.linesIntersecting(surface), this.expander::lineItemCovered, this::newLine), matcher); } @Override public Iterable linesWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.linesWithin(surface), this.expander::lineItemCovered, this::newLine); } @Override public AtlasMetaData metaData() { return this.current.metaData(); } @Override public Node node(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subNode(identifier)), this.expander::locationItemCovered, this::newNode) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable nodes() { return this.expander.expand(() -> this.current.nodes(), this.expander::locationItemCovered, this::newNode); } @Override public Iterable nodesAt(final Location location) { return this.expander.expand(() -> this.current.nodesAt(location), this.expander::locationItemCovered, this::newNode); } @Override public Iterable nodesWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.nodesWithin(surface), this.expander::locationItemCovered, this::newNode); } @Override public Iterable nodesWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.nodesWithin(surface), this.expander::locationItemCovered, this::newNode), matcher); } @Override public long numberOfAreas() { return this.current.numberOfAreas(); } @Override public long numberOfEdges() { return this.current.numberOfEdges(); } @Override public long numberOfLines() { return this.current.numberOfLines(); } @Override public long numberOfNodes() { return this.current.numberOfNodes(); } @Override public long numberOfPoints() { return this.current.numberOfPoints(); } @Override public long numberOfRelations() { return this.current.numberOfRelations(); } @Override public Point point(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subPoint(identifier)), this.expander::locationItemCovered, this::newPoint) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable points() { return this.expander.expand(() -> this.current.points(), this.expander::locationItemCovered, this::newPoint); } @Override public Iterable pointsAt(final Location location) { return this.expander.expand(() -> this.current.pointsAt(location), this.expander::locationItemCovered, this::newPoint); } @Override public Iterable pointsWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.pointsWithin(surface), this.expander::locationItemCovered, this::newPoint); } @Override public Iterable pointsWithin(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter(this.expander.expand(() -> this.current.pointsWithin(surface), this.expander::locationItemCovered, this::newPoint), matcher); } /** * Do a preemptive load of the {@link DynamicAtlas} as far as the {@link DynamicAtlasPolicy} * allows. *

* In some very specific cases, where the {@link DynamicAtlasPolicy} allows expansion only if * new shards intersect at least one feature that crosses the initial set of shards, it is * possible that expanding only one time misses out some shard candidates. This happens when * some feature intersects the initial shards but does not have any shape point inside any * initial shard. This way, the initial shards do not contain that feature even though they * intersect it. That feature is discovered as we load the neighboring shards which contain that * feature. If that said feature also intersects a third neighboring shard, then that third * neighboring shard becomes eligible for expansion, as that specific feature crosses it and the * initial shards. To work around that case, the preemptive load will do a multi-staged loading. */ public void preemptiveLoad() { this.expander.preemptiveLoad(); } @Override public Relation relation(final long identifier) { final Iterator result = this.expander .expand(() -> Iterables.from(subRelation(identifier)), this.expander::relationCovered, this::newRelation) .iterator(); return result.hasNext() ? result.next() : null; } @Override public Iterable relations() { return this.expander.expand(() -> this.current.relations(), this.expander::relationCovered, this::newRelation); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface) { return this.expander.expand(() -> this.current.relationsWithEntitiesIntersecting(surface), this.expander::relationCovered, this::newRelation); } @Override public Iterable relationsWithEntitiesIntersecting(final GeometricSurface surface, final Predicate matcher) { return Iterables.filter( this.expander.expand(() -> this.current.relationsWithEntitiesIntersecting(surface), this.expander::relationCovered, this::newRelation), matcher); } @Override public Iterable relationsWithEntitiesWithin(final GeometricSurface surface) { return this.expander.expand(() -> this.current.relationsWithEntitiesWithin(surface), this.expander::relationCovered, this::newRelation); } @Override public void save(final WritableResource writableResource) { throw new CoreException("DynamicAtlas cannot be saved"); } protected Area subArea(final long identifier) { return this.current.area(identifier); } protected Edge subEdge(final long identifier) { return this.current.edge(identifier); } protected Line subLine(final long identifier) { return this.current.line(identifier); } protected Node subNode(final long identifier) { return this.current.node(identifier); } protected Point subPoint(final long identifier) { return this.current.point(identifier); } protected Relation subRelation(final long identifier) { return this.current.relation(identifier); } synchronized void swapCurrentAtlas(final Atlas current) { this.current = current; } private DynamicArea newArea(final Area area) { return new DynamicArea(this, area.getIdentifier()); } private DynamicEdge newEdge(final Edge edge) { return new DynamicEdge(this, edge.getIdentifier()); } private DynamicLine newLine(final Line line) { return new DynamicLine(this, line.getIdentifier()); } private DynamicNode newNode(final Node node) { return new DynamicNode(this, node.getIdentifier()); } private DynamicPoint newPoint(final Point point) { return new DynamicPoint(this, point.getIdentifier()); } private DynamicRelation newRelation(final Relation relation) { return new DynamicRelation(this, relation.getIdentifier()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicAtlasExpander.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometryPrintable; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.WktPrintable; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.dynamic.policy.DynamicAtlasPolicy; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StreamIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ class DynamicAtlasExpander { private static final Logger logger = LoggerFactory.getLogger(DynamicAtlasExpander.class); private final DynamicAtlas dynamicAtlas; private Set shardsUsedForCurrent; private final Map loadedShards; private final Function> atlasFetcher; private final Sharding sharding; private final DynamicAtlasPolicy policy; // This is true when the loading of the initial shard has been completed private final boolean initialized; // This is true when, in case of deferred loading, the loading of the shards has been called // (unlocking further automatic loading later) private boolean isAlreadyLoaded = false; private boolean preemptiveLoadDone = false; // Number of times the udnerlying Multi-Atlas has been built. private int timesMultiAtlasWasBuiltUnderneath; private final Set areaCoveredCache; private final Set edgeCoveredCache; private final Set lineCoveredCache; DynamicAtlasExpander(final DynamicAtlas dynamicAtlas, final DynamicAtlasPolicy policy) { this.dynamicAtlas = dynamicAtlas; this.timesMultiAtlasWasBuiltUnderneath = 0; this.sharding = policy.getSharding(); this.loadedShards = new HashMap<>(); this.shardsUsedForCurrent = new HashSet<>(); this.atlasFetcher = policy.getAtlasFetcher(); // Still keep the policy this.policy = policy; this.addNewShards(policy.getInitialShards()); this.initialized = true; // DynamicAtlas always expands, so it is ok to cache the features already covered by shards. this.areaCoveredCache = new HashSet<>(); this.edgeCoveredCache = new HashSet<>(); this.lineCoveredCache = new HashSet<>(); } public DynamicAtlasPolicy getPolicy() { return this.policy; } boolean areaCovered(final Area area) { if (!entityNotCached(area)) { return true; } final Polygon polygon = area.asPolygon(); final MultiPolygon initialShardsBounds = this.policy.getInitialShardsBounds(); if (!this.policy.isExtendIndefinitely() && !(polygon.overlaps(initialShardsBounds) || initialShardsBounds.overlaps(polygon))) { // If the policy is to not extend indefinitely, then assume that the loading is not // necessary. return true; } cacheEntity(area); final Iterable neededShards = this.sharding.shards(polygon); for (final Shard neededShard : neededShards) { if (!this.loadedShards.containsKey(neededShard)) { newPolygon(polygon, area); return false; } } return true; } void buildUnderlyingMultiAtlas() { final Time buildTime = Time.now(); final Set nonNullShards = nonNullShards(); if (this.shardsUsedForCurrent.equals(nonNullShards)) { // Same Multi-Atlas, let's not reload. return; } final List nonNullAtlasShards = getNonNullAtlasShards(); if (!nonNullAtlasShards.isEmpty()) { this.policy.getShardSetChecker().accept(nonNullShards()); if (nonNullAtlasShards.size() == 1) { this.dynamicAtlas.swapCurrentAtlas(nonNullAtlasShards.get(0)); } else { if (logger.isDebugEnabled()) { logger.debug("{}: Loading MultiAtlas with {}", this.dynamicAtlas.getName(), nonNullShards().stream().map(Shard::getName) .collect(Collectors.toList())); } this.dynamicAtlas.swapCurrentAtlas(new MultiAtlas(nonNullAtlasShards)); this.timesMultiAtlasWasBuiltUnderneath++; } this.shardsUsedForCurrent = nonNullShards; if (this.initialized) { this.isAlreadyLoaded = true; } } else { throw new CoreException("Cannot load shards with no data!"); } logger.trace("{}: Built underlying MultiAtlas in {}", this.dynamicAtlas.getName(), buildTime.elapsedSince()); } /** * Expand the Atlas if needed. This method loops through the provided {@link Iterable}, then * checks if each entity found warrants loading another neighboring {@link Shard}. If it does, * it loads all the necessary {@link Shard}s and retries looping through the new * {@link Iterable}. Once everything is included, then the final {@link Iterable} is returned. * * @param entitiesSupplier * The {@link Supplier} of the {@link Iterable} of items that will be called as long * as there are overlaps to new shards. There is a need of a supplier here so that * the {@link Iterable} is re-built every time with the latest Atlas. * @param entityCoveredPredicate * The function that decides if an entity is already covered or not. * @param mapper * What to do with the result. This is to replace the regular items with * DynamicItems. * @param * The object type the returned iterable will return * @param * The original entity type * @return The {@link Iterable} of DynamicItems */ Iterable expand(final Supplier> entitiesSupplier, final Predicate entityCoveredPredicate, final Function mapper) { StreamIterable result = Iterables.stream(entitiesSupplier.get()) .filter(Objects::nonNull); final boolean shouldStopExploring = this.policy.isDeferLoading() && this.preemptiveLoadDone; while (!shouldStopExploring && !entitiesCovered(result, entityCoveredPredicate)) { result = Iterables.stream(entitiesSupplier.get()).filter(Objects::nonNull); } return result.map(mapper).collect(); } Map getLoadedShards() { return this.loadedShards; } /** * @return The number of times that {@link DynamicAtlas} has (re-)built its {@link MultiAtlas} * underneath. */ int getTimesMultiAtlasWasBuiltUnderneath() { return this.timesMultiAtlasWasBuiltUnderneath; } boolean lineItemCovered(final LineItem item) { if (!entityNotCached(item)) { return true; } final PolyLine polyLine = item.asPolyLine(); final MultiPolygon initialShardsBounds = this.policy.getInitialShardsBounds(); if (!this.policy.isExtendIndefinitely() && !initialShardsBounds.overlaps(polyLine) || !this.policy.getAtlasEntitiesToConsiderForExpansion().test(item)) { // If the policy is to not extend indefinitely, then assume that the loading is not // necessary. return true; } cacheEntity(item); final Iterable neededShards = this.sharding.shardsIntersecting(polyLine); for (final Shard neededShard : neededShards) { if (!this.loadedShards.containsKey(neededShard)) { newPolyLine(polyLine, item); return false; } } return true; } boolean locationItemCovered(final LocationItem item) { final Location location = item.getLocation(); final MultiPolygon initialShardsBounds = this.policy.getInitialShardsBounds(); if (!this.policy.isExtendIndefinitely() && !initialShardsBounds.fullyGeometricallyEncloses(location) || !this.policy.getAtlasEntitiesToConsiderForExpansion().test(item)) { // If the policy is to not extend indefinitely, then assume that the loading is not // necessary. return true; } final Iterable neededShards = this.sharding.shardsCovering(location); for (final Shard neededShard : neededShards) { if (!this.loadedShards.containsKey(neededShard)) { newLocation(location, item); return false; } } return true; } /** * Do a preemptive load of the {@link DynamicAtlas} as far as the {@link DynamicAtlasPolicy} * allows. *

* In some very specific cases, where the {@link DynamicAtlasPolicy} allows expansion only if * new shards intersect at least one feature that crosses the initial set of shards, it is * possible that expanding only one time misses out some shard candidates. This happens when * some feature intersects the initial shards but does not have any shape point inside any * initial shard. This way, the initial shards do not contain that feature even though they * intersect it. That feature is discovered as we load the neighboring shards which contain that * feature. If that said feature also intersects a third neighboring shard, then that third * neighboring shard becomes eligible for expansion, as that specific feature crosses it and the * initial shards. To work around that case, the preemptive load will do a multi-staged loading. */ void preemptiveLoad() { if (!this.policy.isDeferLoading()) { logger.warn( "{}: Skipping preemptive loading as it is useful only when the DynamicAtlasPolicy is deferLoading = true.", this.dynamicAtlas.getName()); return; } if (this.preemptiveLoadDone) { return; } // Loop through the entities to find potential shards to add browseForPotentialNewShards(); // Load all the shards into a multiAtlas buildUnderlyingMultiAtlas(); // Record the current list of shards Set currentShards = new HashSet<>(this.loadedShards.keySet()); // Loop through the entities again to find potential shards to add. This can still happen if // a way intersects the initial shard without shapepoints inside the initial shards, and was // revealed by loading a new neighboring shard. At that point, if that way also intersects a // third shard which was not loaded before, that third shard might become now eligible. browseForPotentialNewShards(); browseForPotentialNewShardsFromAggressiveRelations(); // Repeat the same process as long as we find some of those third party shards. while (!this.loadedShards.keySet().equals(currentShards)) { if (logger.isInfoEnabled()) { final Set missingShards = new HashSet<>(this.loadedShards.keySet()); missingShards.removeAll(currentShards); logger.info("{}: Preemptive load found new unexpected 2nd degree shard(s): {}", this.dynamicAtlas.getName(), missingShards.stream().map(Shard::getName).collect(Collectors.toList())); } // Load all the shards into a multiAtlas buildUnderlyingMultiAtlas(); // Record the current list of shards currentShards = new HashSet<>(this.loadedShards.keySet()); // Loop through the entities again to find potential shards to add. browseForPotentialNewShards(); browseForPotentialNewShardsFromAggressiveRelations(); } this.preemptiveLoadDone = true; } boolean relationCovered(final Relation relation) { final Set parentRelationIdentifierTree = new HashSet<>(); parentRelationIdentifierTree.add(relation.getIdentifier()); return relationCoveredInternal(relation, parentRelationIdentifierTree); } private void addNewShardLog(final Shard shard) { if (logger.isInfoEnabled()) { final Atlas loaded = this.loadedShards.get(shard); if (loaded == null) { logger.info("{}: Loading new shard {} found no new Atlas.", this.dynamicAtlas.getName(), shard.getName()); } else { logger.info("{}: Loading new shard {} found a new Atlas {} of size {}", this.dynamicAtlas.getName(), shard.getName(), loaded.getName(), loaded.size()); } } } private void addNewShards(final Iterable shards) { final Set initialNonEmptyLoadedShards = nonNullShards(); for (final Shard shard : shards) { if (!this.loadedShards.containsKey(shard)) { this.loadedShards.put(shard, this.atlasFetcher.apply(shard).orElse(null)); addNewShardLog(shard); } } final List nonNullAtlasShards = getNonNullAtlasShards(); if (!nonNullAtlasShards.isEmpty()) { if (shouldBuildUnderlyingMultiAtlasWhenAddingNewShards(initialNonEmptyLoadedShards)) { // Load the new current atlas only if it is the first time, or it is not the // first time, and the policy is not to defer loading. buildUnderlyingMultiAtlas(); } } else { // There should always be a non-null atlas in that list, coming from the initial Shard. throw new CoreException("{}: There is no data to load for initial shard!", this.dynamicAtlas.getName()); } } private boolean areaCoversInitialShardBounds(final Area area) { return this.policy.getInitialShardsBounds().overlaps(area.asPolygon()); } private void browseForPotentialNewShards() { // Look at regular entities this.dynamicAtlas.entities(); } private void browseForPotentialNewShardsFromAggressiveRelations() { // In case we want to aggressively explore relations, we constrain it to only when the // policy is to defer loading. if (this.policy.isAggressivelyExploreRelations() && this.policy.isDeferLoading()) { // Get all the neighboring shards final Set onlyNeighboringShards = new HashSet<>(); this.loadedShards.keySet().forEach( shard -> this.sharding.neighbors(shard).forEach(onlyNeighboringShards::add)); onlyNeighboringShards.removeAll(this.loadedShards.keySet()); if (logger.isTraceEnabled()) { final Set shardNames = onlyNeighboringShards.stream().map(Shard::getName) .collect(Collectors.toSet()); final String wktCollection = WktPrintable.toWktCollection(onlyNeighboringShards); logger.trace("{}: Aggressively exploring relations in shards {} - {}", this.dynamicAtlas.getName(), shardNames, wktCollection); } // For each of those shards, load the Atlas individually and find the relation and its // members if it is there too. final Set neighboringShardsContainingRelation = neighboringShardsContainingInitialRelation( onlyNeighboringShards); // Add the neighboring shards as new shards to be loaded. if (!neighboringShardsContainingRelation.isEmpty()) { addNewShards(neighboringShardsContainingRelation); } } } private void cacheEntity(final AtlasEntity atlasEntity) { if (atlasEntity instanceof Area) { this.areaCoveredCache.add(atlasEntity.getIdentifier()); } else if (atlasEntity instanceof LineItem) { cacheEntity((LineItem) atlasEntity); } } private void cacheEntity(final LineItem lineItem) { if (lineItem instanceof Edge) { this.edgeCoveredCache.add(lineItem.getIdentifier()); } else { this.lineCoveredCache.add(lineItem.getIdentifier()); } } /** * @param entities * The items to test for full coverage by the current shards * @param entityCoveredPredicate * The function that decides if an entity is already covered or not. * @return False if any of the items is not fully covered by the current shards */ private boolean entitiesCovered(final Iterable entities, final Predicate entityCoveredPredicate) { return Iterables.stream(entities).filter(this::entityNotCached).filter(entity -> { final boolean toConsiderForExpansion = this.policy .getAtlasEntitiesToConsiderForExpansion().test(entity); if (!toConsiderForExpansion) { cacheEntity(entity); } return toConsiderForExpansion; }).allMatch(entityCoveredPredicate); } private boolean entityNotCached(final LineItem lineItem) { if (lineItem instanceof Edge) { return !this.edgeCoveredCache.contains(lineItem.getIdentifier()); } else { return !this.lineCoveredCache.contains(lineItem.getIdentifier()); } } private boolean entityNotCached(final AtlasEntity atlasEntity) { if (atlasEntity instanceof Area) { return !this.areaCoveredCache.contains(atlasEntity.getIdentifier()); } else if (atlasEntity instanceof LineItem) { return entityNotCached((LineItem) atlasEntity); } else { return true; } } private List getNonNullAtlasShards() { return this.loadedShards.values().stream().filter(Objects::nonNull) .collect(Collectors.toList()); } private boolean lineItemCoversInitialShardBounds(final LineItem lineItem) { return this.policy.getInitialShardsBounds().overlaps(lineItem.asPolyLine()); } private boolean loadedShardsfullyGeometricallyEncloseLocation(final Location location) { return Iterables.stream(this.sharding.shardsCovering(location)) .allMatch(this.loadedShards::containsKey); } private boolean loadedShardsfullyGeometricallyEnclosePolyLine(final PolyLine polyLine) { return Iterables.stream(this.sharding.shardsIntersecting(polyLine)) .allMatch(this.loadedShards::containsKey); } private boolean loadedShardsfullyGeometricallyEnclosePolygon(final Polygon polygon) { return Iterables.stream(this.sharding.shards(polygon)) .allMatch(this.loadedShards::containsKey); } private boolean locationItemCoversInitialShardBounds(final LocationItem locationItem) { return this.policy.getInitialShardsBounds() .fullyGeometricallyEncloses(locationItem.getLocation()); } private boolean neighboringAtlasContainingInitialRelation(final Atlas atlas) { for (final Relation newRelation : atlas.relations()) { final Relation currentRelation = this.dynamicAtlas .subRelation(newRelation.getIdentifier()); if (currentRelation != null && this.policy.getAtlasEntitiesToConsiderForExpansion().test(currentRelation) && relationCoversInitialShardBounds(currentRelation)) { final RelationBean newMembers = newRelation.members().asBean(); final RelationBean currentMembers = currentRelation.members().asBean(); for (final RelationBeanItem newMember : newMembers) { if (!currentMembers.contains(newMember)) { newShapeLog(newRelation, currentRelation); return true; } } } } return false; } private Set neighboringShardsContainingInitialRelation( final Set neighboringShardCandidates) { final Set neighboringShardsContainingRelation = new HashSet<>(); neighboringShardCandidates .forEach(shard -> this.policy.getAtlasFetcher().apply(shard).ifPresent(atlas -> { if (neighboringAtlasContainingInitialRelation(atlas)) { neighboringShardsContainingRelation.add(shard); } })); return neighboringShardsContainingRelation; } private void newLocation(final Location location, final LocationItem... source) { if (!loadedShardsfullyGeometricallyEncloseLocation(location)) { newShapeLog(location, source); addNewShards(this.sharding.shardsCovering(location)); } } private void newPolyLine(final PolyLine polyLine, final LineItem... source) { if (!loadedShardsfullyGeometricallyEnclosePolyLine(polyLine)) { newShapeLog(polyLine, source); addNewShards(this.sharding.shardsIntersecting(polyLine)); } } private void newPolygon(final Polygon polygon, final AtlasEntity... source) { if (!loadedShardsfullyGeometricallyEnclosePolygon(polygon)) { newShapeLog(polygon, source); addNewShards(this.sharding.shards(polygon)); } } private void newShapeLog(final GeometryPrintable geometry, final AtlasEntity... source) { if (logger.isDebugEnabled()) { logger.debug("{}: Triggering new shard load for {}{}", this.dynamicAtlas.getName(), source.length > 0 ? "Atlas " + new StringList(Iterables.stream(Iterables.asList(source)) .map(item -> item.getType() + " " + item.getIdentifier())) .join(", ") + " with shape " : "", geometry.toWkt()); } } private Set nonNullShards() { return new HashSet<>(this.loadedShards.keySet().stream() .filter(shard -> this.loadedShards.get(shard) != null).collect(Collectors.toSet())); } // NOSONAR here as complexity 16 is ok. private boolean relationCoveredInternal(final Relation relation, // NOSONAR final Set parentRelationIdentifierTree) { final RelationMemberList members = relation.members(); boolean result = true; for (final RelationMember member : members) { final AtlasEntity entity = member.getEntity(); if (entity instanceof Area) { if (!areaCovered((Area) entity)) { result = false; } } else if (entity instanceof LineItem) { if (!lineItemCovered((LineItem) entity)) { result = false; } } else if (entity instanceof LocationItem) { if (!locationItemCovered((LocationItem) entity)) { result = false; } } else if (entity instanceof Relation) { result = relationMemberCoveredInternal(relation, (Relation) entity, parentRelationIdentifierTree); } else { throw new CoreException("Unknown Relation Member Type: {}", entity.getClass().getName()); } } return result; } private boolean relationCoversInitialShardBounds(final Relation relation) { final Set parentRelationIdentifierTree = new HashSet<>(); parentRelationIdentifierTree.add(relation.getIdentifier()); return relationCoversInitialShardBoundsInternal(relation, parentRelationIdentifierTree); } // NOSONAR here as complexity 16 is ok. private boolean relationCoversInitialShardBoundsInternal(final Relation relation, // NOSONAR final Set parentRelationIdentifierTree) { final RelationMemberList members = relation.members(); boolean result = false; for (final RelationMember member : members) { final AtlasEntity entity = member.getEntity(); if (entity instanceof Area) { if (areaCoversInitialShardBounds((Area) entity)) { result = true; } } else if (entity instanceof LineItem) { if (lineItemCoversInitialShardBounds((LineItem) entity)) { result = true; } } else if (entity instanceof LocationItem) { if (locationItemCoversInitialShardBounds((LocationItem) entity)) { result = true; } } else if (entity instanceof Relation) { result = relationMemberCoversInitialShardBoundsInternal(relation, (Relation) entity, parentRelationIdentifierTree); } else { throw new CoreException("Unknown Relation Member Type: {}", entity.getClass().getName()); } } return result; } private boolean relationMemberCoveredInternal(final Relation parentRelation, final Relation relation, final Set parentRelationIdentifierTree) { boolean result = true; final long newIdentifier = relation.getIdentifier(); if (parentRelationIdentifierTree.contains(newIdentifier)) { logger.error( "Skipping! Unable to expand on relation which has a loop: {}. Parent tree: {}", parentRelation, parentRelationIdentifierTree); } else { final Set newParentRelationIdentifierTree = new HashSet<>(); newParentRelationIdentifierTree.addAll(parentRelationIdentifierTree); newParentRelationIdentifierTree.add(newIdentifier); if (!relationCoveredInternal(relation, newParentRelationIdentifierTree)) { result = false; } } return result; } private boolean relationMemberCoversInitialShardBoundsInternal(final Relation parentRelation, final Relation relation, final Set parentRelationIdentifierTree) { boolean result = false; final long newIdentifier = relation.getIdentifier(); if (parentRelationIdentifierTree.contains(newIdentifier)) { logger.error( "Skipping! Unable to expand on relation which has a loop: {}. Parent tree: {}", parentRelation, parentRelationIdentifierTree); } else { final Set newParentRelationIdentifierTree = new HashSet<>(); newParentRelationIdentifierTree.addAll(parentRelationIdentifierTree); newParentRelationIdentifierTree.add(newIdentifier); if (relationCoversInitialShardBoundsInternal(relation, newParentRelationIdentifierTree)) { result = true; } } return result; } private boolean shouldBuildUnderlyingMultiAtlasWhenAddingNewShards( final Set initialNonEmptyLoadedShards) { final boolean thereAreNewViableShards = !initialNonEmptyLoadedShards .equals(nonNullShards()); // If DynamicAtlas is not initialized yet, it means this call is within the constructor. // We always load the initial shards first. final boolean dynamicAtlasNotInitializedYet = !this.initialized; // This is either: // 1. The opposite waiting for a preemptive load call // OR // 2. The preemptive load call has already happened and we are in a subsequent call. final boolean loadingIsNotDeferredOrItIsAndAlreadyHappened = !this.policy.isDeferLoading() || this.isAlreadyLoaded; final boolean shouldBuildUnderlyingMultiAtlas = thereAreNewViableShards // NOSONAR && (dynamicAtlasNotInitializedYet || loadingIsNotDeferredOrItIsAndAlreadyHappened); return shouldBuildUnderlyingMultiAtlas; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * @author matthieun */ public class DynamicEdge extends Edge { private static final long serialVersionUID = -3839789846949424342L; // Not index! private final long identifier; protected DynamicEdge(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public PolyLine asPolyLine() { return subEdge().asPolyLine(); } @Override public Node end() { return new DynamicNode(dynamicAtlas(), subEdge().end().getIdentifier()); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return subEdge().getTags(); } @Override public Set relations() { return subEdge().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } @Override public Node start() { return new DynamicNode(dynamicAtlas(), subEdge().start().getIdentifier()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private Edge subEdge() { final Edge result = dynamicAtlas().subEdge(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * @author matthieun */ public class DynamicLine extends Line { private static final long serialVersionUID = -7689258557279641445L; // Not index! private final long identifier; protected DynamicLine(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public PolyLine asPolyLine() { return subLine().asPolyLine(); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return subLine().getTags(); } @Override public Set relations() { return subLine().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private Line subLine() { final Line result = dynamicAtlas().subLine(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * @author matthieun */ public class DynamicNode extends Node { private static final long serialVersionUID = 7046248083667389625L; // Not index! private final long identifier; protected DynamicNode(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return subNode().getLocation(); } @Override public Map getTags() { return subNode().getTags(); } @Override public SortedSet inEdges() { return subNode().inEdges().stream() .map(edge -> new DynamicEdge(dynamicAtlas(), edge.getIdentifier())) .collect(Collectors.toCollection(TreeSet::new)); } @Override public SortedSet outEdges() { return subNode().outEdges().stream() .map(edge -> new DynamicEdge(dynamicAtlas(), edge.getIdentifier())) .collect(Collectors.toCollection(TreeSet::new)); } @Override public Set relations() { return subNode().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private Node subNode() { final Node result = dynamicAtlas().subNode(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicPoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * @author matthieun */ public class DynamicPoint extends Point { private static final long serialVersionUID = 5290355290550015953L; // Not index! private final long identifier; protected DynamicPoint(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return subPoint().getLocation(); } @Override public Map getTags() { return subPoint().getTags(); } @Override public Set relations() { return subPoint().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private Point subPoint() { final Point result = dynamicAtlas().subPoint(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/DynamicRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; /** * @author matthieun */ public class DynamicRelation extends Relation { private static final long serialVersionUID = 7994622214805021474L; // Not index! private final long identifier; protected DynamicRelation(final DynamicAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public RelationMemberList allKnownOsmMembers() { return getRelationMembersAsDynamicEntities(subRelation().allKnownOsmMembers()); } @Override public List allRelationsWithSameOsmIdentifier() { return subRelation().allRelationsWithSameOsmIdentifier().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toList()); } @Override public Optional asMultiPolygon() { return subRelation().asMultiPolygon(); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return subRelation().getTags(); } @Override public RelationMemberList members() { return getRelationMembersAsDynamicEntities(subRelation().members()); } @Override public Long osmRelationIdentifier() { return subRelation().osmRelationIdentifier(); } @Override public Set relations() { return subRelation().relations().stream() .map(relation -> new DynamicRelation(dynamicAtlas(), relation.getIdentifier())) .collect(Collectors.toSet()); } private DynamicAtlas dynamicAtlas() { return (DynamicAtlas) this.getAtlas(); } private RelationMemberList getRelationMembersAsDynamicEntities( final RelationMemberList memberList) { final List newMemberList = new ArrayList<>(); for (final RelationMember member : memberList) { final AtlasEntity entity = member.getEntity(); AtlasEntity dynamicEntity = null; switch (entity.getType()) { case NODE: dynamicEntity = new DynamicNode(dynamicAtlas(), entity.getIdentifier()); break; case EDGE: dynamicEntity = new DynamicEdge(dynamicAtlas(), entity.getIdentifier()); break; case POINT: dynamicEntity = new DynamicPoint(dynamicAtlas(), entity.getIdentifier()); break; case LINE: dynamicEntity = new DynamicLine(dynamicAtlas(), entity.getIdentifier()); break; case AREA: dynamicEntity = new DynamicArea(dynamicAtlas(), entity.getIdentifier()); break; case RELATION: dynamicEntity = new DynamicRelation(dynamicAtlas(), entity.getIdentifier()); break; default: throw new CoreException("Invalid entity type {}", entity.getType()); } newMemberList.add(new RelationMember(member.getRole(), dynamicEntity, member.getRelationIdentifier())); } return new RelationMemberList(newMemberList); } private Relation subRelation() { final Relation result = dynamicAtlas().subRelation(this.identifier); if (result != null) { return result; } else { throw new CoreException("DynamicAtlas {} moved too fast! {} {} is missing now.", dynamicAtlas().getName(), this.getClass().getSimpleName(), this.identifier); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/README.md ================================================ # `DynamicAtlas` ## Locally complete map with shards When splitting the map in many shards for easier distributed processing, the complexity of the tasks working on those shards goes up. The data is local, and not global, there are boundary effects at the shard delimitations, and the data might look different. For example, a road might be in shard A, but its intersection with another road is not visible, as it is inside shard B. One process doing anything related to shard A cannot just load shard A, it would then have to load shard A and B. To alleviate those concerns, and make shard-based map processing easier and more intuitive, `DynamicAtlas` abstracts most of the shard resolution and stitching, to present a simple Atlas API that takes care of expansion, stitching and shard fetching under the cover. ## Creation All the user has to provide to create a `DynamicAtlas` is: - **Sharding Tree**: A `Sharding` tree, which contains the definition of how the world is split. The currently supported implementations are `SlippyTileSharding` (all shards at the same zoom level) and `DynamicTileSharding` which is a simple quad tree split of the world. Any other implementation of `Sharding` would work too. - **Starting Point**: The initial shard(s) of interest, or a `Polygon` covering the intial shard(s) of interest. - **Atlas Fetcher**: A function called "fetcher" that tells the `DynamicAtlas` how to find the Atlas shards it needs. This function is very generic, and takes a `Shard` in input and returns an `Optional`. The returned Atlas can either not exist (exhausted the map data), or come from memory, disk, remote storage... The user chooses. - **Expansion Policy**: An "expansion policy" that specifies how the `DynamicAtlas` needs to explore neighboring shards. Several options are described below. ## Atlas Fetcher The most simple use case is where the user has a local folder `myFolder` with all the Dominica (DMA) Atlas shards saved using `PackedAtlas`, and the file names are equal to the shard names. For example, a shard at zoom 8, x index 2345 and y index 4567 would correspond to a file called `myFolder/DMA_8-2345-4567.atlas`. Then a simple atlas fetcher function is as follows: ```java Function> fetcher = shard -> { File folder = new File("myFolder"); File atlasFile = folder.child("DMA_" + shard.getName() + ".atlas"); if (file.exists()) { try { Atlas atlas = PackedAtlas.load(atlasFile); return Optional.of(atlas); } catch(Exception e) { logger.info("Unable to load shard {}", shard.getName(), e); } } return Optional.empty(); } ``` As it is in the above example, the user providing the fetcher is also responsible for handling load failures, and missing or nonexistent resources. ### Shard filtering As the user controls the fetcher, it also controls the whole loading process from resource to Atlas object. Sometimes, it can be useful to also filter Atlas shards prior to providing them to the DynamicAtlas through the fetcher, when the user knows that only a subset of the map data will be useful. Here is the same example but with a user interested only in motorways: ```java Predicate filter = entity -> entity.getType() HighwayTag.tag(entity).orElse(null) == HighwayTag.MOTORWAY; Function> fetcher = shard -> { File folder = new File("myFolder"); File atlasFile = folder.child("DMA_" + shard.getName() + ".atlas"); if (file.exists()) { try { Atlas atlas = PackedAtlas.load(atlasFile); return atlas.subAtlas(filter); } catch(Exception e) { logger.info("Unable to load shard {}", shard.getName(), e); } } return Optional.empty(); } ``` ### Remote access and caching When loading shards from a remote location (network drive, object storage service like Azure Blob or S3) it is recommended to cache the Atlas files locally on disk to avoid un-necessary bandwidth usage. Remote without caching: ```java // For the example, we assume this Hadoop FileSystem is already // initialized. FileSystem fileSystem; Function> fetcher = shard -> { String path = "myStorage://myFolder/" + "DMA_" + shard.getName() + ".atlas"; if (fileSystem.exists(path)) { try { Resource atlasResource = new InputStreamResource(() -> fileSystem.open(path)); Atlas atlas = PackedAtlas.load(atlasResource); return Optional.of(atlas); } catch(Exception e) { logger.info("Unable to load shard {}", shard.getName(), e); } } return Optional.empty(); } ``` Here is the same example with caching enabled (Uses [`HadoopAtlasFileCache`](https://github.com/osmlab/atlas-generator/blob/4.0.9/src/main/java/org/openstreetmap/atlas/generator/tools/caching/HadoopAtlasFileCache.java) from the [atlas-generator](https://github.com/osmlab/atlas-generator) project): ```java // Here are the Hadoop configuration strings, that can contain the // credentials to connect to the remote storage. Map hadoopConfiguration; HadoopAtlasFileCache cache = new HadoopAtlasFileCache("myStorage://myFolder/", hadoopConfiguration); Function> fetcher = shard -> { Optional atlasResource = cache.get("DMA", shard); if (atlasResource.isPresent()) { try { Atlas atlas = PackedAtlas.load(atlasResource.get()); return Optional.of(atlas); } catch(Exception e) { logger.info("Unable to load shard {}", shard.getName(), e); } } return Optional.empty(); } ``` ## Expansion Policy The expansion policy ([`DynamicAtlasPolicy`](policy/DynamicAtlasPolicy.java)) determines the behavior of the expansion steps of the `DynamicAtlas`, and can have vast impact on performance and ease of use. It also contains the sharding tree, initial shard(s) and fetcher function. ### Expansion As the user queries features from the `DynamicAtlas`, the `DynamicAtlas` intercepts the calls and checks if that feature intersects the initial shard boundaries. If it does, it blocks the call, removes the current stitched set of shards, loads the needed shards using the fetcher, and re-stitches (using `MultiAtlas`). It then returns the proper feature with all the data around it properly loaded. #### Indefinite Expansion By default, the policy is set for indefinite expansion. That means that as long as features intersect loaded shards boundaries, they will trigger a new MultiAtlas load. This is good for debugging and testing, but most production cases need to cap extension. #### Finite Expansion Finite expansion is toggled with `DynamicAtlasPolicy.withExtendIndefinitely(false);`. Once this is set, the features that will trigger a new shard load and MultiAtlas load are only the ones that intersect the initial shard(s) of interest, or the initial `Polygon`. That way, all the features intersecting the initial area of interest will always have a locally complete map around them, but the expansion will be capped. #### Indiscriminate Expansion By default, the expansion can be triggered by any feature requested that falls outside of the shard boundaries. #### Discriminate Expansion Discriminate expansion can be enabled by using a `DynamicAtlasPolicy` that contains `final Predicate atlasEntitiesToConsiderForExpansion`. When this is provided, every time the `DynamicAtlas` needs to check if a feature needs to trigger a shard expansion, it will do so only if that feature passes the `atlasEntitiesToConsiderForExpansion` predicate. For example, this is useful when a user is interested only in a certain type of roads, and expanding based on parks or lakes does not provide any more context. ### Deferred and Preemptive Loading When _finite expansion is enabled_ (or indefinite expansion disabled), and deferred loading is true (`DynamicAtlasPolicy.withDeferredLocaing(true);`), there is an option to toggle preemptive loading: `DynamicAtlas.preemptiveLoad();`. No shard load will be triggered until `preemptiveLoad` is called. This will ensure that all the shards needed will be pre-loaded once and for all. It also disables the "interception" of queries for features by the user, as the `DynamicAtlas` is already sure everything is properly loaded. This is the same as having an "intelligent" MultiAtlas that knows what shards the user needs based on each area of interest. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/policy/DynamicAtlasPolicy.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic.policy; import java.util.ArrayList; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.dynamic.DynamicAtlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.utilities.maps.MultiMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class DynamicAtlasPolicy { private static final Logger logger = LoggerFactory.getLogger(DynamicAtlasPolicy.class); private final Polygon maximumBounds; private final Sharding sharding; private final Function> atlasFetcher; private final Set initialShards; private boolean extendIndefinitely = true; private boolean deferLoading = false; private Consumer> shardSetChecker = set -> { }; private Predicate atlasEntitiesToConsiderForExpansion = entity -> true; private boolean aggressivelyExploreRelations = false; // In case the initial shards were found using a Polygon or a MultiPolygon, remember it to // provide the initial shards shape. This will be useful to not over-extend when using // extendIndefinitely=false private Optional shapeCoveringInitialShards = Optional.empty(); public DynamicAtlasPolicy(final Function> atlasFetcher, final Sharding sharding, final MultiPolygon shapeCoveringInitialShards, final Polygon maximumBounds) { this.initialShards = new HashSet<>(); this.shapeCoveringInitialShards = Optional.of(shapeCoveringInitialShards); sharding.shards(shapeCoveringInitialShards).forEach(this.initialShards::add); this.atlasFetcher = atlasFetcher; this.maximumBounds = maximumBounds; this.sharding = sharding; } public DynamicAtlasPolicy(final Function> atlasFetcher, final Sharding sharding, final Polygon shapeCoveringInitialShards, final Polygon maximumBounds) { this.initialShards = new HashSet<>(); this.shapeCoveringInitialShards = Optional .of(MultiPolygon.forPolygon(shapeCoveringInitialShards)); sharding.shards(shapeCoveringInitialShards).forEach(this.initialShards::add); this.atlasFetcher = atlasFetcher; this.maximumBounds = maximumBounds; this.sharding = sharding; } public DynamicAtlasPolicy(final Function> atlasFetcher, final Sharding sharding, final Set initialShards, final Polygon maximumBounds) { this.initialShards = initialShards; this.atlasFetcher = atlasFetcher; this.maximumBounds = maximumBounds; this.sharding = sharding; } public DynamicAtlasPolicy(final Function> atlasFetcher, final Sharding sharding, final Shard initialShard, final Polygon maximumBounds) { this.initialShards = new HashSet<>(); this.initialShards.add(initialShard); this.atlasFetcher = atlasFetcher; this.maximumBounds = maximumBounds; this.sharding = sharding; } public Predicate getAtlasEntitiesToConsiderForExpansion() { return this.atlasEntitiesToConsiderForExpansion; } public Function> getAtlasFetcher() { // Here, make sure to not load outside the bounds. return shard -> { if (this.maximumBounds.overlaps(shard.bounds())) { return this.atlasFetcher.apply(shard); } else { logger.debug( "Skipping atlasFetcher for {} because shard bounds are outside the policy's maximumBounds", shard.getName()); } return Optional.empty(); }; } public Set getInitialShards() { return this.initialShards; } public MultiPolygon getInitialShardsBounds() { if (this.shapeCoveringInitialShards.isPresent()) { return this.shapeCoveringInitialShards.get(); } final MultiMap outerToInners = new MultiMap<>(); this.initialShards.forEach(shard -> outerToInners.put(shard.bounds(), new ArrayList<>())); this.shapeCoveringInitialShards = Optional.of(new MultiPolygon(outerToInners)); return this.shapeCoveringInitialShards.get(); } public Polygon getMaximumBounds() { return this.maximumBounds; } public Consumer> getShardSetChecker() { return this.shardSetChecker; } public Sharding getSharding() { return this.sharding; } public boolean isAggressivelyExploreRelations() { return this.aggressivelyExploreRelations; } public boolean isDeferLoading() { return this.deferLoading; } public boolean isExtendIndefinitely() { return this.extendIndefinitely; } /** * This switch tells the {@link DynamicAtlas} to preemptively and temporarily load the * neighboring shards to see if they contain the relation in the current shard and if the member * list is different. In which case it expands to the neighboring shards that are including * those members. * * @param aggressivelyExploreRelations * True to aggressively explore relations * @return The modified policy */ public DynamicAtlasPolicy withAggressivelyExploreRelations( final boolean aggressivelyExploreRelations) { this.aggressivelyExploreRelations = aggressivelyExploreRelations; return this; } /** * @param atlasEntitiesToConsiderForExpansion * A predicate that defines what entities will be considered when deciding to expand * or not to a neighboring shard. * @return The modified policy */ public DynamicAtlasPolicy withAtlasEntitiesToConsiderForExpansion( final Predicate atlasEntitiesToConsiderForExpansion) { this.atlasEntitiesToConsiderForExpansion = atlasEntitiesToConsiderForExpansion; return this; } /** * Defer loading until the load command is sent. * * @param deferLoading * True to defer loading until the load command is sent. * @return The modified policy */ public DynamicAtlasPolicy withDeferredLoading(final boolean deferLoading) { this.deferLoading = deferLoading; return this; } /** * The extension policy: if this is set to true, then the loading of shards will go as far as it * can within the boundary polygon. If this is set to false, then the loading will extend only * if the feature warranting the extension is included within or intersects the initial shard * boundaries. * * @param extendIndefinitely * True to extend indefinitely. * @return The modified policy */ public DynamicAtlasPolicy withExtendIndefinitely(final boolean extendIndefinitely) { this.extendIndefinitely = extendIndefinitely; return this; } /** * @param shardSetChecker * A function that will inspect the shards prior to loading them in a MultiAtlas. The * default version just does nothing, but this could for example throw an exception * if the DynamicAtlas is loading too many neighboring shards, or send shard * downloading statistics. This is up to the end user. * @return The modified policy */ public DynamicAtlasPolicy withShardSetChecker(final Consumer> shardSetChecker) { this.shardSetChecker = shardSetChecker; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/dynamic/policy/DynamicAtlasResourcePolicy.java ================================================ package org.openstreetmap.atlas.geography.atlas.dynamic.policy; import java.util.Optional; import java.util.function.Function; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Policy that uses {@link Resource}s and takes care of converting them to {@link Atlas} * * @author matthieun */ public class DynamicAtlasResourcePolicy extends DynamicAtlasPolicy { public DynamicAtlasResourcePolicy(final Function> atlasFetcher, final Sharding sharding, final Shard initialShard, final Polygon maximumBounds) { super(shard -> { final Optional resourceOption = atlasFetcher.apply(shard); if (resourceOption.isPresent()) { return Optional.ofNullable(new AtlasResourceLoader().load(resourceOption.get())); } else { return Optional.empty(); } }, sharding, initialShard, maximumBounds); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/exception/AtlasIntegrityException.java ================================================ package org.openstreetmap.atlas.geography.atlas.exception; import org.openstreetmap.atlas.exception.CoreException; /** * @author matthieun */ public class AtlasIntegrityException extends CoreException { private static final long serialVersionUID = -2780280960455310936L; public AtlasIntegrityException(final String message) { super(message); } public AtlasIntegrityException(final String message, final Object... arguments) { super(message, arguments); } public AtlasIntegrityException(final String message, final Throwable cause) { super(message, cause); } public AtlasIntegrityException(final String message, final Throwable cause, final Object... arguments) { super(message, cause, arguments); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/geojson/AtlasGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.geojson; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasLoadingCommand; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utility to save an Atlas as GeoJson * * @author matthieun */ public class AtlasGeoJsonConverter extends AtlasLoadingCommand { private static final Logger logger = LoggerFactory.getLogger(AtlasGeoJsonConverter.class); private static final Switch GEOJSON = new Switch<>("geojson", "The file where to save as GeoJson", File::new, Optionality.REQUIRED); public static void main(final String[] args) { new AtlasGeoJsonConverter().run(args); } @Override protected int onRun(final CommandMap command) { final Atlas atlas = loadAtlas(command); final File output = (File) command.get(GEOJSON); atlas.saveAsGeoJson(output); logger.info("Saved to {}", output); return 0; } @Override protected SwitchList switches() { return super.switches().with(GEOJSON); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/geojson/LineDelimitedGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.geojson; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.threads.Pool; import org.openstreetmap.atlas.utilities.time.Time; import org.openstreetmap.atlas.utilities.vectortiles.TippecanoeCommands; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * This CLI takes a directory of atlas files and turns them into line-delimited GeoJSON. If you * would also like to convert into MBTiles with tippecanoe, use TippecanoeExporter. * * @author hallahan */ public class LineDelimitedGeoJsonConverter extends Command { /** * After all of your files are converted to LD GeoJSON, it is then concatenated into * EVERYTHING.geojson */ public static final String EVERYTHING = "EVERYTHING.geojson"; public static final int EXIT_FAILURE = 1; protected static final Switch GEOJSON_DIRECTORY = new Switch<>("geojsonDirectory", "The directory to write line-delimited GeoJSON.", Paths::get, Optionality.REQUIRED); protected static final Switch OVERWRITE = new Switch<>("overwrite", "Choose to automatically overwrite a GeoJSON file if it exists at the given path.", Boolean::parseBoolean, Optionality.OPTIONAL, "false"); // Works great on a MacBook Pro (Retina, 15-inch, Mid 2015) private static final int DEFAULT_THREADS = 8; private static final Logger logger = LoggerFactory .getLogger(LineDelimitedGeoJsonConverter.class); private static final AtlasResourceLoader ATLAS_RESOURCE_LOADER = new AtlasResourceLoader(); private static final Switch ATLAS_DIRECTORY = new Switch<>("atlasDirectory", "The directory of atlases to convert.", Paths::get, Optionality.REQUIRED); private static final Switch THREADS = new Switch<>("threads", "The number of threads to work on processing atlas shards.", Integer::valueOf, Optionality.OPTIONAL, String.valueOf(DEFAULT_THREADS)); /** * We only want positive (main) edges, because the negative edge can be derived at the * application level, and this encodes extraneous data that can be easily derived by the map * viewer. For relations, we only want multipolygon relations, as the rest can be derived from * their members. */ private static final Predicate ENTITY_PREDICATE = entity -> { // We only want positive atlas entities. No negative ids. if (ItemType.EDGE.equals(entity.getType())) { final Edge edge = (Edge) entity; if (!edge.isMainEdge()) { return false; } } // Because we're writing the multipolygon relations, we don't want to also write the area // components that are pieces of the multipolygon relation. if (ItemType.AREA.equals(entity.getType())) { final Set relations = entity.relations(); if (!relations.isEmpty()) { return relations.stream().noneMatch(relation -> Validators.isOfType(relation, RelationTypeTag.class, RelationTypeTag.MULTIPOLYGON)); } } return true; }; /** * If we are rendering vector tiles, we may want to examine various tags of a given atlas entity * and make decisions for the layer name, min zoom, and max zoom for the feature. Depending on * your vector tile renderer, as well as map data visualization needs, you can override this * BiConsumer to mutate your JSON object as you see fit. */ private BiConsumer jsonMutator = (atlasEntity, feature) -> { }; public static void main(final String[] args) { new LineDelimitedGeoJsonConverter().run(args); } private static List fetchAtlasFilesInDirectory(final Path directory) { return new File(directory.toFile()).listFilesRecursively().stream() .filter(AtlasResourceLoader.HAS_ATLAS_EXTENSION).collect(Collectors.toList()); } @Override protected int onRun(final CommandMap command) { final Time time = Time.now(); final Path atlasDirectory = (Path) command.get(ATLAS_DIRECTORY); final Path geojsonDirectory = (Path) command.get(GEOJSON_DIRECTORY); final Boolean overwrite = (Boolean) command.get(OVERWRITE); final int threads = (Integer) command.get(THREADS); if (overwrite) { try { FileUtils.deleteDirectory(geojsonDirectory.toFile()); } catch (final IOException noDelete) { logger.warn( "Tried to delete GeoJSON output directory {} for overwrite, but unable.", geojsonDirectory, noDelete); } } final List atlases = fetchAtlasFilesInDirectory(atlasDirectory); if (atlases.isEmpty()) { logger.error("There are no atlas files in {}. Exiting...", atlasDirectory); System.exit(EXIT_FAILURE); } logger.info("About to convert {} atlas shards into line-delimited GeoJSON...", atlases.size()); // Execute in a pool of threads so we limit how many atlases get loaded in parallel. try (Pool pool = new Pool(threads, "atlas-converter-worker")) { atlases.forEach( atlasFile -> pool.queue(() -> this.convertAtlas(atlasFile, geojsonDirectory))); } TippecanoeCommands.concatenate(geojsonDirectory); logger.info( "Finished converting directory of atlas shards into line-delimited GeoJSON in {}!", time.elapsedSince()); return 0; } protected void setJsonMutator(final BiConsumer jsonMutator) { this.jsonMutator = jsonMutator; } @Override protected SwitchList switches() { return new SwitchList().with(ATLAS_DIRECTORY, GEOJSON_DIRECTORY, OVERWRITE, THREADS); } private void convertAtlas(final File atlasFile, final Path geojsonDirectory) { final Time time = Time.now(); final Atlas atlas = ATLAS_RESOURCE_LOADER.load(atlasFile); final String name = FilenameUtils.removeExtension(atlasFile.getName()) + FileSuffix.GEO_JSON.toString(); final File geojsonFile = new File(geojsonDirectory.resolve(name).toFile()); atlas.saveAsLineDelimitedGeoJsonFeatures(geojsonFile, ENTITY_PREDICATE, this.jsonMutator); logger.info("Saved {} in {}.", name, time.elapsedSince()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/inspection/EntityClassifier.java ================================================ package org.openstreetmap.atlas.geography.atlas.inspection; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * This interface provides default implementations for common types of entity classification. It's * unlikely that implementations of this interface would need to override the default behaviour, * though still an obvious option. * * @author brian_l_davis */ public interface EntityClassifier { /** * Checks if the feature is an edge * * @param entity * The entity to check in * @return True if it's a edge */ default boolean isEdge(final AtlasEntity entity) { return entity instanceof Edge; } /** * Checks if the feature is a line * * @param entity * The entity to check in * @return True if it's a line */ default boolean isLine(final AtlasEntity entity) { return entity instanceof Line; } /** * Checks if the feature is a road * * @param entity * The entity to check in * @return True if it's a road */ default boolean isRoad(final AtlasEntity entity) { return isEdge(entity) || isLine(entity) && Validators.isOfType(entity, HighwayTag.class, HighwayTag.PROPOSED, HighwayTag.CONSTRUCTION); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Area.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonObject; /** * Area from an {@link Atlas} * * @author matthieun */ public abstract class Area extends AtlasItem { private static final long serialVersionUID = 5244165133018408045L; protected Area(final Atlas atlas) { super(atlas); } @Override public JsonObject asGeoJsonGeometry() { return asPolygon().asGeoJsonGeometry(); } /** * @return The {@link PolyLine} that represents this {@link LineItem} */ public abstract Polygon asPolygon(); @Override public Rectangle bounds() { return asPolygon().bounds(); } /** * @return The closed {@link Polygon}, with the end {@link Location} equal to the start * {@link Location}. */ public Polygon getClosedGeometry() { return new Polygon(asPolygon().closedLoop()); } /** * @return The underlying {@link PolyLine}, which will not have the end {@link Location} equal * to the start {@link Location}. i.e. the area will not be closed. */ @Override public Iterable getRawGeometry() { return asPolygon(); } @Override public ItemType getType() { return ItemType.AREA; } @Override public boolean intersects(final GeometricSurface surface) { return surface.overlaps(asPolygon()); } @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); return "[Area: id=" + this.getIdentifier() + ", polygon=" + this.asPolygon() + ", relations=(" + relationsString + "), " + tagString() + "]"; } @Override public LocationIterableProperties toGeoJsonBuildingBlock() { final Map tags = getTags(); tags.put("identifier", String.valueOf(getIdentifier())); tags.put("osmIdentifier", String.valueOf(getOsmIdentifier())); tags.put("itemType", String.valueOf(getType())); final Optional shardName = getAtlas().metaData().getShardName(); shardName.ifPresent(shard -> tags.put("shard", shard)); final StringList parentRelations = new StringList(); this.relations().forEach(relation -> { final RelationMember member = relation.members().get(getIdentifier(), getType()); parentRelations.add(member.getRelationIdentifier() + "-" + member.getRole()); }); if (!parentRelations.isEmpty()) { tags.put("parentRelations", parentRelations.join(", ")); } return new GeoJsonBuilder.LocationIterableProperties(getClosedGeometry(), tags); } @Override public String toString() { return "[Area: id=" + this.getIdentifier() + ", polygon=" + this.asPolygon() + ", " + tagString() + "]"; } @Override public byte[] toWkb() { return this.asPolygon().toWkb(); } @Override public String toWkt() { return this.asPolygon().toWkt(); } @Override public boolean within(final GeometricSurface surface) { return this.asPolygon().within(surface); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/AtlasEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.GeometryPrintable; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.ReverseIdentifierFactory; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonFeature; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.tags.LastEditTimeTag; import org.openstreetmap.atlas.tags.LastEditUserIdentifierTag; import org.openstreetmap.atlas.tags.LastEditUserNameTag; import org.openstreetmap.atlas.tags.names.NameTag; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Duration; import org.openstreetmap.atlas.utilities.time.Time; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * A located entity with tags * * @author matthieun * @author mgostintsev * @author Sid * @author hallahan */ public abstract class AtlasEntity implements AtlasObject, DiffViewFriendlyItem, GeometryPrintable, GeoJsonFeature { private static final long serialVersionUID = -6072525057489468736L; // The atlas this item belongs to private final Atlas atlas; protected AtlasEntity(final Atlas atlas) { this.atlas = atlas; } /** * Utility function to test if an entity's tags match some given tag keys. * * @param matches * The given tag keys to match * @return True if at least one tag of the entity matches one of the given keys */ public boolean containsKey(final Iterable matches) { final Map tags = this.getTags(); for (final String candidate : matches) { if (tags.containsKey(candidate)) { return true; } } return false; } /** * Utility function to test if an entity's tags start with some given tag keys. * * @param matches * The given tag keys to match * @return True if at least one tag of the entity starts with one of the given keys */ public boolean containsKeyStartsWith(final Iterable matches) { final Map tags = this.getTags(); for (final String candidate : matches) { for (final String key : tags.keySet()) { if (key.startsWith(candidate)) { return true; } } } return false; } /** * For comparison, we require the AtlasEntities to be from the same instance of Atlas. Edges * with same attributes from two instances of Atlas (with same data) are considered different */ @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other != null && this.getClass() == other.getClass()) { final AtlasEntity that = (AtlasEntity) other; // Do not call atlas.equals() which would browse all the items and create a stack // overflow return this.getAtlas() == that.getAtlas() && this.getIdentifier() == that.getIdentifier(); } return false; } @Override public Atlas getAtlas() { return this.atlas; } /** * A method that creates properties for a GeoJSON Feature from the tags. * * @return A GeoJSON properties object that is to be put in a Feature. */ @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = new JsonObject(); getTags().forEach(properties::addProperty); properties.addProperty(GeoJsonUtils.IDENTIFIER, getIdentifier()); properties.addProperty(GeoJsonUtils.OSM_IDENTIFIER, getOsmIdentifier()); properties.addProperty(GeoJsonUtils.ITEM_TYPE, String.valueOf(getType())); final Set relations = relations(); if (!relations.isEmpty()) { final JsonArray relationsArray = new JsonArray(); properties.add("relations", relationsArray); for (final Relation relation : relations) { relationsArray.add(new JsonPrimitive(relation.getIdentifier())); } } return properties; } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.FEATURE; } /** * The value in the "name" attribute. * * @return an optional string representing the value of the name tag. */ public Optional getName() { return this.getTag(NameTag.KEY); } @Override public long getOsmIdentifier() { return new ReverseIdentifierFactory().getOsmIdentifier(this.getIdentifier()); } @Override public Optional getTag(final String key) { return Optional.ofNullable(getTags().get(key)); } public abstract ItemType getType(); @Override public int hashCode() { return new HashCodeBuilder().append(getIdentifier()).append(getClass()).hashCode(); } /** * Return true if the entity intersects the geometricSurface. If it is a {@link LocationItem}, * the polygon fully encloses it. For a {@link LineItem} the geometricSurface overlaps it. For * an {@link Area} the geometricSurface overlaps it. For a relation, at least one member of the * relation returns true to this method. * * @param surface * The {@link GeometricSurface} to test * @return True if it intersects */ public abstract boolean intersects(GeometricSurface surface); /** * @return If available, the {@link Time} at which the entity was last edited. */ public Optional

* NOSONAR here as the {@link AtlasEntity} equals and hashcode are good enough. ""equals(Object * obj)" should be overridden along with the "compareTo(T obj)" method (squid:S1210)" */ @Override public int compareTo(final Edge other) // NOSONAR { return Long.compare(this.getIdentifier(), other.getIdentifier()); } /** * @return All the {@link Edge}s connected to the end {@link Node}s of this {@link Edge}, except * self. If this {@link Edge} is a two-way road, then the reversed {@link Edge} will be * included in the set. */ public Set connectedEdges() { final Set result = new HashSet<>(); for (final Edge edge : this.end().connectedEdges()) { if (!this.equals(edge)) { result.add(edge); } } for (final Edge edge : this.start().connectedEdges()) { if (!this.equals(edge)) { result.add(edge); } } return result; } public Node connectedNode(final ConnectedNodeType connectedNodeType) { Validate.notNull(connectedNodeType); final Node connectedNode = connectedNodeType.getAccessFunction().apply(this); return connectedNode; } public Set connectedNodes() { final Set result = new HashSet<>(); result.add(this.start()); result.add(this.end()); return result; } /** * @return The same {@link Edge} but with the tags interpreted with this {@link Edge}'s * direction. For example, if this {@link Edge} is backwards from its OSM way, and the * way has a maxspeed:backward tag, here it will be translated into a maxspeed tag. Also * the maxspeed:forward tag will be filtered out (it will be used by the reverse edge). */ public Edge directionalized() { return new DirectionalizedEdge(this); } /** * @return The {@link Node} at the end of this {@link Edge} */ public abstract Node end(); @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = super.getGeoJsonProperties(); properties.addProperty(ConnectedNodeType.START.getPropertyName(), start().getIdentifier()); properties.addProperty(ConnectedNodeType.END.getPropertyName(), end().getIdentifier()); return properties; } /** * @return the main for this {@link Edge}, which may or may not be the main. */ public Edge getMainEdge() { return this.isMainEdge() ? this : this.reversed() .orElseThrow(() -> new CoreException( "Reverse edge should be available for edge {}", this.getIdentifier())); } public long getMainEdgeIdentifier() { return Math.abs(this.getIdentifier()); } /** * @return the main for this {@link Edge}, which may or may not be the main. * @deprecated Use getMainEdge instead. */ @Deprecated(since = "") public Edge getMasterEdge() { return getMainEdge(); } /** * @return The main edge identifier * @deprecated Use getMainEdgeIdentifier instead */ @Deprecated(since = "") public long getMasterEdgeIdentifier() { return getMainEdgeIdentifier(); } @Override public ItemType getType() { return ItemType.EDGE; } /** * @return {@code true} if there is a reverse edge to this one */ public boolean hasReverseEdge() { return this.getAtlas().edge(-this.getIdentifier()) != null; } /** * @return The {@link HighwayTag} of the Edge, if it is present. Return HighwayTag.NO if it is * not. */ public HighwayTag highwayTag() { final Optional result = HighwayTag.highwayTag(this); if (result.isPresent()) { return result.get(); } else { return HighwayTag.NO; } } /** * @return All the {@link Edge}s connected and pointing to the start {@link Node} of this * {@link Edge}. If this {@link Edge} is a two-way road, then the reversed {@link Edge} * will be included in the set. */ public Set inEdges() { return this.start().inEdges(); } /** * @param candidates * Set of edges and nodes to test connectivity to. * @return True if the edge is directly connected at its end to at least one of the candidate * items */ public boolean isConnectedAtEndTo(final Set candidates) { for (final AtlasItem item : candidates) { if (item instanceof Node && end().equals(item)) { return true; } if (item instanceof Edge && end().equals(((Edge) item).start())) { return true; } } return false; } /** * @param candidates * Set of edges and nodes to test connectivity to. * @return True if the edge is directly connected at its start to at least one of the candidate * items */ public boolean isConnectedAtStartTo(final Set candidates) { for (final AtlasItem item : candidates) { if (item instanceof Node && start().equals(item)) { return true; } if (item instanceof Edge && start().equals(((Edge) item).end())) { return true; } } return false; } /** * Checks if edge is a main edge, by verifying that its identifier is a main edge identifier. * * @return True if the edge's identifier is a main edge identifier */ public boolean isMainEdge() { return isMainEdgeIdentifier(this.getIdentifier()); } /** * @return True if the edge's identifier is a main edge identifier * @deprecated Use isMainEdge instead. */ @Deprecated(since = "") public boolean isMasterEdge() { return isMainEdge(); } /** * @param candidate * candidate for reverseEdge * @return {@code true} if candidate is the reverse of this {@link Edge} */ public boolean isReversedEdge(final Edge candidate) { return this.getIdentifier() == -candidate.getIdentifier(); } /** * @return {@code true} if the {@link Edge} is a way-sectioned road. */ public boolean isWaySectioned() { return reverseIdentifierFactory.getWaySectionIndex(this.getIdentifier()) != 0; } /** * @return All the {@link Edge}s connected and pointing out of the end {@link Node} of this * {@link Edge}. If this {@link Edge} is a two-way road, then the reversed {@link Edge} * will be included in the set. */ public Set outEdges() { return this.end().outEdges(); } /** * @return An {@link Edge} that is reversed to this one if it exists, empty otherwise. */ public Optional reversed() { final Edge edge = this.getAtlas().edge(-this.getIdentifier()); if (edge != null) { return Optional.of(edge); } return Optional.empty(); } public abstract Node start(); @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); final String startNodeString = start() != null ? Long.toString(start().getIdentifier()) : "null"; final String endNodeString = start() != null ? Long.toString(end().getIdentifier()) : "null"; final String polyLineWkt = this.asPolyLine() != null ? this.asPolyLine().toWkt() : "null"; return "[Edge" + ": id=" + this.getIdentifier() + ", startNode=" + startNodeString + ", endNode=" + endNodeString + ", polyLine=" + polyLineWkt + ", relations=(" + relationsString + "), " + tagString() + "]"; } @Override public String toString() { return "[Edge" + ": id=" + this.getIdentifier() + ", startNode=" + start().getIdentifier() + ", endNode=" + end().getIdentifier() + ", polyLine=" + this.asPolyLine().toWkt() + ", " + tagString() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/ItemType.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; /** * Type of item in an {@link Atlas} * * @author matthieun * @author Yazad Khambata */ public enum ItemType { NODE(0) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.nodes(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfNodes(); } }, EDGE(1) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.edges(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfEdges(); } }, AREA(2) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.areas(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfAreas(); } }, LINE(3) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.lines(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfLines(); } }, POINT(4) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.points(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfPoints(); } }, RELATION(5) { @Override public Iterable entitiesForIdentifiers(final Atlas atlas, final Long... identifiers) { return atlas.relations(identifiers); } @Override public long numberOfEntities(final Atlas atlas) { return atlas.numberOfRelations(); } }; private final int value; public static ItemType forEntity(final AtlasEntity entity) { return entity.getType(); } public static ItemType forValue(final int value) { for (final ItemType type : values()) { if (type.getValue() == value) { return type; } } throw new CoreException("Invalid value: {}", value); } public static ItemType shortValueOf(final String value) { switch (value) { case "N": return NODE; case "E": return EDGE; case "A": return AREA; case "L": return LINE; case "P": return POINT; case "R": return RELATION; default: throw new CoreException("Invalid short value {}", value); } } ItemType(final int value) { this.value = value; } public abstract Iterable entitiesForIdentifiers(Atlas atlas, Long... identifiers); public AtlasEntity entityForIdentifier(final Atlas atlas, final long identifier) { switch (this) { case NODE: return atlas.node(identifier); case EDGE: return atlas.edge(identifier); case AREA: return atlas.area(identifier); case LINE: return atlas.line(identifier); case POINT: return atlas.point(identifier); case RELATION: return atlas.relation(identifier); default: throw new CoreException("Invalid type {}", this); } } @SuppressWarnings("unchecked") public Class getMemberClass() { switch (this) { case NODE: return (Class) Node.class; case EDGE: return (Class) Edge.class; case AREA: return (Class) Area.class; case LINE: return (Class) Line.class; case POINT: return (Class) Point.class; case RELATION: return (Class) Relation.class; default: throw new CoreException("Invalid type {}", this); } } public int getValue() { return this.value; } public abstract long numberOfEntities(Atlas atlas); public String toShortString() { switch (this) { case NODE: return "N"; case EDGE: return "E"; case AREA: return "A"; case LINE: return "L"; case POINT: return "P"; case RELATION: return "R"; default: throw new CoreException("Invalid type {}", this); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Line.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.openstreetmap.atlas.geography.atlas.Atlas; /** * A line that is not navigable * * @author matthieun */ public abstract class Line extends LineItem { private static final long serialVersionUID = 5348604376185677L; protected Line(final Atlas atlas) { super(atlas); } @Override public ItemType getType() { return ItemType.LINE; } @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); return "[Line: id=" + this.getIdentifier() + ", polyLine=" + this.asPolyLine() + ", relations=(" + relationsString + "), " + tagString() + "]"; } @Override public String toString() { return "[Line: id=" + this.getIdentifier() + ", polyLine=" + this.asPolyLine() + ", " + tagString() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/LineItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Heading; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * {@link AtlasItem} that is in shape of a {@link PolyLine} * * @author matthieun * @author mgostintsev */ public abstract class LineItem extends AtlasItem { private static final long serialVersionUID = -2053566750957119655L; private static final Logger logger = LoggerFactory.getLogger(LineItem.class); protected LineItem(final Atlas atlas) { super(atlas); } @Override public JsonObject asGeoJsonGeometry() { return asPolyLine().asGeoJsonGeometry(); } /** * @return The {@link PolyLine} that represents this {@link LineItem} */ public abstract PolyLine asPolyLine(); @Override public Rectangle bounds() { return asPolyLine().bounds(); } @Override public Iterable getRawGeometry() { return asPolyLine(); } @Override public boolean intersects(final GeometricSurface surface) { return surface.overlaps(asPolyLine()); } /** * Check if this {@link LineItem} is closed. Closed is defined when the first {@link Location} * is the same as the last {@link Location}. * * @return {@code true} if it's closed. */ public boolean isClosed() { final PolyLine polyLine = asPolyLine(); return polyLine.first().equals(polyLine.last()); } /** * @return flag to denote zero length line items */ public boolean isZeroLength() { return length().equals(Distance.ZERO); } /** * @return The length of this item */ public Distance length() { return asPolyLine().length(); } /** * @return the number of shape-points for this item, including start and end points. */ public int numberOfShapePoints() { return asPolyLine().size(); } /** * @return The overall heading of the {@link PolyLine}: the heading between the start point and * the end point. */ public Optional overallHeading() { final PolyLine polyLine = this.asPolyLine(); if (polyLine.first().equals(polyLine.last())) { if (logger.isWarnEnabled()) { logger.warn( "Cannot compute ({},{})'s overall heading when the polyline has " + "same start and end locations : {}", this.getType(), this.getIdentifier(), polyLine.first().toWkt()); } return Optional.empty(); } return this.asPolyLine().overallHeading(); } @Override public LocationIterableProperties toGeoJsonBuildingBlock() { final Map tags = getTags(); tags.put("identifier", String.valueOf(getIdentifier())); tags.put("osmIdentifier", String.valueOf(getOsmIdentifier())); tags.put("itemType", String.valueOf(getType())); final Optional shardName = getAtlas().metaData().getShardName(); shardName.ifPresent(shard -> tags.put("shard", shard)); if (this instanceof Edge) { tags.put("startNode", String.valueOf(((Edge) this).start().getIdentifier())); tags.put("endNode", String.valueOf(((Edge) this).end().getIdentifier())); } final StringList parentRelations = new StringList(); this.relations().forEach(relation -> { final RelationMember member = relation.members().get(getIdentifier(), getType()); parentRelations.add(member.getRelationIdentifier() + "-" + member.getRole()); }); if (!parentRelations.isEmpty()) { tags.put("parentRelations", parentRelations.join(", ")); } return new GeoJsonBuilder.LocationIterableProperties(getRawGeometry(), tags); } @Override public byte[] toWkb() { return this.asPolyLine().toWkb(); } @Override public String toWkt() { return this.asPolyLine().toWkt(); } @Override public boolean within(final GeometricSurface surface) { return this.asPolyLine().within(surface); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/LocationItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.Map; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.Snapper.SnappedLocation; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonObject; /** * An {@link AtlasItem} that is represented by one single location * * @author matthieun */ public abstract class LocationItem extends AtlasItem { private static final long serialVersionUID = -2616559591051747286L; protected LocationItem(final Atlas atlas) { super(atlas); } @Override public JsonObject asGeoJsonGeometry() { return getLocation().asGeoJsonGeometry(); } @Override public Rectangle bounds() { return getLocation().bounds(); } /** * @return The item's {@link Location} */ public abstract Location getLocation(); @Override public Iterable getRawGeometry() { return getLocation(); } @Override public boolean intersects(final GeometricSurface surface) { return surface.fullyGeometricallyEncloses(getLocation()); } public SnappedLocation snapTo(final Area other) { return this.getLocation().snapTo(other.asPolygon()); } public SnappedLocation snapTo(final LineItem other) { return this.getLocation().snapTo(other.asPolyLine()); } @Override public LocationIterableProperties toGeoJsonBuildingBlock() { final Map tags = getTags(); tags.put("identifier", String.valueOf(getIdentifier())); tags.put("osmIdentifier", String.valueOf(getOsmIdentifier())); tags.put("itemType", String.valueOf(getType())); if (this instanceof Node) { final StringList inEdges = new StringList(); final StringList outEdges = new StringList(); ((Node) this).inEdges() .forEach(edge -> inEdges.add(edge != null ? edge.getIdentifier() : "null")); ((Node) this).outEdges() .forEach(edge -> outEdges.add(edge != null ? edge.getIdentifier() : "null")); tags.put("inEdges", inEdges.join(", ")); tags.put("outEdges", outEdges.join(", ")); } final Location location = this.getLocation(); tags.put("latitude", String.valueOf(location.getLatitude().asDm7())); tags.put("longitude", String.valueOf(location.getLongitude().asDm7())); final StringList parentRelations = new StringList(); this.relations().forEach(relation -> { final RelationMember member = relation.members().get(getIdentifier(), getType()); parentRelations.add(member.getRelationIdentifier() + "-" + member.getRole()); }); if (!parentRelations.isEmpty()) { tags.put("parentRelations", parentRelations.join(", ")); } return new GeoJsonBuilder.LocationIterableProperties(getRawGeometry(), tags); } @Override public byte[] toWkb() { return this.getLocation().toWkb(); } @Override public String toWkt() { return this.getLocation().toWkt(); } @Override public boolean within(final GeometricSurface surface) { return this.getLocation().within(surface); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Node.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Supplier; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.atlas.Atlas; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * Navigable Node * * @author matthieun * @author Yazad Khambata */ public abstract class Node extends LocationItem { private static final long serialVersionUID = 2082593591946379000L; protected Node(final Atlas atlas) { super(atlas); } /** * @return The absolute valence, considering all {@link Edge}s, irrespective of * bi-directionality. */ public long absoluteValence() { return this.connectedEdges().size(); } public SortedSet connectedEdges() { final SortedSet result = new TreeSet<>(); result.addAll(inEdges()); result.addAll(outEdges()); return result; } /** * Get the appropriate set {@link Edge}s of {@link ConnectedEdgeType}. * * @param connectedEdgeType * - The type of {@link Edge}-{@link Node} connection. * @return - A set of {@link Edge}s connected to the {@link Node} of {@link ConnectedEdgeType}. */ public SortedSet connectedEdges(final ConnectedEdgeType connectedEdgeType) { Validate.notNull(connectedEdgeType); final SortedSet connectedEdges = connectedEdgeType.getAccessFunction().apply(this); return connectedEdges; } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = super.getGeoJsonProperties(); final JsonArray inEdgesArray = new JsonArray(); final JsonArray outEdgesArray = new JsonArray(); for (final Edge edge : this.inEdges()) { inEdgesArray.add(new JsonPrimitive(edge.getIdentifier())); } for (final Edge edge : this.outEdges()) { outEdgesArray.add(new JsonPrimitive(edge.getIdentifier())); } // Adding a JSON array with the edge IDs. // In the RFC spec, nested objects are ok in properties. // https://tools.ietf.org/html/rfc7946#section-1.5 properties.add(ConnectedEdgeType.IN.getPropertyName(), inEdgesArray); properties.add(ConnectedEdgeType.OUT.getPropertyName(), outEdgesArray); return properties; } @Override public ItemType getType() { return ItemType.NODE; } /** * @return The {@link Edge}s that end at this node */ public abstract SortedSet inEdges(); /** * @return The {@link Edge}s that start at this node */ public abstract SortedSet outEdges(); @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); return "[Node: id=" + this.getIdentifier() + ", location=" + this.getLocation() + ", inEdges=" + connectedEdgesIdentifiers(() -> inEdges()) + ", outEdges=" + connectedEdgesIdentifiers(() -> outEdges()) + ", relations=(" + relationsString + "), " + tagString() + "]"; } @Override public String toString() { return "[Node: id=" + this.getIdentifier() + ", location=" + this.getLocation() + ", inEdges=" + connectedEdgesIdentifiers(() -> inEdges()) + ", outEdges=" + connectedEdgesIdentifiers(() -> outEdges()) + ", " + tagString() + "]"; } /** * @return The valence considering only the main {@link Edge}s */ public long valence() { return this.connectedEdges().stream().filter(Edge::isMainEdge).count(); } private SortedSet connectedEdgesIdentifiers( final Supplier> getConnectedEdges) { final SortedSet result = new TreeSet<>(); getConnectedEdges.get().forEach(edge -> result.add(edge.getIdentifier())); return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Point.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.openstreetmap.atlas.geography.atlas.Atlas; /** * A Point that is not navigable. * * @author matthieun */ public abstract class Point extends LocationItem { private static final long serialVersionUID = -7888952319754555424L; protected Point(final Atlas atlas) { super(atlas); } @Override public ItemType getType() { return ItemType.POINT; } @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); return "[Point: id=" + this.getIdentifier() + ", location=" + this.getLocation() + ", relations=(" + relationsString + "), " + tagString() + "]"; } @Override public String toString() { return "[Point: id=" + this.getIdentifier() + ", location=" + this.getLocation() + ", " + tagString() + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/README.md ================================================ # Atlas Items ## TurnRestriction A TurnRestriction is a Route that is restricted based on an [OSM turn restriction](http://wiki.openstreetmap.org/wiki/Relation:restriction) relation. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Relation.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.Deque; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.locationtech.jts.geom.GeometryFactory; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.WktPrintable; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonFeatureCollection; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.geojson.GeoJsonUtils; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonObject; /** * An OSM relation * * @author matthieun * @author Sid * @author hallahan * @author Yazad Khambata */ public abstract class Relation extends AtlasEntity implements Iterable, GeoJsonFeatureCollection { /** * The ring type of a {@link MultiPolygon} member. * * @author matthieun */ public enum Ring { OUTER, INNER } public static final Comparator RELATION_ID_COMPARATOR = Comparator .comparingLong(AtlasObject::getIdentifier); private static final Logger logger = LoggerFactory.getLogger(Relation.class); private static final long serialVersionUID = -9013894610780915685L; private static final RelationOrAreaToMultiPolygonConverter MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final JtsMultiPolygonToMultiPolygonConverter JTS_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private org.locationtech.jts.geom.MultiPolygon geom; private boolean badGeom = false; private Rectangle bounds = null; protected Relation(final Atlas atlas) { super(atlas); } /** * @return All the members of this relation's OSM ancestor. If this relation has not been * sliced, then this will return the same as members(). If this relation is * sliced, and is part of a pool of other relations that belong to the same OSM * ancestor, this method will pool together all the members of all those relations in * its Atlas. */ public abstract RelationMemberList allKnownOsmMembers(); public abstract List allRelationsWithSameOsmIdentifier(); @Override public JsonObject asGeoJson() { return GeoJsonUtils.feature(this); } @Override public JsonObject asGeoJsonGeometry() { // Can't be final due to catch block may reassign. JsonObject geometry; // We should only be writing relations as GeoJSON when they are polygons and multipolygons. // We want multipolygons, but not boundaries, as we can render boundaries' ways by // themselves fine. // The isMultiPolygon() method also includes boundaries, which we do not want. if (this.isGeometric()) { try { final MultiPolygon multiPolygon = MULTI_POLYGON_CONVERTER.convert(this); geometry = multiPolygon.asGeoJsonGeometry(); } // It seems like we get caught in this exception a lot! We don't ingest coastline // features, so polygons that touch coastlines will fail. It's good to include // the exception in the data, along with the bounding box. That way, we can // notice the problem when browsing the map. catch (final CoreException exception) { final String message = String.format("%s - %s", exception.getClass().getSimpleName(), exception.getMessage()); logger.error("Unable to recreate multipolygon for relation {}. {}", getIdentifier(), message); geometry = GeoJsonUtils.boundsToPolygonGeometry(bounds()); } } // Otherwise, we'll fall back to just providing the properties of the relation with the // bounding box as a polygon geometry. else { geometry = GeoJsonUtils.boundsToPolygonGeometry(bounds()); } return geometry; } public Optional asMultiPolygon() { return this.asMultiPolygon(false); } public Optional asMultiPolygon(final boolean assemble) { try { if (assemble && !this.badGeom && this.geom == null && isGeometric()) { this.geom = JTS_CONVERTER.backwardConvert(MULTI_POLYGON_CONVERTER.convert(this)); } } catch (final Exception exc) { logger.trace("Exception making multipolygon geometry for relation {}", this.getIdentifier(), exc); this.badGeom = true; } return Optional.ofNullable(this.geom); } @Override public Rectangle bounds() { final Optional geometry = this.asMultiPolygon(); if (!this.getBadGeom() && geometry.isPresent()) { if (this.bounds == null) { this.bounds = Rectangle.forLocated(new JtsPolygonConverter() .backwardConvert((org.locationtech.jts.geom.Polygon) new GeometryFactory() .toGeometry(geometry.get().getEnvelopeInternal()))); } return this.bounds; } return boundsInternal(new LinkedHashSet<>()); } public String configurableString(final String betweenEachMemberAndRelation, final String betweenEachMember) { final StringBuilder builder = new StringBuilder(); builder.append("[Relation: id="); builder.append(getIdentifier()); builder.append(", [Members: \n\t\t\t\t"); final StringList list = new StringList(); for (final RelationMember member : this) { list.add(betweenEachMemberAndRelation + betweenEachMember + member.toString()); } builder.append(list.join(", \n\t\t\t\t")); builder.append("\n\t\t\t"); builder.append(betweenEachMemberAndRelation); builder.append("], "); builder.append(tagString()); final Optional multipolygon = this.asMultiPolygon(); if (multipolygon.isPresent()) { builder.append(", multipolygon="); builder.append(multipolygon.get().toText()); } builder.append("]"); return builder.toString(); } /** * "Flattens" the relation by returning the set of non-Relation members. Adds any non-Relation * members to the set, then loops on any Relation members to add their non-Relation members as * well. Keeps track of Relations whose identifiers have already been operated on, so that * recursively defined relations don't cause problems. * * @return a Set of AtlasObjects all related to this Relation, with no Relations. */ public Set flatten() { final Deque toProcess = new LinkedList<>(); final Set relationsSeen = new HashSet<>(); AtlasObject polledMember; final Set relationMembers = new HashSet<>(); toProcess.add(this); while (!toProcess.isEmpty()) { polledMember = toProcess.poll(); if (polledMember instanceof Relation) { if (relationsSeen.contains(polledMember.getIdentifier())) { continue; } ((Relation) polledMember).members() .forEach(member -> toProcess.add(member.getEntity())); relationsSeen.add(polledMember.getIdentifier()); } else { relationMembers.add(polledMember); } } return relationMembers; } /** * "Flattens" the relation by returning the set of child Relation members, recursively. * * @return a Set of IDs for all sub Relations. */ public Set flattenRelations() { final Deque toProcess = new LinkedList<>(); final Set subrelations = new HashSet<>(); AtlasObject polledMember; toProcess.add(this); while (!toProcess.isEmpty()) { polledMember = toProcess.poll(); if (polledMember instanceof Relation) { if (subrelations.contains(polledMember.getIdentifier())) { continue; } ((Relation) polledMember).members() .forEach(member -> toProcess.add(member.getEntity())); subrelations.add(polledMember.getIdentifier()); } } return subrelations; } /** * @return the {@link RelationBean} representation of the Relation */ public RelationBean getBean() { final RelationBean bean = new RelationBean(); for (final RelationMember member : this) { final AtlasEntity entity = member.getEntity(); bean.addItem(entity.getIdentifier(), member.getRole(), entity.getType()); } return bean; } @Override public Iterable getGeoJsonObjects() { return this; } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = super.getGeoJsonProperties(); addMembersToProperties(properties); return properties; } public JsonObject getGeoJsonPropertiesWithoutMembers() { return super.getGeoJsonProperties(); } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.FEATURE; } @Override public ItemType getType() { return ItemType.RELATION; } public boolean hasMultiPolygonMembers(final Ring ring) { if (isGeometric()) { for (final RelationMember member : members()) { switch (ring) { case OUTER: if (RelationTypeTag.MULTIPOLYGON_ROLE_OUTER.equals(member.getRole())) { return true; } break; case INNER: if (RelationTypeTag.MULTIPOLYGON_ROLE_INNER.equals(member.getRole())) { return true; } break; default: throw new CoreException("Unknown ring type: {}", ring); } } } return false; } @Override public boolean intersects(final GeometricSurface surface) { return intersectsInternal(surface, new LinkedHashSet<>()); } public boolean isGeometric() { return Validators.isOfType(this, RelationTypeTag.class, RelationTypeTag.MULTIPOLYGON, RelationTypeTag.BOUNDARY, RelationTypeTag.LAND_AREA); } public boolean isMultiPolygon() { return Validators.isOfType(this, RelationTypeTag.class, RelationTypeTag.MULTIPOLYGON); } @Override public Iterator iterator() { return members().iterator(); } /** * @return All the members of this specific (potentially sliced) relation. */ public abstract RelationMemberList members(); /** * Get a subset of {@link #members()} matching the predicate. * * @param predicate * - the predicate to filter on. * @return - {@link #members()} matching the predicate. */ public RelationMemberList membersMatching(final Predicate predicate) { if (this.members() == null) { return new RelationMemberList(new ArrayList<>()); } return members().stream().filter(predicate).collect(RelationMemberList.collect()); } /** * Get a subset of {@link #members()} matching a certain {@link ItemType}. * * @param itemTypes * - the types of members to filter. * @return - {@link #members()} of type itemType. */ public RelationMemberList membersOfType(final ItemType... itemTypes) { final List> itemTypePredicates = Arrays.stream(itemTypes) .map(itemType -> { final Predicate relationMemberPredicate = member -> member .getEntity().getType() == itemType; return relationMemberPredicate; }).collect(Collectors.toList()); final RelationMemberList relationMemberList = itemTypePredicates.stream() .map(this::membersMatching).flatMap(RelationMemberList::stream) .collect(RelationMemberList.collect()); return relationMemberList; } /** * In case a {@link Relation} is spanning multiple {@link Atlas}, keep track of the parent OSM * relation identifier to be able to match it back to other sliced relations. * * @return The OSM identifier */ public abstract Long osmRelationIdentifier(); @Override public String toDiffViewFriendlyString() { final String relationsString = this.parentRelationsAsDiffViewFriendlyString(); final StringBuilder builder = new StringBuilder(); builder.append("[Relation: id="); builder.append(getIdentifier()); builder.append(", [Members: "); final StringList list = new StringList(); for (final RelationMember member : this) { list.add(member.toString()); } builder.append(list.join(", ")); builder.append("], "); builder.append("relations=(" + relationsString + "), "); builder.append(tagString()); builder.append("]"); return builder.toString(); } @Override public LocationIterableProperties toGeoJsonBuildingBlock() { final Map tags = getTags(); tags.put("identifier", String.valueOf(getIdentifier())); tags.put("osmIdentifier", String.valueOf(getOsmIdentifier())); tags.put("itemType", String.valueOf(getType())); tags.put("relation", this.toSimpleString()); final Optional shardName = getAtlas().metaData().getShardName(); shardName.ifPresent(shard -> tags.put("shard", shard)); return new GeoJsonBuilder.LocationIterableProperties(bounds().center(), tags); } public String toSimpleString() { final StringBuilder builder = new StringBuilder(); builder.append("[Relation: id="); builder.append(getIdentifier()); builder.append(", [Members: "); final StringList list = new StringList(); for (final RelationMember member : this) { list.add(member.toString()); } builder.append(list.join(", ")); builder.append("], "); builder.append(tagString()); builder.append("]"); return builder.toString(); } @Override public String toString() { return configurableString("", ""); } @Override public byte[] toWkb() { throw new UnsupportedOperationException("Relation.toWkb not implemented yet."); } @Override public String toWkt() { final Optional geom = this.asMultiPolygon(); if (geom.isPresent()) { return geom.get().toText(); } return WktPrintable.toWktCollection(leafMembers().collect(Collectors.toList())); } /** * Return {@code true} if this Relation has all members fully within the supplied * {@link GeometricSurface}. * * @param surface * The {@link GeometricSurface} to check for * @return {@code true} if the relation has all members within the given * {@link GeometricSurface} */ @Override public boolean within(final GeometricSurface surface) { return withinInternal(surface, new LinkedHashSet<>()); } /** * Avoid stack overflows in case a relation has looping members. This should never happen with a * {@link PackedAtlas} but could happen when two {@link Atlas} are combined into a * {@link MultiAtlas}. * * @param parentRelationIdentifiers * The identifiers of the parent relations that have already been visited. * @return The bounds */ protected Rectangle boundsInternal(final Set parentRelationIdentifiers) { if (this.members().isEmpty()) { return Rectangle.MINIMUM; } final List itemsToConsider = new ArrayList<>(); for (final AtlasEntity member : Iterables.stream(this).map(RelationMember::getEntity) .filter(Objects::nonNull)) { if (member instanceof Relation) { final long identifier = member.getIdentifier(); if (parentRelationIdentifiers.contains(identifier)) { continue; } else { parentRelationIdentifiers.add(identifier); itemsToConsider .add(((Relation) member).boundsInternal(parentRelationIdentifiers)); } } else { final Rectangle memberBounds = member.bounds(); if (memberBounds != null) { itemsToConsider.add(memberBounds); } } } if (Iterables.size(itemsToConsider) == 0) { return Rectangle.MINIMUM; } return Rectangle.forLocated(itemsToConsider); } protected boolean getBadGeom() { return this.badGeom; } protected org.locationtech.jts.geom.MultiPolygon getGeom() { return this.geom; } /** * Avoid stack overflows in case a relation has looping members. This should never happen with a * {@link PackedAtlas} but could happen when two {@link Atlas} are combined into a * {@link MultiAtlas}. * * @param surface * The {@link GeometricSurface} to check for * @param parentRelationIdentifiers * The identifiers of the parent relations that have already been visited. * @return True if the relation intersects the geometricSurface */ protected boolean intersectsInternal(final GeometricSurface surface, final Set parentRelationIdentifiers) { for (final RelationMember member : this) { final AtlasEntity entity = member.getEntity(); if (entity instanceof Relation) { final long identifier = entity.getIdentifier(); if (parentRelationIdentifiers.contains(identifier)) { continue; } else { parentRelationIdentifiers.add(identifier); if (((Relation) entity).intersectsInternal(surface, parentRelationIdentifiers)) { return true; } } } else if (entity.intersects(surface)) { return true; } } return false; } protected void setGeom(final org.locationtech.jts.geom.MultiPolygon geom) { this.geom = geom; } /** * Avoid stack overflows in case a relation has looping members. This should never happen with a * {@link PackedAtlas} but could happen when two {@link Atlas} are combined into a * {@link MultiAtlas}. * * @param surface * The {@link GeometricSurface} to check for * @param parentRelationIdentifiers * The identifiers of the parent relations that have already been visited. * @return {@code true} if the relation has all members within the given * {@link GeometricSurface} */ protected boolean withinInternal(final GeometricSurface surface, final Set parentRelationIdentifiers) { for (final RelationMember member : this) { final AtlasEntity entity = member.getEntity(); if (entity instanceof Relation) { final long identifier = entity.getIdentifier(); if (parentRelationIdentifiers.contains(identifier)) { continue; } else { parentRelationIdentifiers.add(identifier); if (!((Relation) entity).withinInternal(surface, parentRelationIdentifiers)) { return false; } } } else if (isUnenclosedNonRelationEntity(surface, entity)) { return false; } } return true; } /** * We explicitly want to add member metadata to the properties of Relations, but only when we * are serializing relation entities. Overriding getGeoJsonProperties() would not work properly, * because that gets called when you are listing metadata about relations a non-relation entity * may be in. Calling this method, only in this class, avoids a recursive call that would list * members of relations in relation metadata for non-relation entities. * * @param properties * The JsonObject properties object we will add member metadata to. */ private void addMembersToProperties(final JsonObject properties) { final RelationMemberList members = members(); final JsonArray membersArray = new JsonArray(); properties.add("members", membersArray); for (final RelationMember member : members) { final JsonObject memberObject = new JsonObject(); membersArray.add(memberObject); final AtlasEntity entity = member.getEntity(); if (entity != null) { final long identifier = entity.getIdentifier(); memberObject.addProperty(GeoJsonUtils.IDENTIFIER, identifier); memberObject.addProperty("itemType", entity.getType().name()); } else { // We shouldn't get here, but if we do, let's know about it in the data... memberObject.addProperty(GeoJsonUtils.IDENTIFIER, "MISSING"); logger.warn("Missing identifier for relation entity: Relation ID: {}", getIdentifier()); } // Sometimes a member doesnt have a role. That's normal. final String role = member.getRole(); if (role != null) { // And sometimes the role is "", but we should keep it that way... memberObject.addProperty("role", role); } } } private boolean isUnenclosedArea(final AtlasEntity entity, final GeometricSurface surface) { return entity instanceof Area && !surface.fullyGeometricallyEncloses(((Area) entity).asPolygon()); } private boolean isUnenclosedLineItem(final AtlasEntity entity, final GeometricSurface surface) { return entity instanceof LineItem && !surface.fullyGeometricallyEncloses(((LineItem) entity).asPolyLine()); } private boolean isUnenclosedLocationItem(final AtlasEntity entity, final GeometricSurface surface) { return entity instanceof LocationItem && !surface.fullyGeometricallyEncloses(((LocationItem) entity).getLocation()); } private boolean isUnenclosedNonRelationEntity(final GeometricSurface surface, final AtlasEntity entity) { switch (entity.getType()) { case NODE: case POINT: return isUnenclosedLocationItem(entity, surface); case EDGE: case LINE: return isUnenclosedLineItem(entity, surface); case AREA: return isUnenclosedArea(entity, surface); case RELATION: default: throw new CoreException("Relations not supported in this method"); } } private Stream leafMembers() { final Stream nonRelationMembers = members().stream() .map(RelationMember::getEntity).filter(entity -> !(entity instanceof Relation)) .map(entity -> (AtlasItem) entity); final Stream relationMembers = members().stream().map(RelationMember::getEntity) .filter(entity -> entity instanceof Relation).map(entity -> (Relation) entity) .flatMap(Relation::leafMembers); return Stream.concat(nonRelationMembers, relationMembers); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/RelationMember.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.geojson.GeoJsonFeature; import com.google.gson.JsonObject; /** * A {@link Relation} member. It has a role and an {@link AtlasEntity}. * * @author matthieun */ public class RelationMember implements Comparable, Located, GeoJsonFeature { private final String role; private final AtlasEntity entity; private final long relationIdentifier; public RelationMember(final String role, final AtlasEntity entity, final long relationIdentifier) { this.role = role; this.entity = entity; this.relationIdentifier = relationIdentifier; } @Override public JsonObject asGeoJsonGeometry() { return this.entity.asGeoJsonGeometry(); } @Override public Rectangle bounds() { return this.entity.bounds(); } @Override public int compareTo(final RelationMember other) { // Order by type first, then by identifier, then by role final int itemTypeValue1 = this.getEntity().getType().getValue(); final int itemTypeValue2 = other.getEntity().getType().getValue(); final int deltaTypeValue = itemTypeValue1 - itemTypeValue2; if (deltaTypeValue > 0) { return 1; } else if (deltaTypeValue < 0) { return -1; } else { final long identifier1 = this.getEntity().getIdentifier(); final long identifier2 = other.getEntity().getIdentifier(); final int comparisonMarker = Long.compare(identifier1, identifier2); if (comparisonMarker != 0) { return comparisonMarker; } else { final String thisRole = this.getRole(); final String otherRole = other.getRole(); if (thisRole == null && otherRole == null) { return 0; } if (thisRole == null) { return -1; } if (otherRole == null) { return 1; } return thisRole.compareTo(otherRole); } } } @Override public boolean equals(final Object other) { if (other instanceof RelationMember) { final RelationMember that = (RelationMember) other; return StringUtils.equals(this.getRole(), that.getRole()) && this.getRelationIdentifier() == that.getRelationIdentifier() && this.entity.getIdentifier() == that.getEntity().getIdentifier(); } return false; } /** * @return The {@link AtlasEntity} pointed out by this relation. null if the {@link Atlas} that * created it has a sliced {@link Relation}. */ public AtlasEntity getEntity() { return this.entity; } @Override public JsonObject getGeoJsonProperties() { final JsonObject properties = this.entity.getGeoJsonProperties(); properties.addProperty("role", this.role); return properties; } public long getRelationIdentifier() { return this.relationIdentifier; } public String getRole() { return this.role; } @Override public int hashCode() { return this.role.hashCode() + this.entity.hashCode() + Long.hashCode(this.relationIdentifier); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("{Member: ID = "); builder.append(this.getEntity().getIdentifier()); builder.append(", Type = "); builder.append(this.getEntity().getType()); builder.append(", Role = "); builder.append(this.getRole()); builder.append("}"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/RelationMemberList.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.util.AbstractCollection; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.Function; import java.util.stream.Collector; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean.RelationBeanItem; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedRelation; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author matthieun * @author Yazad Khambata */ public class RelationMemberList extends AbstractCollection implements Located { private final List members; /** * This set has no concept of how many {@link RelationBeanItem}s of a given value have been * removed. Technically, OSM allows for duplicate {@link RelationBeanItem}s in a given relation. * However, these duplicates are disallowed by {@link PackedAtlas#relationMembers} and by * extension {@link PackedRelation#members}. As a result, we need not worry about that edge case * here. */ private final Set explicitlyExcluded; /** * A {@link Collectors#collectingAndThen(Collector, Function)} wrapper for * {@link RelationMember}s. * * @return - the collector. */ public static Collector collect() { return Collectors.collectingAndThen(Collectors.toList(), RelationMemberList::new); } public RelationMemberList(final Iterable members) { if (members instanceof List) { this.members = (List) members; } else { this.members = new ArrayList<>(); members.forEach(this.members::add); } this.explicitlyExcluded = new HashSet<>(); } @Override public boolean add(final RelationMember item) { return this.members.add(item); } public void addItemExplicitlyExcluded(final RelationBeanItem item) { this.explicitlyExcluded.add(item); } public RelationBean asBean() { final RelationBean result = new RelationBean(); for (final RelationMember member : this.members) { result.addItem(member.getEntity().getIdentifier(), member.getRole(), member.getEntity().getType()); } this.explicitlyExcluded.forEach(result::addItemExplicitlyExcluded); return result; } @Override public Rectangle bounds() { return Rectangle.forLocated(this.members); } @Override public boolean equals(final Object other) { if (other instanceof RelationMemberList) { final RelationMemberList that = (RelationMemberList) other; if (this.getMemberList().size() != that.getMemberList().size()) { return false; } int index = 0; for (final RelationMember thisMember : this.members) { final RelationMember thatMember = that.get(index++); if (thisMember == null && thatMember != null || thisMember != null && thatMember == null) { return false; } if (thisMember == null && thatMember == null) { continue; } if (!thisMember.equals(thatMember)) { return false; } } return true; } return false; } /** * Check if the two {@link RelationMemberList}s are the same, without looking at the List order. * Also, ensure that their explicitlyExcluded sets match. * * @param other * The other object * @return True if the other object satisfies {@link RelationMemberList#equals(Object)} AND has * a matching explicitlyExcluded set. */ public boolean equalsIncludingExplicitlyExcluded(final Object other) { if (other instanceof RelationMemberList) { final boolean basicEquals = this.equals(other); final RelationMemberList otherBean = (RelationMemberList) other; return basicEquals && this.explicitlyExcluded.equals(otherBean.explicitlyExcluded); } return false; } public RelationMember get(final int index) { if (index < 0 || index >= size()) { throw new CoreException( "No RelationMember with index {}. This list has only {} members.", index, size()); } return this.members.get(index); } public RelationMember get(final long identifier, final ItemType type) { for (final RelationMember member : this) { if (member.getEntity().getIdentifier() == identifier) { switch (type) { case NODE: if (member.getEntity() instanceof Node) { return member; } break; case EDGE: if (member.getEntity() instanceof Edge) { return member; } break; case AREA: if (member.getEntity() instanceof Area) { return member; } break; case LINE: if (member.getEntity() instanceof Line) { return member; } break; case POINT: if (member.getEntity() instanceof Point) { return member; } break; case RELATION: if (member.getEntity() instanceof Relation) { return member; } break; default: break; } } } return null; } @Override public int hashCode() { return this.members.hashCode(); } @Override public Iterator iterator() { return Iterables.filter(this.members, member -> member != null).iterator(); } @Override public int size() { return this.members.size(); } @Override public String toString() { return this.members.toString(); } private List getMemberList() { return this.members; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/Route.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.io.Serializable; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Heading; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A route is a set of {@link Edge}s that are connected to each other. * * @author matthieun * @author Sid * @author mgostintsev */ @SuppressWarnings("serial") public abstract class Route implements Iterable, Located, Serializable { /** * A {@link Route} implementation made of many {@link Edge}s. * * @author matthieun */ // NOSONAR here to override "Subclasses that add fields should override "equals" (squid:S2160)" // as the parent equals is good enough. private static final class MultiRoute extends Route // NOSONAR { private static final long serialVersionUID = -4562811506650155750L; private final Route upstream; private final Route downstream; private MultiRoute(final Route upstream, final Route downstream) { this.upstream = upstream; this.downstream = downstream; } @Override public PolyLine asPolyLine() { final PolyLine one = this.upstream.asPolyLine(); final PolyLine two = this.downstream.asPolyLine(); final List points = new ArrayList<>(); one.forEach(points::add); for (int i = 1; i < two.size(); i++) { points.add(two.get(i)); } return new PolyLine(points); } @Override public Rectangle bounds() { return Rectangle.forLocated(this.upstream, this.downstream); } @Override public Edge end() { return this.downstream.end(); } @Override public Edge get(final int index) { final int size = size(); if (index < 0 || index >= size) { throw new CoreException("Index {} out of Route's bounds: size = {}", index, size); } else { final int upstreamSize = this.upstream.size(); if (index < upstreamSize) { return this.upstream.get(index); } else { return this.downstream.get(index - upstreamSize); } } } @Override public int indexOf(final Edge edge) { final int indexUp = this.upstream.indexOf(edge); if (indexUp >= 0) { return indexUp; } final int indexDown = this.downstream.indexOf(edge); if (indexDown >= 0) { return this.upstream.size() + indexDown; } return indexDown; } @Override public Iterator iterator() { return new MultiIterable<>(this.upstream, this.downstream).iterator(); } @Override public Distance length() { return this.upstream.length().add(this.downstream.length()); } @Override public List nodes() { final List nodes = new ArrayList<>(); nodes.addAll(this.upstream.nodes()); final List downNodes = this.downstream.nodes(); for (int i = 1; i < downNodes.size(); i++) { nodes.add(downNodes.get(i)); } return nodes; } @Override public Optional reverse() { Route reversed = null; final Iterator iterator = this.iterator(); while (iterator.hasNext()) { final Edge edge = iterator.next(); if (edge.hasReverseEdge()) { final Edge reverse = edge.reversed().orElseThrow( () -> new CoreException("Edge should have a reverse edge.")); if (reversed == null) { reversed = Route.forEdge(reverse); } else { reversed = reversed.prepend(reverse); } } else { return Optional.empty(); } } return Optional.ofNullable(reversed); } @Override public int size() { return this.upstream.size() + this.downstream.size(); } @Override public Edge start() { return this.upstream.start(); } } /** * A {@link Route} made of a single {@link Edge} * * @author matthieun */ // NOSONAR here to override "Subclasses that add fields should override "equals" (squid:S2160)" // as the parent equals is good enough. private static final class SingleRoute extends Route // NOSONAR { private static final long serialVersionUID = -3870416343539125425L; private final Edge edge; SingleRoute(final Edge edge) { this.edge = edge; } @Override public PolyLine asPolyLine() { return this.edge.asPolyLine(); } @Override public Rectangle bounds() { return this.edge.bounds(); } @Override public Edge end() { return this.edge; } @Override public Edge get(final int index) { if (index != 0) { throw new CoreException("Invalid SingleRoute index: {}. Only 0 is permitted.", index); } return this.edge; } @Override public int indexOf(final Edge edge) { return this.edge.getIdentifier() == edge.getIdentifier() ? 0 : -1; } @Override public Iterator iterator() { return Iterables.from(this.edge).iterator(); } @Override public Distance length() { return this.edge.length(); } @Override public List nodes() { final List result = new ArrayList<>(); result.add(this.edge.start()); result.add(this.edge.end()); return result; } @Override public Optional reverse() { return this.edge.hasReverseEdge() ? Optional.of(new SingleRoute(this.edge.reversed() .orElseThrow(() -> new CoreException("Edge should have a reverse.")))) : Optional.empty(); } @Override public int size() { return 1; } @Override public Edge start() { return this.edge; } } /** * Comparator that sorts {@link Route}s from longest to shortest and then by individual * {@link Route} hashCode. */ public static final Comparator ROUTE_COMPARATOR = (final Route route1, final Route route2) -> new CompareToBuilder().append(route2.size(), route1.size()) .append(route1.hashCode(), route2.hashCode()).toComparison(); private static final Logger logger = LoggerFactory.getLogger(Route.class); /** * Given a set of {@link Edge}s, which may or may not have reverse {@link Edge}s, build a * {@link Route} that uses each unique {@link Edge} exactly once. Throws an exception if it * cannot build a {@link Route}. * * @param candidates * The {@link Edge}s * @param startNode * Starting {@link Node} of the {@link Route} * @param endNode * Ending {@link Node} of the {@link Route} * @return The corresponding {@link Route}. */ public static Route buildFullRouteIgnoringReverseEdges(final Set candidates, final Node startNode, final Node endNode) { Route route = null; int numberOfConsecutiveFailures = 0; final long maxEdgesToAdd = candidates.stream().map(edge -> edge.getMainEdgeIdentifier()) .distinct().count(); final Set idsAdded = new HashSet<>(); if (maxEdgesToAdd == 0) { throw new CoreException("Can't have a route with no members"); } while (route == null || route.size() < maxEdgesToAdd && !(route.start().start().equals(startNode) && route.end().end().equals(endNode))) { if (route == null) { // Find an edge that connects to the startNode for (final Edge edge : candidates) { if (edge.start().equals(startNode)) { route = Route.forEdge(edge); idsAdded.add(edge.getMainEdgeIdentifier()); break; } } if (route == null) { throw new CoreException( "Can't find an edge that connects to the startNode. StartNode: {} EndNode: {}", startNode.getIdentifier(), endNode.getIdentifier()); } } else { boolean edgeAdded = false; for (final Edge edge : candidates) { if (idsAdded.contains(edge.getMainEdgeIdentifier())) { // this edge or reverseEdge is already used, continue continue; } // Can use equals here, as the items all come from the same atlas. // Note: Here the order has a great importance. It is edge start to route // end before edge end to route start, otherwise all the self-intersecting // osm ways will not be able to create a route. In the case of MultiAtlas // re-creating ways that have been mis-way-sectioned at borders, this is because // the edges are sorted in ascending order and processed here in the same order. if (edge.start().equals(route.end().end())) { edgeAdded = true; numberOfConsecutiveFailures = 0; route = route.append(edge); idsAdded.add(edge.getMainEdgeIdentifier()); break; } } // To ensure there's no infinite loop, number of consecutive loops where an edge is // not added cannot exceed the total number of unique edges passed in if (!edgeAdded && ++numberOfConsecutiveFailures >= maxEdgesToAdd) { throw new CoreException( "No edge that connects to the current route. StartNode: {} EndNode: {}", startNode.getIdentifier(), endNode.getIdentifier()); } } } if (route.size() != maxEdgesToAdd) { throw new CoreException( "A route was found from start to end, but not every unique edge was used. StartNode: {} EndNode: {}", startNode.getIdentifier(), endNode.getIdentifier()); } return route; } /** * Create a {@link Route} from a single {@link Edge} * * @param edge * The {@link Edge} * @return The single-{@link Edge} {@link Route} */ public static Route forEdge(final Edge edge) { if (edge == null) { throw new CoreException("Cannot create a Route from a null Edge."); } return new SingleRoute(edge); } /** * Create a {@link Route} from an {@link Iterable} of {@link Edge}s that are already in the * proper order to be connected. * * @param edges * The {@link Edge}s to link in a {@link Route} * @return The corresponding {@link Route} */ public static Route forEdges(final Edge... edges) { return forEdges(Iterables.asList(edges)); } /** * Create a {@link Route} from an {@link Iterable} of {@link Edge}s that are already in the * proper order to be connected. * * @param edges * The {@link Edge}s to link in a {@link Route} * @return The corresponding {@link Route} */ public static Route forEdges(final Iterable edges) { if (!edges.iterator().hasNext()) { throw new CoreException("Cannot have no edges"); } int counter = 0; Route result = null; for (final Edge edge : edges) { if (counter == 0) { result = Route.forEdge(edge); } else { result = result.append(edge); } counter++; } return result; } public static Route forRoutes(final Iterable routes) { if (!routes.iterator().hasNext()) { throw new CoreException("Cannot have no edges"); } Route result = null; for (final Route route : routes) { if (result == null) { result = route; } else { result = result.append(route); } } return result; } public static Route forRoutes(final Route... routes) { return forRoutes(Iterables.asList(routes)); } /** * Get a {@link Route} from a set of {@link Edge}s, that we assume are connected. However, this * does not require the {@link Edge}s to be in any order. The order should be inferred by this * method. Throws an exception if it cannot build a {@link Route}. * * @param candidates * The {@link Edge}s * @param shuffle * When no {@link Route} is found on the first pass, if this is true, the set of * {@link Edge}s will be shuffled to find routes that might have been missed. This is * way slower. * @return The corresponding {@link Route}. */ public static Route fromNonArrangedEdgeSet(final Set candidates, final boolean shuffle) { Route route = null; int numberFailures = 0; final List members = new ArrayList<>(); members.addAll(candidates); if (members.isEmpty()) { throw new CoreException("Cannot have a route with no members"); } while (route == null || route.size() < members.size()) { if (route == null) { route = Route.forEdge(members.iterator().next()); } else { final int initialSize = route.size(); for (final Edge edge : members) { if (route.includes(edge)) { // this edge is already used, continue continue; } // Can use equals here, as the items all come from the same atlas. // Note: Here the order has a great importance. It is edge start to route // end before edge end to route start, otherwise all the self-intersecting // osm ways will not be able to create a route. In the case of MultiAtlas // re-creating ways that have been mis-way-sectioned at borders, this is because // the edges are sorted in ascending order and processed here in the same order. if (edge.start().equals(route.end().end())) { route = route.append(edge); break; } if (edge.end().equals(route.start().start())) { route = route.prepend(edge); break; } } if (initialSize + 1 != route.size()) { if (shuffle && ++numberFailures < candidates.size()) { // The user suggested that the algorithm is sensitive to which edge is the // first in case of loops // Try another first edge final Edge firstMember = members.remove(0); members.add(firstMember); // Make the loop restart route = null; } else { // Format and throw an exception. final StringList edges = new StringList(); final StringList debug = new StringList(); candidates.forEach(edge -> edges.add(edge.getIdentifier())); candidates.forEach(edge -> debug.add(edge.asPolyLine().toWkt())); throw new CoreException( "Unable to build a route from edges {}\nLocations:\n{}", edges.join(", "), debug.join("\n")); } } } } return route; } protected Route() { } public Route append(final Edge edge) { return append(Route.forEdge(edge)); } public Route append(final Route route) { if (route == null) { throw new CoreException( "Cannot append a route that is null to a route {} that ends at {}", this, this.end()); } if (!end().end().equals(route.start().start())) { throw new CoreException( "Cannot append a disconnected route:\nOne: {}\nAt: {}\nTo\nTwo: {}\nAt: {}", this, this.end(), route, route.start()); } return new MultiRoute(this, route); } public abstract PolyLine asPolyLine(); /** * @return All the {@link Edge}s connected to the start/end {@link Node}s of this {@link Route}, * excluding {@link Edge}s in {@link Route}. If this {@link Edge} is a {@link Route} * with two-way road, then the reversed {@link Edge}s will be included in the set. This * does not include {@link Edge} connected to the interior {@link Node}s of the * {@link Route}. */ public Set connectedEdges() { final Set result = new HashSet<>(); for (final Edge edge : this.end().end().connectedEdges()) { if (!this.includes(edge)) { result.add(edge); } } for (final Edge edge : this.start().start().connectedEdges()) { if (!this.includes(edge)) { result.add(edge); } } return result; } public abstract Edge end(); @Override public boolean equals(final Object other) { if (other instanceof Route) { final Route that = (Route) other; if (this.size() == that.size()) { return new EqualsBuilder() .append(this.start().start().getLocation(), that.start().start().getLocation()) .append(this.end().end().getLocation(), that.end().end().getLocation()) .append(Iterables.asList(this), Iterables.asList(that)).isEquals(); } } return false; } /** * @param index * An index in the {@link Route} * @return The {@link Edge} at the specified index in the {@link Route} */ public abstract Edge get(int index); /** * Note: The start and end {@link Node}s of the {@link Route} are part of the hash code to * reduce the probability of a collision. There are other candidates to add here, like distance * between start/end, but start/end by themselves are the least computationally intensive to * derive. */ @Override public int hashCode() { return new HashCodeBuilder().append(this.start().start().getLocation()) .append(this.end().end().getLocation()).append(Iterables.asList(this)).hashCode(); } /** * @param edge * The {@link Edge} to test for * @return true if the {@link Edge} provided belongs in the {@link Route} */ public boolean includes(final Edge edge) { return this.indexOf(edge) >= 0; } /** * @param edge * The {@link Edge} to test for * @return The index of the {@link Edge} in the {@link Route} if it is included, -1 otherwise. */ public abstract int indexOf(Edge edge); /** * Identifies whether the entire given {@link Route} is overlapping. * * @param route * The {@link Route} to check * @return true if the given {@link Route} is overlapping */ public boolean isOverlapping(final Route route) { return overlapIndex(route) > -1; } /** * Identifies whether any of the given {@link Route} is entirely overlapping this one. * * @param routes * The {@link Iterable} of {@link Route}s to check * @return true if any of the given {@link Route}s is overlapping */ public boolean isOverlappingForAtLeastOneOf(final Iterable routes) { return overlapIndex(routes) > -1; } /** * Identifies whether this @{link Route} is a simple U-Turn (route follows along a path to a * point and returns the exact same way it came in). *

* NOTE: A route could still be a U-Turn that doesn't follow an identical path out based on * {@link Heading}, but this method won't catch that case. * * @return true if this route is a simple U-Turn */ public boolean isSimpleUTurn() { final int numberOfEdges = this.size(); // A simple UTurn route cannot have an odd number of edges if (numberOfEdges % 2 == 1) { return false; } int index = 0; // Start by comparing the first and last edge, and incrementally move in from each side. // Stop after we compare the middle two edges while (index < numberOfEdges / 2) { // If any comparison doesn't match, it's not a simple U-Turn if (!this.get(index).isReversedEdge(this.get(numberOfEdges - index - 1))) { return false; } index++; } // If the comparisons all succeeded, it's a simple U-Turn! return true; } /** * Identifies whether the given {@link Route} is a sub-route of this one. If the given * {@link Route} is greater than this {@link Route}, this method will return false. * * @param route * The {@link Route} to check * @return true if the given {@link Route} is a sub-route */ public boolean isSubRoute(final Route route) { return subRouteIndex(route) > -1; } /** * Identifies whether any of the given {@link Route} is a sub-route of this one. * * @param routes * The {@link Iterable} of {@link Route}s to check * @return true if any of the given {@link Route}s is a sub-route. */ public boolean isSubRouteForAtLeastOneOf(final Iterable routes) { return subRouteIndex(routes) > -1; } /** * @return true if this {@link Route} contains a Turn Restriction given OSM's definition. */ public boolean isTurnRestriction() { return TurnRestriction.isTurnRestriction(this); } public abstract Distance length(); public abstract List nodes(); /** * Calculates the first occurring overlapping index from the given {@link Route}s. For details, * see {@link #overlapIndex(Route)}. * * @param routes * The {@link Route}s to compare with * @return first occurring calculated index */ public int overlapIndex(final Iterable routes) { for (final Route route : routes) { final int overlapIndex = overlapIndex(route); if (overlapIndex > -1) { return overlapIndex; } } return -1; } /** * Calculates the index of the last {@link Edge} from this {@link Route} that overlaps the given * {@link Route}. If there is no overlap, -1 is returned. Note: The given {@link Route} can be * of any size. Example 1 - this route: [A,B], given route: [A,B,C] will return 1 since the last * overlap occurs at Edge B, index 1 for this route. Example 2 - this route: [A,B,C], given * route: [C] will return 2, since overlap is at C, index 2 for this route. * * @param route * The {@link Route} to compare with * @return the calculated index */ public int overlapIndex(final Route route) { int overlapIndex; boolean givenPathIsLonger = false; // Find overlap index relative to this route, but use the longer of the two routes to find // the overlap section if (route.size() > this.size()) { givenPathIsLonger = true; overlapIndex = route.subRouteIndex(this); } else { overlapIndex = subRouteIndex(route); } // If there is an overlap and the given route was longer, go back and find the index for the // overlapping edge in this route if (givenPathIsLonger && overlapIndex > -1) { final Edge lastOverlap = route.get(overlapIndex); if (this.includes(lastOverlap)) { return this.indexOf(lastOverlap); } logger.error("Detected overlap at edge {}, but unable to find in current route {}", lastOverlap.getIdentifier(), this); overlapIndex = -1; } return overlapIndex; } public Route prepend(final Edge edge) { return prepend(Route.forEdge(edge)); } public Route prepend(final Route route) { if (!start().start().equals(route.end().end())) { throw new CoreException("Cannot prepend a disconnected route."); } return new MultiRoute(route, this); } /** * @return The reversed {@link Route}, if all the reversed {@link Edge}s exist. */ public abstract Optional reverse(); /** * @return The number of {@link Edge}s in this {@link Route} */ public abstract int size(); /** * @return The first {@link Edge} in this route */ public abstract Edge start(); /** * Determine if this route starts with the {@link Route} passed in. * * @param other * The {@link Route} to compare * @return true if this route starts with the route passed in */ public boolean startsWith(final Route other) { // If the other route is longer than this route, return false if (other.size() > this.size()) { return false; } final Iterator otherIterator = other.iterator(); final Iterator thisIterator = this.iterator(); // otherIterator has to be shorter than thisIterator, so loop over otherIterator while (otherIterator.hasNext()) { final Edge otherEdge = otherIterator.next(); final Edge thisEdge = thisIterator.next(); if (!thisEdge.equals(otherEdge)) { return false; } } return true; } /** * Creates a new {@link Route} from the original route based on the start and end indexes passed * in * * @param startIndex * The starting index to create the new route from * @param endIndex * The ending index (exclusive) of the new route * @return a new route based which is a subset of the original route */ public Route subRoute(final int startIndex, final int endIndex) { // Create a new ArrayList for safety reasons because subList returns a list backed by the // original list. return Route .forEdges(new ArrayList<>(Iterables.asList(this).subList(startIndex, endIndex))); } /** * Calculates the first occurring subRoute index from the given {@link Route}s. For details, see * {@link #subRouteIndex(Route)}. * * @param routes * The {@link Route}s to compare with * @return first occurring calculated index */ public int subRouteIndex(final Iterable routes) { for (final Route route : routes) { final int overlapIndex = subRouteIndex(route); if (overlapIndex > -1) { return overlapIndex; } } return -1; } /** * Calculates the index of the last {@link Edge} from this {@link Route} that overlaps the given * {@link Route}. If there is no overlap, -1 is returned. Note: The given {@link Route} must be * shorter than or equal to this {@link Route}. Example: This route: [A,B]. Given route: [A,B,C] * will return -1 since given route goes beyond this route. To avoid the size constraint, and * detect any overlap, use {@link #overlapIndex(Route)} instead. * * @param route * The {@link Route} to compare with * @return the calculated index */ public int subRouteIndex(final Route route) { // Keep track of the last index at which the last Edge was overlapping this route, to avoid // returning false positives in case of routes making a loop. int lastOverlapIndex = -1; // Fail-fast optimization if (route == null || route.size() > this.size()) { return -1; } for (final Edge routeEdge : route) { final int index = this.indexOf(routeEdge); if (index <= lastOverlapIndex) { // The edge does not overlap, or it does but at a smaller index which would indicate // a loop. return -1; } lastOverlapIndex = index; } return lastOverlapIndex; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[Route: "); final StringList edgeIdentifiers = new StringList(); this.forEach(edge -> edgeIdentifiers.add(String.valueOf(edge.getIdentifier()))); builder.append(edgeIdentifiers.join(", ")); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/SnappedEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.openstreetmap.atlas.geography.Snapper.SnappedLocation; /** * {@link SnappedLocation} on an {@link Edge} * * @author matthieun */ public class SnappedEdge extends SnappedLocation { private static final long serialVersionUID = 804405113068154275L; private final Edge edge; public SnappedEdge(final SnappedLocation snap, final Edge edge) { super(snap.getOrigin(), snap, edge.asPolyLine()); this.edge = edge; } public Edge getEdge() { return this.edge; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[SnappedEdge: Edge: "); builder.append(this.edge.getIdentifier()); builder.append(", Origin: "); builder.append(getOrigin()); builder.append(", Snap: "); builder.append(super.toString()); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/SnappedLineItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.geography.Snapper; /** * {@link Snapper.SnappedLocation} on an {@link LineItem} * * @author chunzc */ public class SnappedLineItem extends Snapper.SnappedLocation { private static final long serialVersionUID = 4935931723612324295L; private final LineItem lineItem; public SnappedLineItem(final Snapper.SnappedLocation snap, final LineItem lineItem) { super(snap.getOrigin(), snap, lineItem.asPolyLine()); this.lineItem = lineItem; } @Override public boolean equals(final Object object) { if (object == null || object.getClass() != this.getClass()) { return false; } final SnappedLineItem other = (SnappedLineItem) object; return new EqualsBuilder() .append(this.getLineItem().getIdentifier(), other.getLineItem().getIdentifier()) .append(this.getLineItem().asPolyLine(), other.getLineItem().asPolyLine()) .append(this.getLineItem().getTags(), other.getLineItem().getTags()).isEquals(); } public LineItem getLineItem() { return this.lineItem; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getLineItem().asPolyLine()) .append(this.getLineItem().getIdentifier()).append(this.getLineItem().getTags()) .toHashCode(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[SnappedLineItem: LineItem: "); builder.append(this.lineItem.getIdentifier()); builder.append(", Origin: "); builder.append(getOrigin()); builder.append(", Snap: "); builder.append(super.toString()); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/TurnRestriction.java ================================================ package org.openstreetmap.atlas.geography.atlas.items; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.TurnRestrictionTag; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * OSM Turn restriction, modeled from an {@link Atlas}. * * @author matthieun * @author sbhalekar */ public final class TurnRestriction implements Located, Serializable, Taggable { /** * The type of a {@link TurnRestriction} * * @author matthieun */ public enum TurnRestrictionType { NO, ONLY, OTHER } private static final Logger logger = LoggerFactory.getLogger(TurnRestriction.class); private static final long serialVersionUID = 7043701090121113526L; private Route from; private final Relation relation; private Route route; private Route too; private final TurnRestrictionType type; private Route via; private String invalidReason = null; /** * Create a {@link TurnRestriction} from a {@link Relation} * * @param relation * The {@link Relation} to use * @return An option on a {@link TurnRestriction}, which is filled if the {@link Relation} could * be translated to a {@link TurnRestriction} */ public static Optional from(final Relation relation) { final TurnRestriction turnRestriction = new TurnRestriction(relation); if (turnRestriction.isValid()) { return Optional.of(turnRestriction); } else { return Optional.empty(); } } public static TurnRestrictionType getTurnRestrictionType(final Relation relation) { if (TurnRestrictionTag.isNoPathRestriction(relation)) { return TurnRestrictionType.NO; } else if (TurnRestrictionTag.isOnlyPathRestriction(relation)) { return TurnRestrictionType.ONLY; } else { return TurnRestrictionType.OTHER; } } /** * Test if a {@link Route} is restricted. * * @param candidate * The {@link Route} to test * @return true if the {@link Route} is restricted */ public static boolean isTurnRestriction(final Route candidate) { for (final Edge edge : candidate) { final Set relations = edge.relations(); for (final Relation relation : relations) { if (TurnRestrictionTag.isRestriction(relation)) { final Optional turnRestrictionOption = TurnRestriction .from(relation); if (turnRestrictionOption.isPresent()) { final TurnRestriction turnRestriction = turnRestrictionOption.get(); final Route path = turnRestriction.route(); switch (turnRestriction.getTurnRestrictionType()) { case NO: // There is a "No" turn restriction on one edge of the path! if (path == null) { continue; } if (candidate.isOverlapping(path) && routeContainsAllTurnRestrictionParts(turnRestriction, candidate)) { // All the edges in the turn restriction's path are included in // the initial BigNode route and all of the turn restriction's // parts (to/via/from) are on the path, so we can flag it as a // turn restriction. The reason for the second piece of criteria // is to avoid false positives that may overlap or contain // pieces of the turn restriction path, but not the entire // thing. For example: if there is a BigNode route that overlaps // with a turn restriction's from and via, but not the to, then // we cannot say it's a turn restriction. return true; } break; case ONLY: // There is an "Only" turn restriction on one edge of the path // This is a tricky one. if (path == null) { continue; } final Route from = turnRestriction.getFrom(); /* * The path should be a turn restriction if all the following * criteria are met: 1. The path includes the from edge of the * turnRestriction 2. The path isn't identical to the * turnRestriction path (this is the ONLY case) 3. The path includes * an edge which is connected to the via Node 4. The edge from the * path that matches the from edge comes before the edge which is * connected to the via node (this matters for u-turn scenarios) */ // First make sure the from route is found within the candidate // route and make sure the specified path is not a // subRoute of the candidate route if (candidate.isSubRoute(from) && !candidate.isSubRoute(path)) { final int fromSubRouteIndex = candidate.subRouteIndex(from); final int routeEndIndex = candidate.size() - 1; if (fromSubRouteIndex == routeEndIndex) { // If the from edge is the last edge in the route, the route // should not be restricted as the only directive isn't // broken, so continue searching. continue; } // Create a partial route containing everything after the from // edge(s). // fromSubRouteIndex + 1 because the subRouteIndex returns the // index of the last edge in the route. To create a partial // route of everything after the from edge, we need the next // index after the last from edge. // routeEndIndex + 1 because subRoute's endingIndex is // exclusive, so we need the next index to get the desired // partialRoute final Route partialRoute = candidate .subRoute(fromSubRouteIndex + 1, routeEndIndex + 1); if (turnRestriction.otherToOptions().stream() .anyMatch(partialRoute::startsWith)) { // If the partial route starts with any other to options, // the only directive has not been followed and this path // should be restricted return true; } } // If the path fully overlaps the "only" restriction, do not return // true, as the path might belong to other restrictions break; case OTHER: // There are some other (not in "No" or "Only") cases that we ignore logger.trace("Not using Other TurnRestrictionType: {}", turnRestriction.getTurnRestrictionType()); return false; default: // Not in the NO, ONLY, or OTHER category: None expected unless // there is an expansion of TurnRestrictionTag enums throw new CoreException("Unknown TurnRestrictionType: {}", turnRestriction.getTurnRestrictionType()); } } } } } return false; } /** * @param turnRestriction * The {@link TurnRestriction} to use for comparison * @param route * The target {@link Route} to examine * @return {@code true} if the given {@link Route} contains all parts - via/from/to edges */ private static boolean routeContainsAllTurnRestrictionParts( final TurnRestriction turnRestriction, final Route route) { final Optional possibleVia = turnRestriction.getVia(); boolean viaMatches = true; if (possibleVia.isPresent()) { viaMatches = route.isSubRoute(possibleVia.get()); } return viaMatches && route.isSubRoute(turnRestriction.getTo()) && route.isSubRoute(turnRestriction.getFrom()); } public TurnRestriction(final Relation relation) { Route fromMember = null; Route viaMember = null; Route toMember = null; this.relation = relation; this.type = getTurnRestrictionType(relation); if (this.type == TurnRestrictionType.OTHER) { this.from = null; this.via = null; this.too = null; return; } try { if (!TurnRestrictionTag.isRestriction(relation)) { throw new CoreException("Relation {} is not a restriction.", relation); } // Try to re-build the route, based on the "from", "via" and "to" members // Build the via members final Set viaMembers = relation.members().stream() .filter(member -> member.getRole().equals(RelationTypeTag.RESTRICTION_ROLE_VIA)) .filter(member -> member.getEntity() instanceof Node || member.getEntity() instanceof Edge) .map(RelationMember::getEntity).map(entity -> (AtlasItem) entity) .collect(Collectors.toSet()); // According to OSM Wiki a restriction relation member can not have more than 1 via node // https://wiki.openstreetmap.org/wiki/Relation:restriction#Members // If the relation has more than 1 via node then discard the restriction as it is // incorrectly modeled. // To bring back the turn restriction OSM data needs to be modeled correctly final long viaNodeCount = viaMembers.stream() .filter(atlasItem -> atlasItem instanceof Node).count(); if (viaNodeCount > 1) { throw new CoreException( "Restriction relation should not have more than 1 via node. But, {} has {} via nodes", relation.getOsmIdentifier(), viaNodeCount); } // If there are no via members, create a temporary unfiltered set of "to" items, to help // with future filtering of "from" items by connectivity. final Set temporaryToMembers = new HashSet<>(); if (viaMembers.isEmpty()) { if (isSameRoadViaAndTo(relation)) { throw new CoreException( "Relation {} has same members in from and to, but has no via members to disambiguate.", relation.getIdentifier()); } relation.members().stream().filter( member -> member.getRole().equals(RelationTypeTag.RESTRICTION_ROLE_TO)) .forEach(member -> temporaryToMembers.add((AtlasItem) member.getEntity())); } // Filter the members to extract only the "from" members that are connected at the end // to via members if any, or to the start of "to" members. final Set fromMembers = new TreeSet<>(); relation.members().stream().filter(member -> member.getRole() .equals(RelationTypeTag.RESTRICTION_ROLE_FROM) && member.getEntity() instanceof Edge && (!viaMembers.isEmpty() && ((Edge) member.getEntity()).isConnectedAtEndTo(viaMembers) || ((Edge) member.getEntity()).isConnectedAtEndTo(temporaryToMembers))) .forEach(member -> fromMembers.add((Edge) member.getEntity())); fromMember = Route.fromNonArrangedEdgeSet(fromMembers, false); // Filter the members to extract only the "to" members that are connected at the // beginning to via members if any, or to the end of "from" members. final Set toMembers = new TreeSet<>(); relation.members().stream().filter(member -> member.getRole() .equals(RelationTypeTag.RESTRICTION_ROLE_TO) && member.getEntity() instanceof Edge && (!viaMembers.isEmpty() && ((Edge) member.getEntity()).isConnectedAtStartTo(viaMembers) || ((Edge) member.getEntity()).isConnectedAtStartTo(fromMembers))) .forEach(member -> toMembers.add((Edge) member.getEntity())); toMember = Route.fromNonArrangedEdgeSet(toMembers, false); // Take only the via members that are edges. Build this last to guarantee a route from // from to too. final Set viaEdges = viaMembers.stream().filter(member -> member instanceof Edge) .map(member -> (Edge) member).collect(Collectors.toSet()); if (!viaEdges.isEmpty()) { // It's possible that the via edge is bi-directional. To prevent both edges from // being put into the route, build a route using all unique edges once. This method // attempts to build a route from from the end of from to start of too. viaMember = Route.buildFullRouteIgnoringReverseEdges(viaEdges, fromMember.end().end(), toMember.start().start()); } this.from = fromMember; this.via = viaMember; this.too = toMember; // Make sure that the route can be built route(); } catch (final CoreException e) { this.invalidReason = e.getMessage(); logger.trace("Could not build TurnRestriction from relation {}", relation, e); this.from = null; this.via = null; this.too = null; } } @SuppressWarnings("deprecation") public LocationIterableProperties asGeoJson() { final Map tagsNo = Maps.hashMap("highway", "primary", "oneway", "yes", "type", "NO"); final Map tagsOnly = Maps.hashMap("highway", "primary", "oneway", "yes", "type", "ONLY"); return new LocationIterableProperties(this.route().asPolyLine(), this.getTurnRestrictionType() == TurnRestrictionType.NO ? tagsNo : tagsOnly); } @Override public Rectangle bounds() { if (!isValid()) { throw new CoreException("An invalid TurnRestriction cannot be Located."); } return route().bounds(); } /** * @return The "from" members of this {@link TurnRestriction} */ public Route getFrom() { return this.from; } public String getInvalidReason() { return this.invalidReason; } @Override public Optional getTag(final String key) { return this.relation.getTag(key); } @Override public Map getTags() { return new HashMap<>(this.relation.getTags()); } /** * @return The "to" members of this {@link TurnRestriction} */ public Route getTo() { return this.too; } public TurnRestrictionType getTurnRestrictionType() { return this.type; } public Optional getVia() { return Optional.ofNullable(this.via); } public boolean isValid() { return this.from != null && this.too != null && route() != null; } /** * @return The {@link TurnRestriction}'s full {@link Route} */ public Route route() { if (this.route == null) { final List routes = new ArrayList<>(); if (this.from != null) { routes.add(this.from); } if (this.via != null) { routes.add(this.via); } if (this.too != null) { routes.add(this.too); } try { this.route = Route.forRoutes(routes); } catch (final Exception e) { throw new CoreException("Can't build route from {}", this.relation, e); } } return this.route; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("["); builder.append(this.type.toString()); builder.append(": "); if (this.from != null) { final StringList froms = new StringList(); this.from.forEach(edge -> froms.add(String.valueOf(edge.getIdentifier()))); builder.append(froms.join(",")); } builder.append("_"); if (this.via != null) { final StringList vias = new StringList(); this.via.forEach(edge -> vias.add(String.valueOf(edge.getIdentifier()))); builder.append(vias.join(",")); } builder.append("_"); if (this.too != null) { final StringList tos = new StringList(); this.too.forEach(edge -> tos.add(String.valueOf(edge.getIdentifier()))); builder.append(tos.join(",")); } builder.append("]"); return builder.toString(); } /** * @return The set of {@link Route} that are emaning from the Via route, but which are not the * current To option. */ protected Set otherToOptions() { final Set result = new HashSet<>(); for (final Edge toEdge : this.too.start().start().outEdges()) { if (toEdge.equals(this.too.start())) { continue; } result.add(Route.forEdge(toEdge)); } return result; } private boolean isSameRoadViaAndTo(final Relation relation) { final Set fromIdentifiers = new TreeSet<>(); final Set toIdentifiers = new TreeSet<>(); relation.members().stream().filter(member -> "to".equals(member.getRole())) .forEach(member -> toIdentifiers.add(member.getEntity().getIdentifier())); relation.members().stream().filter(member -> "from".equals(member.getRole())) .forEach(member -> fromIdentifiers.add(member.getEntity().getIdentifier())); return fromIdentifiers.equals(toIdentifiers); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/ComplexEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.Serializable; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasObject; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Complex entity built on the fly from an existing {@link Atlas}. Examples include buildings with * holes (Relation type=multipolygon with inner and outer members, and a building=yes tag), lakes * with islands, etc. * * @author matthieun */ public abstract class ComplexEntity implements AtlasObject { /** * Validation errors are reported through this class to any interested callers * * @author cstaylor */ public static final class ComplexEntityError implements Serializable { private static final long serialVersionUID = 3162352792545207168L; private final transient ComplexEntity source; private final String reason; private final Throwable oops; public ComplexEntityError(final ComplexEntity source, final String reason) { this(source, reason, null); } public ComplexEntityError(final ComplexEntity source, final String reason, final Throwable oops) { this.source = source; this.reason = reason; this.oops = oops; } public Throwable getException() { return this.oops; } public String getReason() { return this.reason; } public ComplexEntity getSource() { return this.source; } @Override public String toString() { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final PrintStream stream = new PrintStream(baos); stream.printf("%s\n", this.source); stream.printf("OSM id: %d\n", this.source.getOsmIdentifier()); if (this.reason != null) { stream.printf("Why: %s\n", this.reason); } if (this.oops != null) { this.oops.printStackTrace(stream); } stream.close(); return new String(baos.toByteArray(), Charset.forName("UTF-8")); } } private static final long serialVersionUID = -553026286746440299L; private final Atlas atlas; private final AtlasEntity source; private Tuple invalidReason; protected ComplexEntity(final AtlasEntity source) { this.source = source; this.atlas = source.getAtlas(); } @Override public Rectangle bounds() { return getSource().bounds(); } @Override public boolean equals(final Object other) { if (other instanceof ComplexEntity) { return other.getClass().equals(this.getClass()) && this.getSource().equals(((ComplexEntity) other).getSource()); } return false; } public List getAllInvalidations() { final List returnValue = new ArrayList<>(); if (!isValid()) { getError().ifPresent(returnValue::add); } return returnValue; } @Override public Atlas getAtlas() { return this.atlas; } public Optional getError() { if (this.invalidReason == null) { return Optional.empty(); } return Optional.of(new ComplexEntityError(this, this.invalidReason.getFirst(), this.invalidReason.getSecond())); } @Override public long getIdentifier() { return getSource().getIdentifier(); } @Override public long getOsmIdentifier() { return this.source.getOsmIdentifier(); } public AtlasEntity getSource() { return this.source; } @Override public Optional getTag(final String tagName) { return getSource().getTag(tagName); } @Override public Map getTags() { return getSource().getTags(); } @Override public int hashCode() { return this.source.hashCode(); } /** * @return True if there are not any missing data in the Atlas to be able to properly build this * {@link ComplexEntity}. This is used so any Atlas does not offer any * {@link ComplexEntity} that would be compromised to an end user. */ public boolean isValid() { return this.invalidReason == null; } @Override public abstract String toString(); protected void setInvalidReason(final String reason, final Throwable oops) { this.invalidReason = new Tuple<>(reason, oops); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/Finder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.util.function.Consumer; import org.openstreetmap.atlas.geography.atlas.Atlas; import com.google.common.collect.Iterables; /** * @author matthieun * @author cstaylor * @param * the type of ComplexEntity we'll be searching for */ public interface Finder { /** * Helper method that can be used when searching a Finder so we skip any bad entities * * @param badEntity * the entity considered bad by the finder implementation * @param * the type of bad complex entity we want to ignore */ static void ignore(final T badEntity) { } /** * @param atlas * The {@link Atlas} to browse. * @return The simple entities first, then the complex ones. */ Iterable find(Atlas atlas); /** * Automatically filters out invalid complex entities and passes them to the badEntityConsumer * for further processing * * @param atlas * the atlas we're searching * @param badEntityConsumer * the consumer receiving each invalid complex entity * @return an iterable of only valid complex entities */ default Iterable find(final Atlas atlas, final Consumer badEntityConsumer) { return Iterables.filter(find(atlas), entity -> { final boolean wasEntityValid = entity.isValid(); if (!wasEntityValid) { badEntityConsumer.accept(entity); } return wasEntityValid; }); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/MultiPolygonRelationToMemberConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * @author Sid */ public class MultiPolygonRelationToMemberConverter implements Converter> { /** * @author Sid */ public enum Ring { OUTER, INNER } private final Ring ring; public MultiPolygonRelationToMemberConverter(final Ring ring) { this.ring = ring; } @Override public Iterable convert(final Relation relation) { if (!relation.isGeometric()) { throw new CoreException("Not a MultiPolygon: {}", relation); } final List list = new ArrayList<>(); for (final RelationMember member : relation.members()) { switch (this.ring) { case OUTER: if (RelationTypeTag.MULTIPOLYGON_ROLE_OUTER.equals(member.getRole())) { list.add(member.getEntity()); } break; case INNER: if (RelationTypeTag.MULTIPOLYGON_ROLE_INNER.equals(member.getRole())) { list.add(member.getEntity()); } break; default: throw new CoreException("Unknown ring type: {}", this.ring); } } return list; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/README.md ================================================ # Complex Entities Complex entities are higher level features made of relationships between Atlas features. Those complex entities are never stored in the Atlas directly, but rather built on demand using Finders. ## Some examples ### ComplexTurnRestriction One relation with type=restriction and from, via and to members. This is aggregated to build the restricted route and fails to build if the relation is malformed. ### ComplexBuilding One building with multiple parts, or a hole. The Finder reads the relations with the right tags and members, and provides a ComplexBuilding that can return the built-out `MultiPolygon` object. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/RelationOrAreaToMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.geom.prep.PreparedPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.Relation.Ring; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.conversion.Converter; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * Take in a {@link Relation} or an {@link Area} and return the corresponding {@link MultiPolygon} * if any. * * @author matthieun * @author bbreithaupt */ public class RelationOrAreaToMultiPolygonConverter implements Converter { private final RelationToMultiPolygonMemberConverter outerConverter; private final RelationToMultiPolygonMemberConverter innerConverter; public RelationOrAreaToMultiPolygonConverter() { this(false); } public RelationOrAreaToMultiPolygonConverter(final boolean usePolygonizer) { this.outerConverter = new RelationToMultiPolygonMemberConverter(Ring.OUTER, usePolygonizer); this.innerConverter = new RelationToMultiPolygonMemberConverter(Ring.INNER, usePolygonizer); } @Override public MultiPolygon convert(final AtlasEntity entity) { if (entity instanceof Relation) { final Relation relation = (Relation) entity; if (relation.isGeometric()) { // Loop through the relation members, extract the inners and outers, and create the // outline. final MultiMap outerToInners = new MultiMap<>(); for (final Polygon outer : this.outerConverter.convert(relation)) { outerToInners.put(outer, new ArrayList<>()); } if (outerToInners.isEmpty()) { throw new CoreException("Unable to find outer polygon."); } final Map preparedOuters = new HashMap<>(); final JtsPolygonConverter converter = new JtsPolygonConverter(); outerToInners.keySet().forEach( outer -> preparedOuters.put(outer, (PreparedPolygon) PreparedGeometryFactory .prepare(converter.convert(outer)))); for (final Polygon inner : this.innerConverter.convert(relation)) { boolean added = false; for (final Map.Entry entry : preparedOuters .entrySet()) { final org.locationtech.jts.geom.Polygon inner2 = converter.convert(inner); if (entry.getValue().containsProperly(inner2)) { outerToInners.add(entry.getKey(), inner); added = true; break; } } if (!added) { throw new CoreException( "Malformed MultiPolygon: inner has no outer host: {}", inner); } } return new MultiPolygon(outerToInners); } else { throw new CoreException("This is not a multipolygon relation"); } } else if (entity instanceof Area) { return MultiPolygon.forPolygon(((Area) entity).asPolygon()); } else { throw new CoreException("The outline is not an area nor a relation: {}", entity); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/RelationToMultiPolygonMemberConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.Relation.Ring; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.converters.MultiplePolyLineToPolygonsConverter; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * Get the {@link MultiPolygon} rings of a certain type from a {@link Relation} of type * multipolygon. * * @author matthieun */ public class RelationToMultiPolygonMemberConverter implements Converter> { private final MultiplePolyLineToPolygonsConverter multiplePolyLineToPolygonsConverter; private final Ring ring; public RelationToMultiPolygonMemberConverter(final Ring ring) { this(ring, false); } public RelationToMultiPolygonMemberConverter(final Ring ring, final boolean usePolygonizer) { this.ring = ring; this.multiplePolyLineToPolygonsConverter = new MultiplePolyLineToPolygonsConverter( usePolygonizer); } @Override public Iterable convert(final Relation relation) { final List candidates = new ArrayList<>(); final List alreadyFormed = new ArrayList<>(); if (!relation.isGeometric()) { throw new CoreException("Not a MultiPolygon: {}", relation); } final ArrayList members = new ArrayList<>(); relation.members().iterator().forEachRemaining(members::add); Collections.sort(members); for (final RelationMember member : members) { final AtlasEntity entity = member.getEntity(); switch (this.ring) { case OUTER: if (RelationTypeTag.MULTIPOLYGON_ROLE_OUTER.equals(member.getRole())) { processEntity(entity, candidates, alreadyFormed); } break; case INNER: if (RelationTypeTag.MULTIPOLYGON_ROLE_INNER.equals(member.getRole())) { processEntity(entity, candidates, alreadyFormed); } break; default: throw new CoreException("Unknown ring type: {}", this.ring); } } return new MultiIterable<>(alreadyFormed, this.multiplePolyLineToPolygonsConverter.convert(candidates)); } private void processEntity(final AtlasEntity entity, final List candidates, final List alreadyFormed) { if (entity instanceof Area) { // Easy alreadyFormed.add(((Area) entity).asPolygon()); } else if (entity instanceof LineItem) { // In case an Edge is an outer/inner, make sure to not double count it by looking at the // main edge only. if (!(entity instanceof Edge) || ((Edge) entity).isMainEdge()) { candidates.add(((LineItem) entity).asPolyLine()); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/WaterIslandConfigurationReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.islands.ComplexIsland; import org.openstreetmap.atlas.geography.atlas.items.complex.water.ComplexWaterEntity; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * Abstract class to define the configuration reader which will read in a json file and provide an * api to use that configuration and generate a {@link ComplexEntity} from {@link AtlasEntity}. * Currently this is used to generate {@link ComplexWaterEntity} or {@link ComplexIsland} * * @param * Type of objects which will map {@link AtlasEntity} to any {@link ComplexEntity} * @author sbhalekar */ public abstract class WaterIslandConfigurationReader implements Converter> { private final T configurationMapper; public WaterIslandConfigurationReader(final Resource configurationResource) { this.configurationMapper = readConfiguration(configurationResource); } @Override public Optional convert(final AtlasEntity atlasEntity) { return createComplexEntity(atlasEntity); } /** * This method will use the configuration mapper to convert {@link AtlasEntity} into * {@link ComplexEntity} * * @param atlasEntity * {@link AtlasEntity} which needs to be converted * @return An optional {@link ComplexEntity} */ @SuppressWarnings("squid:S1452") protected abstract Optional createComplexEntity( AtlasEntity atlasEntity); protected T getConfigurationMapper() { return this.configurationMapper; } /** * Implementation should read the resource and generate a mapper which will be used to convert * an {@link AtlasEntity} to {@link ComplexEntity} * * @param configurationResource * Resource for the configuration file * @return Mapper which would be used for the conversion */ protected abstract T readConfiguration(Resource configurationResource); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/aoi/ComplexAreaOfInterest.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.aoi; import java.io.InputStreamReader; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonParser; /** * Complex Entity for AOI Relations and AOI Areas. AOI relations are those that are * {@link MultiPolygon} and have AOI tags in it and AOI Areas are those {@link Area}s that have AOI * tags in it. AOI tags are checked against the {@link TaggableFilter} passed as an argument or to * the default {@link TaggableFilter} of AOI tags if the tags are not explicitly specified. * * @author sayas01 */ public final class ComplexAreaOfInterest extends ComplexEntity { private static final Logger logger = LoggerFactory.getLogger(ComplexAreaOfInterest.class); private static final RelationOrAreaToMultiPolygonConverter RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final String AOI_RESOURCE = "aoi-tag-filter.json"; // The default AreasOfInterest(AOI) tags private static List defaultTaggableFilter; private static final long serialVersionUID = 1191946548857888704L; private final MultiPolygon multiPolygon; /** * This method creates a {@link ComplexAreaOfInterest} for the specified {@link AtlasEntity} if * it meets the requirements for Complex AOI relation. The AOI tags are checked against the * default tags. * * @param source * The {@link AtlasEntity} for which the ComplexEntity is created * @return {@link ComplexAreaOfInterest} if created, else return empty. */ public static Optional getComplexAOI(final AtlasEntity source) { return getComplexAOI(source, customAoiFilter -> false); } /** * This method creates a {@link ComplexAreaOfInterest} for the specified {@link AtlasEntity} and * {@link TaggableFilter}. The AOI tags are checked against the aoiFilter param as well as the * default tags. * * @param source * The {@link AtlasEntity} for which the ComplexEntity is created * @param aoiFilter * The {@link TaggableFilter} of AOI tags against which the relation is checked for * AOI tags * @return {@link ComplexAreaOfInterest} if created, else return empty. */ public static Optional getComplexAOI(final AtlasEntity source, final Predicate aoiFilter) { try { if (defaultTaggableFilter == null) { computeDefaultFilter(); } return (source instanceof Relation || source instanceof Area) && (hasAOITag(source) || aoiFilter.test(source)) ? Optional.of(new ComplexAreaOfInterest(source)) : Optional.empty(); } catch (final Exception exception) { logger.warn("Unable to create complex AOI relations from {}", source, exception); return Optional.empty(); } } private static void computeDefaultFilter() { try (InputStreamReader reader = new InputStreamReader( ComplexAreaOfInterest.class.getResourceAsStream(AOI_RESOURCE))) { final JsonElement element = new JsonParser().parse(reader); final JsonArray filters = element.getAsJsonObject().get("filters").getAsJsonArray(); defaultTaggableFilter = StreamSupport.stream(filters.spliterator(), false) .map(jsonElement -> TaggableFilter.forDefinition(jsonElement.getAsString())) .collect(Collectors.toList()); } catch (final Exception exception) { throw new CoreException( "There was a problem parsing aoi-tag-filter.json. Check if the JSON file has valid structure.", exception); } } /** * Checks for AOI tags in the object * * @param source * {@link AtlasEntity} that needs to be checked for AOI tags * @return {@code true} if the source has AOI tags */ private static boolean hasAOITag(final AtlasEntity source) { return defaultTaggableFilter.stream() .anyMatch(taggableFilter -> taggableFilter.test(source)); } /** * Construct a {@link ComplexAreaOfInterest} * * @param source * the {@link AtlasEntity} to construct the ComplexAoiRelation */ private ComplexAreaOfInterest(final AtlasEntity source) { super(source); try { if (source.getType().equals(ItemType.RELATION)) { final Optional geom = ((Relation) source) .asMultiPolygon(); if (geom.isEmpty()) { throw new CoreException("No stored geometry for Relation {}", source); } this.multiPolygon = new JtsMultiPolygonToMultiPolygonConverter() .convert(geom.get()); } else { this.multiPolygon = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER.convert(source); } } catch (final Exception exception) { setInvalidReason("Unable to convert the AtlasEntity to MultiPolygon", exception); throw new CoreException("Unable to convert the AtlasEntity to MultiPolygon", exception); } } @Override public boolean equals(final Object other) { return other instanceof ComplexAreaOfInterest && super.equals(other); } public MultiPolygon getGeometry() { return this.multiPolygon; } @Override public int hashCode() { return super.hashCode(); } @Override public String toString() { return this.getClass().getName() + " " + getSource(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/aoi/ComplexAreaOfInterestFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.aoi; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * {@link ComplexAreaOfInterest} finder. * * @author sayas01 */ public class ComplexAreaOfInterestFinder implements Finder { /** * Finds all relations and areas that are candidates for {@link ComplexAreaOfInterest} and * converts them into {@link ComplexAreaOfInterest}. * * @param atlas * The {@link Atlas} to browse. * @return {@link Iterables} of {@link ComplexAreaOfInterest}. */ @Override public Iterable find(final Atlas atlas) { final Iterable iterableOfComplexAOIRelations = StreamSupport .stream(atlas.relations().spliterator(), true) .map(ComplexAreaOfInterest::getComplexAOI).filter(Optional::isPresent) .map(Optional::get).collect(Collectors.toList()); final Iterable iterableOfComplexAOIAreas = StreamSupport .stream(atlas.areas().spliterator(), true).map(ComplexAreaOfInterest::getComplexAOI) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); return new MultiIterable<>(iterableOfComplexAOIRelations, iterableOfComplexAOIAreas); } /** * Finds all relations and areas that are candidates for {@link ComplexAreaOfInterest} with * given AOI tags and converts them into {@link ComplexAreaOfInterest}. * * @param atlas * Atlas to build the ComplexAOiRelation * @param aoiFilter * {@link TaggableFilter} aoi taggable filter * @return {@link Iterables} of {@link ComplexAreaOfInterest}. */ public Iterable find(final Atlas atlas, final TaggableFilter aoiFilter) { final Iterable iterableOfComplexAOIRelations = StreamSupport .stream(atlas.relations().spliterator(), true) .map(relation -> ComplexAreaOfInterest.getComplexAOI(relation, aoiFilter)) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); final Iterable iterableOfComplexAOIAreas = StreamSupport .stream(atlas.areas().spliterator(), true) .map(area -> ComplexAreaOfInterest.getComplexAOI(area, aoiFilter)) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); return new MultiIterable<>(iterableOfComplexAOIRelations, iterableOfComplexAOIAreas); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/BigNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.bignode; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.routing.AStarRouter; import org.openstreetmap.atlas.geography.atlas.routing.AllPathsRouter; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * {@link BigNode} is a {@link ComplexEntity} that represents complex intersections, as one single * object. This is mostly used to model turn restrictions. A {@link BigNode} also provides the * different ways to traverse it as {@link Route} objects. * * @author matthieun * @author mgostintsev */ public class BigNode extends ComplexEntity { /** * This denotes the type path through a {@link BigNode}. Note: other than the start and end * edges, only junction edges are allowed in the path. * * @author mgostintsev */ public enum Path { SHORTEST, ALL } /** * This denotes the type of {@link BigNode} * * @author Sid */ public enum Type { // Intersections represented by a single node SIMPLE, // Dual Carriage Way Intersections represented by a set of nodes DUAL_CARRIAGEWAY } // Maximum allowed nodes for a BigNode public static final int MAXIMUM_NODES = 20; private static final long serialVersionUID = 4102278807908010498L; // BigNode type private Type type; // All the nodes defining this BigNode private final Set nodes; // Lazily initialized sets of edges making up this BigNode private Set edges; private Set inEdges; private Set outEdges; private Set junctionEdges; // Outside of the start/end edge, only junction edges are allowed to be part of a path leading // in/out of a BigNode. Because we're building a new atlas to do the routing one, we can't use // contains for the edge in question, but rely on the identifier to make sure it's actually a // junction edge. private final Predicate isJunctionEdge = edge -> this.junctionEdges().stream() .filter(junctionEdge -> junctionEdge.getIdentifier() == edge.getIdentifier()) .count() > 0; /** * Construct a {@link Type#SIMPLE} {@link BigNode} * * @param source * The source node */ public BigNode(final Node source) { super(source); final Set nodes = new HashSet<>(); nodes.add(source); this.nodes = nodes; this.type = Type.SIMPLE; } /** * Construct a {@link BigNode} * * @param source * The source node * @param nodes * All the nodes belonging to the {@link BigNode} * @param type * Type of the {@link BigNode} */ public BigNode(final Node source, final Set nodes, final Type type) { this(source, nodes); this.type = type; } /** * Construct a {@link BigNode} * * @param source * The source {@link Node} * @param nodes * All the {@link Node}s belonging to the {@link BigNode} */ protected BigNode(final Node source, final Set nodes) { super(source); this.nodes = nodes; } /** * @return The set of all possible {@link Route}s in and out of this {@link BigNode}. * {@link BigNode#shortestPaths} are a subset of this set. Note: this will NOT be cached * by the {@link BigNode}, so be judicious in how many times this is called. */ public Set allPaths() { return allPaths(AllPathsRouter.MAXIMUM_ALLOWED_PATHS); } /** * @param maximumPathCount * The maximum path count of each in out edge pair * @return The set of all possible {@link Route}s in and out of this {@link BigNode}. * {@link BigNode#shortestPaths} are a subset of this set. Note: this will NOT be cached * by the {@link BigNode}, so be judicious in how many times this is called. */ public Set allPaths(final int maximumPathCount) { final Atlas atlas = nodes().iterator().next().getAtlas(); final Atlas bigNodeAtlas = buildBigNodeAtlas(); // Find all the routes from all the in-edges to all the out-edges final Set allPaths = new HashSet<>(); for (final Edge inEdge : inEdges()) { for (final Edge outEdge : outEdges()) { // Translate edges to bigNodeAtlas edges to get all routes final Set bigNodeRoutes = AllPathsRouter.allRoutes( bigNodeAtlas.edge(inEdge.getIdentifier()), bigNodeAtlas.edge(outEdge.getIdentifier()), this.isJunctionEdge, maximumPathCount); if (bigNodeRoutes.isEmpty()) { continue; } // Translate the result routes back to routes made of edges from the original Atlas for (final Route bigNodeRoute : bigNodeRoutes) { Route route = Route.forEdge(atlas.edge(bigNodeRoute.start().getIdentifier())); for (int index = 1; index < bigNodeRoute.size(); index++) { final long edgeIdentifier = bigNodeRoute.get(index).getIdentifier(); route = route.append(atlas.edge(edgeIdentifier)); } allPaths.add(route); } } } return allPaths; } public GeoJsonObject asGeoJson() { return new GeoJsonBuilder().create(Iterables.from(asGeoJsonBigNode())); } public LocationIterableProperties asGeoJsonBigNode() { final List locations = new ArrayList<>(); nodes().stream().forEach(node -> locations .addAll(Rectangle.forLocated(node.getLocation()).expand(Distance.meters(2)))); return new LocationIterableProperties(new Polygon(locations), Maps.hashMap("BigNode", String.valueOf(getSource().getIdentifier()))); } public Iterable asGeoJsonRestrictedPath() { return this.turnRestrictions().stream() .map(turnRestriction -> new LocationIterableProperties( turnRestriction.getRoute().asPolyLine(), Maps.hashMap("highway", "motorway", "oneway", "yes", "route", turnRestriction.getRoute().toString()))) .collect(Collectors.toList()); } @Override public Rectangle bounds() { // Override the regular bounds() method that only looks at the source return Rectangle.forLocated(nodesAndEdges()); } /** * @return All the {@link Edge}s linked to this {@link BigNode} */ public Set edges() { if (this.edges == null) { this.edges = this.nodes.stream().flatMap(node -> { return node.connectedEdges().stream().filter(HighwayTag::isCarNavigableHighway); }).collect(Collectors.toSet()); } return this.edges; } @Override public boolean equals(final Object other) { // Override the regular equals(Object) method that only looks at the source if (other instanceof BigNode) { return this.getSource().equals(((BigNode) other).getSource()) && this.nodes().equals(((BigNode) other).nodes()); } return false; } public Set exteriorEdges() { return exteriorEdgesStream().collect(Collectors.toSet()); } @Override public List getAllInvalidations() { final List returnValue = new ArrayList<>(); if (!isValid()) { returnValue.add(new ComplexEntityError(this, String.format("Too many nodes %d", nodes().size()), null)); } return returnValue; } /** * @return the {@link Type} of {@link BigNode} this is. Can be null. */ public Type getType() { return this.type; } @Override public int hashCode() { // For checkstyle return super.hashCode(); } /** * @return All the {@link Edge}s that drive into the {@link BigNode} */ public Set inEdges() { if (this.inEdges == null) { this.inEdges = inEdgesStream().collect(Collectors.toSet()); } return this.inEdges; } @Override public boolean isValid() { return nodes().size() <= MAXIMUM_NODES; } /** * @return All the {@link Edge}s that represent internal junctions to the {@link BigNode} */ public Set junctionEdges() { if (this.junctionEdges == null) { this.junctionEdges = junctionEdgesStream().collect(Collectors.toSet()); } return this.junctionEdges; } /** * @return All the {@link Node}s forming this {@link BigNode} */ public Set nodes() { return this.nodes; } /** * @return All the {@link Node}s and {@link Edge}s forming this {@link BigNode} */ public Iterable nodesAndEdges() { return new MultiIterable<>(nodes(), edges()); } /** * @return All the {@link Edge}s that drive out of the {@link BigNode}. */ public Set outEdges() { if (this.outEdges == null) { this.outEdges = outEdgesStream().collect(Collectors.toSet()); } return this.outEdges; } /** * Set the {@link Type} for this {@link BigNode}. * * @param type * The {@link Type} to set */ public void setType(final Type type) { this.type = type; } /** * @return The set of shortest {@link Route}s in/out of this {@link BigNode}. Note: this will * NOT be cached by the {@link BigNode}, so be judicious in how many times this is * called. */ public Set shortestPaths() { final Atlas atlas = nodes().iterator().next().getAtlas(); final Atlas bigNodeAtlas = buildBigNodeAtlas(); // Find all the routes from all the in-edges to all the out-edges final Set shortestPaths = new HashSet<>(); for (final Edge inEdge : inEdges()) { for (final Edge outEdge : outEdges()) { // Translate edges to bigNodeAtlas edges to get the route final Route bigNodeRoute = AStarRouter.dijkstra(bigNodeAtlas, Distance.ONE_METER) .route(bigNodeAtlas.edge(inEdge.getIdentifier()), bigNodeAtlas.edge(outEdge.getIdentifier())); if (bigNodeRoute == null) { continue; } // Translate the result route back to a route made of edges from the original Atlas Route route = Route.forEdge(atlas.edge(bigNodeRoute.start().getIdentifier())); for (int index = 1; index < bigNodeRoute.size(); index++) { final long edgeIdentifier = bigNodeRoute.get(index).getIdentifier(); route = route.append(atlas.edge(edgeIdentifier)); } shortestPaths.add(route); } } return shortestPaths; } @Override public String toString() { return "[BigNode: nodes=" + nodes().stream().map(Node::getIdentifier).collect(Collectors.toSet()) + "]"; } /** * @return All the paths in this {@link BigNode} that overlap at least one restricted path. */ public Set turnRestrictions() { return this.allPaths().stream().filter(Route::isTurnRestriction) .map(route -> new RestrictedPath(this, route)).collect(Collectors.toSet()); } protected Stream exteriorEdgesStream() { return edges().stream() .filter(edge -> !nodes().contains(edge.end()) || !nodes().contains(edge.start())); } protected Stream inEdgesStream() { return edges().stream().filter(edge -> !nodes().contains(edge.start())); } protected Stream junctionEdgesStream() { return edges().stream() .filter(edge -> nodes().contains(edge.start()) && nodes().contains(edge.end())); } protected Stream outEdgesStream() { return edges().stream().filter(edge -> !nodes().contains(edge.end())); } /** * @return an {@link Atlas} made up of {@link Node}s and {@link Edge}s comprising this * {@link BigNode}, in order to run routing algorithms on it. */ private Atlas buildBigNodeAtlas() { final Set edges = edges(); final Set extendedNodes = new HashSet<>(); edges.forEach(edge -> { extendedNodes.add(edge.start()); extendedNodes.add(edge.end()); }); final PackedAtlasBuilder builder = new PackedAtlasBuilder() .withSizeEstimates(new AtlasSize(edges.size(), extendedNodes.size(), 0, 0, 0, 0)); extendedNodes.forEach( node -> builder.addNode(node.getIdentifier(), node.getLocation(), node.getTags())); edges.forEach( edge -> builder.addEdge(edge.getIdentifier(), edge.asPolyLine(), edge.getTags())); return builder.get(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/BigNodeFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.bignode; import static org.openstreetmap.atlas.tags.names.NameFinder.STANDARD_TAGS; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.CompareToBuilder; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Heading; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode.Type; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.JunctionTag; import org.openstreetmap.atlas.tags.names.NameFinder; import org.openstreetmap.atlas.tags.names.NameTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.direction.EdgeDirectionComparator; import org.openstreetmap.atlas.utilities.scalars.Angle; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Sets; /** * This finds all the {@link BigNode}s in an {@link Atlas} (including dual carriage way junctions * and simple intersections). * * @author sid * @author matthieun */ public class BigNodeFinder implements Finder { /** * This comparator is used to compare atlas node based on their identifier value */ public static final class NodeComparator implements Comparator, Serializable { private static final long serialVersionUID = 816401695743423872L; @Override public int compare(final Node node1, final Node node2) { return new CompareToBuilder().append(node1.getIdentifier(), node2.getIdentifier()) .toComparison(); } } /** * An intermediate {@link BigNode} candidate that is easily merged with other {@link BigNode}s * * @author Sid */ public static final class BigNodeCandidate implements Comparable, Serializable { private static final long serialVersionUID = 7225634482602225746L; private final Set nodeIdentifiers; public static BigNodeCandidate from(final Set nodes) { return new BigNodeCandidate( nodes.stream().map(Node::getIdentifier).collect(Collectors.toSet())); } public BigNodeCandidate() { this.nodeIdentifiers = new TreeSet<>(); } public BigNodeCandidate(final Set nodeIds) { this.nodeIdentifiers = new TreeSet<>(nodeIds); } @Override public int compareTo(final BigNodeCandidate bigNodeCandidate) { final Iterator iterator2 = bigNodeCandidate.nodeIdentifiers.iterator(); for (final Long identifier : this.nodeIdentifiers) { // Shorter sets sort first. if (!iterator2.hasNext()) { return 1; } final int comparison = identifier.compareTo(iterator2.next()); if (comparison != 0) { return comparison; } } // Shorter sets sort first if (iterator2.hasNext()) { return -1; } else { return 0; } } @Override public boolean equals(final Object other) { if (other instanceof BigNodeCandidate) { final BigNodeCandidate that = (BigNodeCandidate) other; if (this.nodeIdentifiers.size() == that.nodeIdentifiers.size()) { return new EqualsBuilder().append(this.nodeIdentifiers, that.nodeIdentifiers) .isEquals(); } } return false; } public Set getNodeIdentifiers() { return this.nodeIdentifiers; } public long getSourceNodeIdentifier() { if (!this.nodeIdentifiers.isEmpty()) { return this.nodeIdentifiers.iterator().next(); } throw new IllegalArgumentException( "Could not get source node identifier since nodesIdentifiers are empty"); } @Override public int hashCode() { return new HashCodeBuilder().append(this.nodeIdentifiers).hashCode(); } public void merge(final BigNodeCandidate mergeBigNodeCandidate) { this.nodeIdentifiers.addAll(mergeBigNodeCandidate.nodeIdentifiers); } } /** * @author Sid */ public class BigNodeIterable implements Iterable { private final BigNodeIterator iterator; public BigNodeIterable(final BigNodeIterator iterator) { this.iterator = iterator; } @Override public Iterator iterator() { return this.iterator; } } /** * The iterator avoids pre-computation of all the {@link BigNode}s. We maintain only the * dualCarriageWay {@link BigNode}s to reduce memory. * * @author Sid */ public class BigNodeIterator extends AbstractIterator { private final Iterator bigNodeCandidateIterator; private final Atlas atlas; private final Iterator nodeIterator; // We maintain a set of processed nodeIds to avoid returning duplicates private final Set bigNodeIdentifiers; public BigNodeIterator(final Atlas atlas, final Set bigNodeCandidates) { this.atlas = atlas; this.bigNodeCandidateIterator = bigNodeCandidates.iterator(); this.nodeIterator = this.atlas.nodes().iterator(); this.bigNodeIdentifiers = new HashSet<>(); } @Override protected BigNode computeNext() { while (this.bigNodeCandidateIterator.hasNext()) { final BigNodeCandidate bigNodeCandidate = this.bigNodeCandidateIterator.next(); // Sorting to ensure deterministic id final Set nodes = new TreeSet<>(new NodeComparator()); bigNodeCandidate.nodeIdentifiers .forEach(nodeIdentifier -> nodes.add(this.atlas.node(nodeIdentifier))); if (!nodes.isEmpty()) { final Node sourceNode = nodes.iterator().next(); final BigNode bigNode = new BigNode(sourceNode, nodes, Type.DUAL_CARRIAGEWAY); nodes.stream() .forEach(node -> this.bigNodeIdentifiers.add(node.getIdentifier())); return bigNode; } } while (this.nodeIterator.hasNext()) { // Next, look for simple intersections final Node candidateNode = this.nodeIterator.next(); if (!this.bigNodeIdentifiers.contains(candidateNode.getIdentifier())) { if (candidateNode.connectedEdges().stream() .anyMatch(HighwayTag::isCarNavigableHighway)) { this.bigNodeIdentifiers.add(candidateNode.getIdentifier()); return new BigNode(candidateNode); } } } // We reached the end of the list return endOfData(); } } /** * Minimum number of main edges involved in dual carriage way intersection */ private static final int MIN_MAIN_EDGE_DUAL_CARRIAGEWAY_INTERSECTION = 2; /** * LEVENSHTEIN limit is used for fuzzy name match */ private static final int LEVENSHTEIN_DISTANCE_THRESHOLD = 1; private static final Logger logger = LoggerFactory.getLogger(BigNodeFinder.class); /** * The limits below are used as upper bounds for length of the junction edge. The limits for the * junction edge are based not on road classification of the junction edge but the highest road * class of the connected edges of the junction edge. These limits might need some tuning. In * some cases like TestCase 6, the junction edge is not straight at an angle (hence longer * length). TODO : Experiment with max length of junction Route? */ private static final Distance SEARCH_RADIUS_MOTORWAY = Distance.meters(70); private static final Distance SEARCH_RADIUS_TRUNK = Distance.meters(60); private static final Distance SEARCH_RADIUS_PRIMARY = Distance.meters(50); private static final Distance SEARCH_RADIUS_SECONDARY = Distance.meters(40); private static final Distance SEARCH_RADIUS_TERTIARY = Distance.meters(35); private static final Distance SEARCH_RADIUS_RESIDENTIAL = Distance.meters(25); /** * This is maximum number of edges in dual carriage way route and is used as safety threshold to * prevent bad edge cases while constructing the big node */ private static final int MAXIMUM_DUAL_CARRIAGEWAY_ROUTE_SIZE = 10; /** * This is a maximum number of possible exploratory routes when searching for dual carriage way * junction routes. This is to prevent exponential slowdowns during rare edge cases, eg. when * OSM ways overlap. */ private static final int MAXIMUM_CANDIDATE_JUNCTION_ROUTE_SET_SIZE = 10_000; public static final String LOWEST_JUNCTION_EDGE_CANDIDATE_HIGHWAY_KEY = "LOWEST_JUNCTION_EDGE_CANDIDATE_HIGHWAY_TAG"; public static final String LONG_JUNCTION_ROUTE_LENGTH_KEY = "LONG_JUNCTION_ROUTE_LENGTH"; public static final String NON_STRAIGHT_JUNCTION_EDGES_ANGLE_KEY = "NON_STRAIGHT_JUNCTION_EDGES_ANGLE"; private Map radiusMap; private Map nonJunctionEdgeTagMap; private HighwayTag lowestJunctionEdgeCandidateHighwayTag = HighwayTag.SERVICE; // if junction route is longer than this length, we need to extra check to determine if it is // a straight long junction route which is considered to be invalid junction route private static final int LONG_JUNCTION_ROUTE_LENGTH = 100; private static final int NON_STRAIGHT_JUNCTION_EDGES_ANGLE = 60; private Distance longJunctionRouteLength = Distance.meters(LONG_JUNCTION_ROUTE_LENGTH); private Angle nonStraightJunctionEdgesAngle = Angle.degrees(NON_STRAIGHT_JUNCTION_EDGES_ANGLE); private final EdgeDirectionComparator edgeDirectionComparator = new EdgeDirectionComparator(); private final NameFinder nameFinder = new NameFinder().withTags(STANDARD_TAGS); public BigNodeFinder() { } public BigNodeFinder(final Map radiusMap, final Map nonJunctionEdgeTagMap, final Map configurationMap) { this.radiusMap = radiusMap; this.nonJunctionEdgeTagMap = nonJunctionEdgeTagMap; if (configurationMap != null) { configure(configurationMap); } } @Override public Iterable find(final Atlas atlas) { /* * Maintain a set of junction edges that are already part of a big Node. Sort the * junctionEdge ids for maintaining order consistency */ final Set junctionEdgeIds = new TreeSet<>(); /* * For junctionRouteEdges (sequence of edges that for a junctionRoute), maintain the first * edge */ final Set junctionRouteEdgeIds = new TreeSet<>(); // First pass through edges for (final Edge candidateEdge : atlas.edges(this::isCandidateJunctionEdge)) { // Check if the candidate edge is already part of another big Node if (!junctionEdgeIds.contains(candidateEdge.getIdentifier())) { if (isDualCarriageWayJunctionEdge(candidateEdge)) { junctionEdgeIds.add(candidateEdge.getIdentifier()); } else { // Expand Junction Edge to Junction Route before checking for Dual Carriage // way intersection final Optional junctionRoute = isDualCarriageWayJunctionRoute( Route.forEdge(candidateEdge)); junctionRoute.ifPresent(route -> { final Edge startEdge = route.start(); junctionRouteEdgeIds.add(startEdge.getIdentifier()); route.forEach(edge -> junctionEdgeIds.add(edge.getIdentifier())); }); } } } final Map nodeIdToBigNodeCandidateMap = new HashMap<>(); /* * There are some cases where a junction edge is bidirectional. We do not explicitly remove * the negative Edge in those cases. This may have to be revisited */ while (!junctionEdgeIds.isEmpty()) { final Long candidateEdgeId = junctionRouteEdgeIds.isEmpty() ? junctionEdgeIds.iterator().next() : junctionRouteEdgeIds.iterator().next(); final Edge candidate = atlas.edge(candidateEdgeId); final Route mergedCandidate = mergeJunctionEdges(Route.forEdge(candidate), junctionEdgeIds); logger.debug("Merged bigNode Route : {}. Number of Edges : {}", mergedCandidate, mergedCandidate.size()); if (!isStraightLongRoute(mergedCandidate)) { /* * mergedRoutes are formed after strict connectivity checks. But mergedRoutes can * share same node. Each node must have 1-1 mapping with a big node. So we merge * routes into same big node if they share a node */ final Set nodes = new HashSet<>(); mergedCandidate.forEach(edge -> nodes.addAll(edge.connectedNodes())); final Set bigNodesMergeCandidates = new HashSet<>(); for (final Node node : nodes) { if (nodeIdToBigNodeCandidateMap.containsKey(node.getIdentifier())) { bigNodesMergeCandidates .add(nodeIdToBigNodeCandidateMap.get(node.getIdentifier())); } } final BigNodeCandidate bigNodeCandidate = BigNodeCandidate.from(nodes); for (final BigNodeCandidate mergeBigNode : bigNodesMergeCandidates) { bigNodeCandidate.merge(mergeBigNode); } for (final Long nodeIdentifier : bigNodeCandidate.getNodeIdentifiers()) { nodeIdToBigNodeCandidateMap.put(nodeIdentifier, bigNodeCandidate); } } // Successfully added the bigNodeCandidate, can safely remove the junction edge mergedCandidate.forEach(edge -> junctionEdgeIds.remove(edge.getIdentifier())); mergedCandidate.forEach(edge -> junctionRouteEdgeIds.remove(edge.getIdentifier())); } final Set bigNodeCandidateSet = new HashSet<>( nodeIdToBigNodeCandidateMap.values()); logger.info( "Atlas has {} DualCarriageWay bigNodes with {} subNodes. Total Number of Big Nodes (including Simple Intersections) : {}", bigNodeCandidateSet.size(), nodeIdToBigNodeCandidateMap.keySet().size(), atlas.numberOfNodes() - nodeIdToBigNodeCandidateMap.keySet().size() + bigNodeCandidateSet.size()); final BigNodeIterator bigNodeIterator = new BigNodeIterator(atlas, bigNodeCandidateSet); return new BigNodeIterable(bigNodeIterator); } /** * Find the big nodes and save them as geojson in a resource * * @param atlas * The atlas to look at * @param writableResource * Where to save the geojson */ public void findAndSaveBigNodesAsGeoJson(final Atlas atlas, final WritableResource writableResource) { final List features = new ArrayList<>(); Iterables.stream(this.find(atlas)).map(BigNode::asGeoJsonBigNode).forEach(features::add); new GeoJsonBuilder().create(features).save(writableResource); } /** * Find restricted paths of all bignodes and save them as geojson in a resource * * @param atlas * The atlas to look at * @param writableResource * Where to save the geojson */ public void findAndSaveRestrictedPathsAsGeoJson(final Atlas atlas, final WritableResource writableResource) { final List features = new ArrayList<>(); Iterables.stream(this.find(atlas)).flatMap(BigNode::asGeoJsonRestrictedPath) .forEach(features::add); new GeoJsonBuilder().create(features).save(writableResource); } /* * Check if an edge contains any tags to be excluded as junction edges */ protected boolean hasJunctionEdgeTags(final Edge edge) { if (this.nonJunctionEdgeTagMap != null) { return this.nonJunctionEdgeTagMap.entrySet().stream() .noneMatch(entry -> edge.getTags().containsKey(entry.getKey()) && edge.getTags().get(entry.getKey()).equals(entry.getValue())); } return true; } // Check if merged junction candidate is a very long straight route which is normally not a // valid junction route protected boolean isStraightLongRoute(final Route mergedCandidate) { // if a junction route is merged route, it must have equal to or more than two junction // edges if (mergedCandidate.size() < 2) { return false; } // if merged candidate is longer than threshold, we will check if all junctions edges are // from the same long and straight road if (mergedCandidate.length().isGreaterThanOrEqualTo(this.longJunctionRouteLength)) { final Iterator iterator = mergedCandidate.iterator(); Edge currentEdge = iterator.next(); while (iterator.hasNext()) { final Edge nextEdge = iterator.next(); final Optional currentHeading = currentEdge.overallHeading(); final Optional nextHeading = nextEdge.overallHeading(); // if the angle of any two edges are large than 60 degree, we think they are valid // junction route if (currentHeading.isPresent() && nextHeading.isPresent() && currentHeading.get().difference(nextHeading.get()).asPositiveAngle() .isGreaterThanOrEqualTo(this.nonStraightJunctionEdgesAngle)) { return false; } // if there are any two edges has different name, we think they are valid if (currentEdge.getTag(NameTag.KEY).isPresent() && nextEdge.getTag(NameTag.KEY).isPresent() && !currentEdge.getTag(NameTag.KEY).get() .equalsIgnoreCase(nextEdge.getTag(NameTag.KEY).get())) { return false; } currentEdge = nextEdge; } if (logger.isInfoEnabled()) { if (mergedCandidate != null && mergedCandidate.asPolyLine() != null) { logger.info("Invalid Merged Candidate Route length is {} with WKT {}", mergedCandidate.length(), mergedCandidate.asPolyLine().toWkt()); } } return true; } else { return false; } } /* * Check if the start and end node of an edge connects to the same edge. Example is * https://www.openstreetmap.org/way/798542598 This type of edge should not be considered as * junction edge */ protected boolean startAndEndNodesConnectedToSameEdge(final Edge edge) { return edge.start().connectedEdges().stream() .filter(connectedEdge -> Math.abs(connectedEdge.getIdentifier()) != Math .abs(edge.getIdentifier())) .anyMatch(connectedEdge -> edge.end().connectedEdges().contains(connectedEdge)); } private void configure(final Map configurationMap) { if (configurationMap.get(LOWEST_JUNCTION_EDGE_CANDIDATE_HIGHWAY_KEY) != null) { this.lowestJunctionEdgeCandidateHighwayTag = HighwayTag.valueOf( configurationMap.get(LOWEST_JUNCTION_EDGE_CANDIDATE_HIGHWAY_KEY).toUpperCase()); } if (configurationMap.get(LONG_JUNCTION_ROUTE_LENGTH_KEY) != null) { this.longJunctionRouteLength = Distance.meters( Double.parseDouble(configurationMap.get(LONG_JUNCTION_ROUTE_LENGTH_KEY))); } if (configurationMap.get(NON_STRAIGHT_JUNCTION_EDGES_ANGLE_KEY) != null) { this.nonStraightJunctionEdgesAngle = Angle.degrees(Double .parseDouble(configurationMap.get(NON_STRAIGHT_JUNCTION_EDGES_ANGLE_KEY))); } } /** * Identify {@link Edge} name matches when both {@link Edge}s have same names. When strict mode * parameter is set to {@code true}, both edge names must be non-empty and must match. Exact * match is recommended for residential roads */ private boolean edgeNameExactMatch(final Edge edgeA, final Edge edgeB, final boolean strictMode) { final Optional edgeAName = this.nameFinder.best(edgeA); final Optional edgeBName = this.nameFinder.best(edgeB); if (edgeAName.isPresent() && edgeBName.isPresent()) { return nameExactMatch(edgeAName.get(), edgeBName.get()); } if (!strictMode && (!edgeAName.isPresent() || !edgeBName.isPresent())) { return true; } return false; } /** * Identify {@link Edge} name matches when both {@link Edge}s have same names. When strict mode * parameter is set to {@code true}, both edge names must be non-empty and must match. Instead * of exact match, we allow for fuzzy match. TODO : Improve normalization (remove cardinal * Directions before match?) */ private boolean edgeNameFuzzyMatch(final Edge edgeA, final Edge edgeB, final boolean strictMode) { final Optional edgeAName = this.nameFinder.best(edgeA); final Optional edgeBName = this.nameFinder.best(edgeB); if (edgeAName.isPresent() && edgeBName.isPresent()) { return nameFuzzyMatch(edgeAName.get(), edgeBName.get()); } if (!strictMode && (!edgeAName.isPresent() || !edgeBName.isPresent())) { return true; } return false; } /** * Test if an {@link Edge} is a candidate for expanding a {@link BigNode}, potentially becoming * a Junction Edge in the process. This filters off all the {@link Edge}s that are less * important than {@link HighwayTag#RESIDENTIAL}. It also filters off all the edges that are * likely roundabouts and link Roads. * * @param edge * The candidate {@link Edge} * @return {@code true} if the {@link Edge} is a candidate for expanding a {@link BigNode} */ private boolean isCandidateJunctionEdge(final Edge edge) { final HighwayTag highwayTag = edge.highwayTag(); return isShortEnough(edge) && highwayTag.isMoreImportantThanOrEqualTo( this.lowestJunctionEdgeCandidateHighwayTag) && hasJunctionEdgeTags(edge) && !JunctionTag.isRoundabout(edge) && !startAndEndNodesConnectedToSameEdge(edge); } private boolean isDualCarriageWayJunctionEdge(final Edge candidateEdge) { return isDualCarriageWayRoute(Route.forEdge(candidateEdge)); } private Optional isDualCarriageWayJunctionRoute(final Route candidateJunctionRoute) { final Set candidateJunctionRoutes = new LinkedHashSet<>(); candidateJunctionRoutes.add(candidateJunctionRoute); return isDualCarriageWayJunctionRoute(candidateJunctionRoutes); } /** * Look through all the edges around this node and expand if edge is connected to another short * Edge in same direction. */ private Optional isDualCarriageWayJunctionRoute(final Set candidateJunctionRoutes) { if (candidateJunctionRoutes.size() > MAXIMUM_CANDIDATE_JUNCTION_ROUTE_SET_SIZE) { logger.warn( "Aborting isDualCarriageWayJunctionRoute, candidate set size {} exceeded {}", candidateJunctionRoutes.size(), MAXIMUM_CANDIDATE_JUNCTION_ROUTE_SET_SIZE); return Optional.empty(); } if (candidateJunctionRoutes.isEmpty()) { return Optional.empty(); } // Maintain a set of expandable routes. final Set expandableJunctionRoutes = new LinkedHashSet<>(); for (final Route candidateJunctionRoute : candidateJunctionRoutes) { final Set expandableEdges = new LinkedHashSet<>(); candidateJunctionRoute.end().outEdges().stream().filter(this::isCandidateJunctionEdge) .filter(edge -> edge.getMainEdgeIdentifier() != candidateJunctionRoute.end() .getMainEdgeIdentifier()) .filter(edge -> this.edgeDirectionComparator .isSameDirection(candidateJunctionRoute.end(), edge, false)) .forEach(expandableEdges::add); for (final Edge expandableEdge : expandableEdges) { Route route = null; try { route = candidateJunctionRoute.append(expandableEdge); } catch (final CoreException e) { throw new CoreException("Could not append dual carriageway route {} with {}", candidateJunctionRoute, expandableEdge.getIdentifier(), e); } // If the routes are DualCarriageWayRoutes, then return if (isDualCarriageWayRoute(route)) { logger.debug("Adding Dual Carriageway Junction Route : {}", route); return Optional.of(route); } // Upper safety threshold to prevent bad edge cases if (route.size() <= MAXIMUM_DUAL_CARRIAGEWAY_ROUTE_SIZE) { expandableJunctionRoutes.add(route); } else { logger.trace( "Maximum number of edges in dual carriageway route ({}) reached. Skipping route : {}", MAXIMUM_DUAL_CARRIAGEWAY_ROUTE_SIZE, route); } } } return isDualCarriageWayJunctionRoute(expandableJunctionRoutes); } private boolean isDualCarriageWayRoute(final Route candidateRoute) { if (candidateRoute == null) { return false; } /* * A restriction with at least 4 main edges in a dual carriage way intersection, used to * filter out false positives, missed Test case 5 in BigNodeFinderTest. To increase * coverage, this is set to 2. */ if (candidateRoute.connectedEdges().stream().filter(Edge::isMainEdge) .count() >= MIN_MAIN_EDGE_DUAL_CARRIAGEWAY_INTERSECTION) { for (final Edge inEdge : candidateRoute.start().inEdges()) { // If inEdge is less important than RESIDENTIAL or has the same name as // candidateEdge or has the same Heading as candidateEdge, then skip if (inEdge.highwayTag().isLessImportantThan(HighwayTag.RESIDENTIAL) || this.edgeDirectionComparator.isSameDirection(candidateRoute.start(), inEdge, false) || edgeNameFuzzyMatch(candidateRoute.start(), inEdge, true)) { continue; } // If the candidateRoute has inEdge and outEdge that are in opposite direction for (final Edge outEdge : candidateRoute.end().outEdges()) { /** * Usually Dual Carriage Way roads are one way. OutEdge and inEdge cannot have * name mismatch. Including other Car Navigable roads like Service roads * increases false positive cases. */ if (outEdge.highwayTag().isMoreImportantThanOrEqualTo(HighwayTag.UNCLASSIFIED) // if an edge is considered as a junction edge, the dual carriage way // it connects to can not be link road && !inEdge.highwayTag().isLink() && !outEdge.highwayTag().isLink() && this.edgeDirectionComparator.isOppositeDirection(inEdge, outEdge, false) && !outEdge.hasReverseEdge() && !inEdge.hasReverseEdge() && edgeNameFuzzyMatch(outEdge, inEdge, false)) { return true; } /** * Enforce stricter name checks for Residential Roads to reduce false positives * and over merging of big nodes */ if (outEdge.highwayTag() == HighwayTag.RESIDENTIAL && this.edgeDirectionComparator.isOppositeDirection(inEdge, outEdge, false) && !outEdge.hasReverseEdge() && !inEdge.hasReverseEdge() && edgeNameExactMatch(outEdge, inEdge, inEdge.highwayTag() != HighwayTag.RESIDENTIAL)) { return true; } } } } return false; } /** * Determines if a candidate {@link Edge} can join the candidate {@link Route} to junction edges * that can be merged to candidate {@link Route} to form a {@link BigNode} */ private boolean isMergeCandidateEdge(final Edge candidateEdge, final Route candidateRoute, final Set junctionEdgeIds) { if (isCandidateJunctionEdge(candidateEdge)) { final Set candidateRouteEdges = Sets.newHashSet(candidateRoute); final Set candidateRouteEdgeIds = candidateRouteEdges.stream() .map(edge -> edge.getIdentifier()).collect(Collectors.toSet()); final Set filteredJunctionEdgeIds = Sets.difference(junctionEdgeIds, candidateRouteEdgeIds); for (final Edge edge : candidateEdge.outEdges()) { // Check if edge is connected to a junctionEdge with same name. Unless // there is a name mismatch we merge them. if (filteredJunctionEdgeIds.contains(edge.getIdentifier()) && edgeNameFuzzyMatch(candidateRoute.end(), edge, false)) { return true; } } } return false; } private boolean isShortEnough(final Edge edge) { final Distance length = edge.length(); if (SEARCH_RADIUS_MOTORWAY.isLessThanOrEqualTo(length)) { return false; } final HighwayTag highwayTag = mostSignificantConnectedHighwayType(edge); return searchRadius(highwayTag).isGreaterThan(length); } /** * Merging/coalescing connected junctionEdges together */ private Route mergeJunctionEdges(final Route candidateRoute, final Set junctionEdgeIds) { // If the start Node and end Node are equal, we have a complete Route if (candidateRoute.start().start().equals(candidateRoute.end().end())) { return candidateRoute; } final Set connectedEdges = candidateRoute.end().outEdges(); connectedEdges.addAll(candidateRoute.start().inEdges()); /* * The perfect case is when we have 4 junction edges that constitute a big node. We check if * there are outgoing or incoming Edges that are connected to other junction edges. Use of a * TreeSet is to consistently order the edge candidates so that we have a stable identifier * for the bigNode */ final Set mergeCandidates = new TreeSet<>((edge1, edge2) -> ComparisonChain.start() .compare(edge1.getIdentifier(), edge2.getIdentifier()).result()); final Set mainEdgeIdentifiers = new HashSet<>(); candidateRoute.forEach(edge -> mainEdgeIdentifiers.add(edge.getMainEdgeIdentifier())); connectedEdges.stream().filter(edge -> junctionEdgeIds.contains(edge.getIdentifier())) .filter(edge -> !candidateRoute.includes(edge)) .filter(edge -> !mainEdgeIdentifiers.contains(edge.getMainEdgeIdentifier())) .forEach(mergeCandidates::add); /* * There are some cases where we have a couple of junction edges (instead of all four) that * are part of big node */ if (mergeCandidates.isEmpty()) { connectedEdges.stream().filter(edge -> !junctionEdgeIds.contains(edge.getIdentifier())) .filter(edge -> !mainEdgeIdentifiers.contains(edge.getMainEdgeIdentifier())) .filter(edge -> isMergeCandidateEdge(edge, candidateRoute, junctionEdgeIds)) .filter(edge -> !candidateRoute.includes(edge)).forEach(mergeCandidates::add); } if (!mergeCandidates.isEmpty()) { for (final Edge mergeCandidate : mergeCandidates) { final Set connectedEdge = Collections.singleton(mergeCandidate); if (candidateRoute.end().isConnectedAtEndTo(connectedEdge)) { return mergeJunctionEdges(candidateRoute.append(mergeCandidate), junctionEdgeIds); } else if (candidateRoute.start().isConnectedAtStartTo(connectedEdge)) { return mergeJunctionEdges(Route.forEdge(mergeCandidate).append(candidateRoute), junctionEdgeIds); } } } return candidateRoute; } /** * @param edge * The {@link Edge} to look at * @return The most significant {@link HighwayTag} directly connected to this {@link Edge} * (including itself) */ private HighwayTag mostSignificantConnectedHighwayType(final Edge edge) { HighwayTag edgeTag = edge.highwayTag(); for (final Edge connected : edge.connectedEdges()) { final HighwayTag connectedTag = connected.highwayTag(); if (connectedTag.isMoreImportantThan(edgeTag)) { edgeTag = connectedTag; } } return edgeTag; } private boolean nameExactMatch(final String nameA, final String nameB) { return nameA.equalsIgnoreCase(nameB); } private boolean nameFuzzyMatch(final String nameA, final String nameB) { return nameA.equalsIgnoreCase(nameB) || StringUtils.getLevenshteinDistance(nameA, nameB, LEVENSHTEIN_DISTANCE_THRESHOLD) != -1; } private Distance searchRadius(final HighwayTag highwayTag) { if (this.radiusMap != null && this.radiusMap.containsKey(highwayTag.getTagValue())) { return this.radiusMap.get(highwayTag.getTagValue()); } if (highwayTag == HighwayTag.MOTORWAY || highwayTag == HighwayTag.MOTORWAY_LINK) { return SEARCH_RADIUS_MOTORWAY; } if (highwayTag == HighwayTag.TRUNK || highwayTag == HighwayTag.TRUNK_LINK) { return SEARCH_RADIUS_TRUNK; } if (highwayTag == HighwayTag.PRIMARY || highwayTag == HighwayTag.PRIMARY_LINK) { return SEARCH_RADIUS_PRIMARY; } if (highwayTag == HighwayTag.SECONDARY || highwayTag == HighwayTag.SECONDARY_LINK) { return SEARCH_RADIUS_SECONDARY; } if (highwayTag == HighwayTag.TERTIARY || highwayTag == HighwayTag.TERTIARY_LINK) { return SEARCH_RADIUS_TERTIARY; } return SEARCH_RADIUS_RESIDENTIAL; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/README.md ================================================ # BigNode BigNode is a concept of "augmented" intersection. It should match single decision points for a driver. This is mostly useful when inferring turn restrictions. ## Structure A BigNode is a group of Nodes, each one Edge away from at least one other, and close to each other. For example, two dual carriageways, when intersecting each other, can be modeled by a BigNode made of 4 Nodes at the 4 intersection points. The 4 internal Edges are referred to as "Junction Edges", and the edges directly connected to one of the Nodes, but at one end only, are the "in-Edges" and "out-Edges" to the BigNode. ## Path A BigNode can provide a set of Route objects called paths. Each path is the shortest way through the BigNode from one in-Edge to one out-Edge. ### RestrictedPath A RestrictedPath is a BigNode path that is restricted because of some TurnRestriction. It can be direct with a "NO\_TURN" type restriction, or indirect with a "ONLY" restriction that the path is close to but not following (For example some BigNode has a right\_turn\_only restriction, then the path that goes straight and the one turning left will be restricted). ## Visualization It is easy to visualize all the BigNodes and inferred RestrictedPaths by calling the following methods, provided an Atlas: ``` BigNodeFinder.findAndSaveBigNodesAsGeoJson(Atlas, WritableResource); ``` and ``` BigNodeFinder.findAndSaveRestrictedPathsAsGeoJson(Atlas, WritableResource); ``` ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/RestrictedPath.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.bignode; import java.io.Serializable; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.Route; /** * A turn restriction within a {@link BigNode}. It indicates which {@link BigNode} it belongs to, as * well as the {@link Route} path that is forbidden within the {@link BigNode} * * @author matthieun */ public class RestrictedPath implements Located, Serializable { private static final long serialVersionUID = 8003332683119555591L; private final Route route; private final BigNode parent; protected RestrictedPath(final BigNode parent, final Route route) { this.route = route; this.parent = parent; } @Override public Rectangle bounds() { return this.route.bounds(); } @Override public boolean equals(final Object other) { if (other instanceof RestrictedPath) { return this.route.equals(((RestrictedPath) other).getRoute()) && this.parent == ((RestrictedPath) other).getParent(); } return false; } public BigNode getParent() { return this.parent; } public Route getRoute() { return this.route; } @Override public int hashCode() { return this.route.hashCode(); } @Override public String toString() { return "[BigNode Restricted Path: " + this.route + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/converters/AtlasBigNodeRestrictedPathToGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.bignode.converters; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNodeFinder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * @author matthieun */ public class AtlasBigNodeRestrictedPathToGeoJsonConverter implements Converter { private final int logFrequency; public AtlasBigNodeRestrictedPathToGeoJsonConverter(final int logFrequency) { this.logFrequency = logFrequency; } @Override public GeoJsonObject convert(final Atlas atlas) { return new GeoJsonBuilder(this.logFrequency).create(Iterables .translateMulti(new BigNodeFinder().find(atlas), BigNode::asGeoJsonRestrictedPath)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/bignode/converters/AtlasBigNodesToGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.bignode.converters; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNode; import org.openstreetmap.atlas.geography.atlas.items.complex.bignode.BigNodeFinder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * @author matthieun */ public class AtlasBigNodesToGeoJsonConverter implements Converter { @Override public GeoJsonObject convert(final Atlas atlas) { return new GeoJsonBuilder().create( Iterables.translate(new BigNodeFinder().find(atlas), BigNode::asGeoJsonBigNode)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/boundaries/ComplexBoundary.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.boundaries; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometryPrintable; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.tags.AdministrativeLevelTag; import org.openstreetmap.atlas.tags.BoundaryTag; import org.openstreetmap.atlas.tags.Iso31662CountryTag; import org.openstreetmap.atlas.tags.Iso31663CountryTag; import org.openstreetmap.atlas.tags.Iso3166DefaultCountryTag; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.names.NameTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.maps.MultiMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * A representation of the Administrative Boundaries defined in the {@link BoundaryTag}. * * @author matthieun */ public class ComplexBoundary extends ComplexEntity implements GeometryPrintable { private static final long serialVersionUID = 3836743004772506528L; private static final Logger logger = LoggerFactory.getLogger(ComplexBoundary.class); private static final RelationOrAreaToMultiPolygonConverter RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private MultiPolygon outline; private Integer administrativeLevel; private Set countries; // The sub-areas as defined by the relation member role subarea private Set subAreas = new HashSet<>(); private final List invalidations = new ArrayList<>(); private final transient Optional administrativeLevelFilter; private final boolean withSubAreas; protected ComplexBoundary(final AtlasEntity source, final boolean withSubAreas, final Optional administrativeLevelFilter) { super(source); this.administrativeLevelFilter = administrativeLevelFilter; this.withSubAreas = withSubAreas; try { this.populateAdministrativeLevelAndOutline(); if (withSubAreas) { this.populateSubAreas(); for (final ComplexBoundary boundary : this.subAreas) { if (!boundary.isValid()) { setInvalidReason("Some subAreas are invalid", new CoreException( "Some subArea(s) are invalid: {}", boundary, boundary.getError().orElseThrow( () -> new CoreException("Should have an error here.")) .getException())); } } } } catch (final Exception e) { setInvalidReason("Unable to create complex boundary from " + source, e); logger.warn("Unable to create complex boundary from {}, id {}. Reason: {}", source.getType(), source.getIdentifier(), e.getMessage()); } } @Override public JsonObject asGeoJson() { return this.outline.asGeoJsonGeometry(); } @Override public boolean equals(final Object other) { if (other instanceof ComplexBoundary) { final ComplexBoundary that = (ComplexBoundary) other; return this.administrativeLevel == that.getAdministrativeLevel() && this.outline.equals(that.getOutline()); } return false; } public int getAdministrativeLevel() { return this.administrativeLevel; } @Override public List getAllInvalidations() { return this.invalidations; } public Iterable getCountries() { return this.countries; } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.forJson(this.asGeoJson()); } public MultiPolygon getOutline() { return this.outline; } public Set getSubAreas() { return this.subAreas; } public boolean hasCountryCode() { return isValid() && Iterables.size(this.countries) > 0; } @Override public int hashCode() { return Objects.hash(this.administrativeLevel, this.outline); } public void removeOuter(final Polygon outerToRemove) { final MultiMap outersToInners = new MultiMap<>(); this.outline.outers().forEach(outer -> { final List innersForThisOuter = this.outline.innersOf(outer); outersToInners.put(outer, innersForThisOuter); }); outersToInners.remove(outerToRemove); setOutline(new MultiPolygon(outersToInners)); } public void setOutline(final MultiPolygon outline) { this.outline = outline; } @Override public String toString() { return toString(""); } @Override public byte[] toWkb() { return this.outline.toWkb(); } @Override public String toWkt() { return this.outline.toWkt(); } protected String toString(final String header) { return String.format( header + "[ComplexBoundary: Source = [%s, ID = %s]\n\t" + header + "Administrative Level = %s\n\t" + header + "Countries = %s\n\t" + header + "Name = %s\n\t" + header + "Outline = %s\n\t" + header + "Children = \n%s\n" + header + "]", this.getSource().getType(), this.getSource().getIdentifier(), this.administrativeLevel, this.countries, NameTag.getNameOf(getSource()).orElse(""), this.outline == null ? "MISSING" : this.outline.toReadableString(), new StringList( this.subAreas.stream().map(subArea -> subArea.toString(header + "\t")) .collect(Collectors.toList())) .join("\n")); } /** * Find the administrative level and the outline of the boundary */ private void populateAdministrativeLevelAndOutline() { final AtlasEntity source = getSource(); if (source instanceof Relation || source instanceof Area) { final Optional administrativeLevelOption = AdministrativeLevelTag .getAdministrativeLevel(source); this.administrativeLevel = administrativeLevelOption.orElseThrow( () -> new CoreException("Invalid or missing administrative level for {} {}", source.getType(), source.getIdentifier())); // Store countries in a set to avoid duplicates. this.countries = Iterables .stream(new MultiIterable<>(Iso31663CountryTag.all(source), Iso31662CountryTag.all(source), Iso3166DefaultCountryTag.all(source))) .collectToSet(); // Don't bother if the admin level is not the one expected if (this.administrativeLevelFilter.isPresent() && !this.administrativeLevel.equals(this.administrativeLevelFilter.get())) { throw new CoreException("Administrative Level {} is not being queried.", this.administrativeLevel); } this.outline = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER.convert(source); } else { throw new CoreException("Supports only relations and areas."); } } /** * Find the sub-areas if any */ private void populateSubAreas() { this.subAreas = new HashSet<>(); final AtlasEntity source = getSource(); if (source instanceof Relation) { for (final RelationMember member : ((Relation) source).members()) { final AtlasEntity childEntity = member.getEntity(); if (BoundaryTag.isAdministrative(childEntity) && RelationTypeTag.ADMINISTRATIVE_BOUNDARY_ROLE_SUB_AREA .equals(member.getRole())) { final ComplexBoundary child = new ComplexBoundary(childEntity, this.withSubAreas, this.administrativeLevelFilter); this.subAreas.add(child); if (!child.isValid()) { this.invalidations.addAll(child.getAllInvalidations()); } } } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/boundaries/ComplexBoundaryFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.boundaries; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.AdministrativeLevelTag; import org.openstreetmap.atlas.tags.BoundaryTag; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * Finder for {@link ComplexBoundary}(ies). * * @author matthieun */ public class ComplexBoundaryFinder implements Finder { // If set to true, the boundaries will be structured with their sub-areas, and any // sub-area that has a parent area will not be standalone. If set to false, all // boundaries at all levels will be returned independently. private boolean withSubAreas; // If any, the administrative level to focus on only. private Optional administrativeLevel; /** * Construct the finder. All boundaries at all levels will be returned independently. */ public ComplexBoundaryFinder() { this.withSubAreas = false; this.administrativeLevel = Optional.empty(); } @Override public Iterable find(final Atlas atlas) { return Iterables.stream(new MultiIterable<>(atlas.relations(), atlas.areas())) .filter(BoundaryTag::isAdministrative) // Filter out the relations that are part of a larger admin boundary, as those will // be brought in as part of the larger boundary. .filter(this::subAreaFilter).map(entity -> new ComplexBoundary(entity, this.withSubAreas, this.administrativeLevel)); } /** * @param administrativeLevel * If any, the administrative level to focus on only. */ public void setAdministrativeLevel(final int administrativeLevel) { final long minimum = AdministrativeLevelTag.minimumAdministrativeLevelValue(); final long maximum = AdministrativeLevelTag.maximumAdministrativeLevelValue(); if (administrativeLevel >= minimum && administrativeLevel <= maximum) { this.administrativeLevel = Optional.of(administrativeLevel); } else { throw new CoreException( "Invalid administrative level: {}. Should be between {} and {} included.", administrativeLevel, minimum, maximum); } } /** * @param withSubAreas * If set to true, the boundaries will be structured with their sub-areas, and any * sub-area that has a parent area will not be standalone. If set to false, all * boundaries at all levels will be returned independently. */ public void setWithSubAreas(final boolean withSubAreas) { this.withSubAreas = withSubAreas; } /** * @param entity * An Atlas entity * @return True when the entity is has a role subarea within one of its administrative boundary * parent relations. */ private boolean isSubArea(final AtlasEntity entity) { final Set parentRelations = entity.relations(); for (final Relation parentRelation : parentRelations) { if (BoundaryTag.isAdministrative(parentRelation)) { final RelationMemberList children = parentRelation.members(); for (final RelationMember child : children) { final AtlasEntity childEntity = child.getEntity(); if (childEntity.getClass().equals(entity.getClass()) && childEntity.getIdentifier() == entity.getIdentifier()) { if (RelationTypeTag.ADMINISTRATIVE_BOUNDARY_ROLE_SUB_AREA .equals(child.getRole())) { return false; } } } } } return true; } /** * Filter out the sub areas only if the finder is looking for boundaries with sub areas * included. * * @param entity * The entity to filter * @return True if the entity is not filtered out */ private boolean subAreaFilter(final AtlasEntity entity) { return this.withSubAreas ? isSubArea(entity) : true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/boundaries/converters/ComplexBoundaryIterableToGeoJsonWriter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.boundaries.converters; import java.util.Map; import org.openstreetmap.atlas.geography.atlas.items.complex.boundaries.ComplexBoundary; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Write a set of complex boundaries to a geojson resource. * * @author matthieun */ public final class ComplexBoundaryIterableToGeoJsonWriter { public static void saveAsGeojson(final Iterable complexBoundaries, final WritableResource output) { try (JsonWriter writer = new JsonWriter(output)) { final Iterable geojsonObjects = Iterables .stream(complexBoundaries).flatMap(boundary -> { final Map tags = boundary.getSource().getTags(); return Iterables .stream(boundary.getOutline().asLocationIterableProperties()) .map(locationIterableProperties -> { locationIterableProperties.getProperties().putAll(tags); return locationIterableProperties; }); }); writer.write(new GeoJsonBuilder().create(geojsonObjects).jsonObject()); } } private ComplexBoundaryIterableToGeoJsonWriter() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/buildings/BuildingPart.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.buildings; import java.util.Map; import java.util.Objects; import java.util.Optional; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Part of a {@link ComplexBuilding}. Can be a simple {@link Area}, or a {@link MultiPolygon} * {@link Relation} that can be a building part with holes * * @author matthieun */ public class BuildingPart extends ComplexEntity { private static final long serialVersionUID = 364620404649236692L; private static final Logger logger = LoggerFactory.getLogger(BuildingPart.class); private static final RelationOrAreaToMultiPolygonConverter BUILDING_OUTLINE_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final HeightConverter HEIGHT_CONVERTER = new HeightConverter(); private MultiPolygon geometry; public BuildingPart(final AtlasEntity source) { super(source); try { this.geometry = BUILDING_OUTLINE_CONVERTER.convert(source); } catch (final Exception e) { logger.warn("Unable to create building part from {}", source, e); setInvalidReason("Unable to create building part", e); } } @Override public boolean equals(final Object other) { if (other instanceof BuildingPart) { return this.getGeometry().equals(((BuildingPart) other).getGeometry()); } return false; } public MultiPolygon getGeometry() { return this.geometry; } @Override public int hashCode() { return Objects.hash(this.geometry); } @Override public String toString() { return "[BuildingPart: Geometry = " + this.geometry.toReadableString() + "]"; } /** * @return The building part's top height */ public Optional topHeight() { final Map tags = getSource().getTags(); final String heightTag = tags.get("height"); try { if (heightTag != null) { return Optional.of(HEIGHT_CONVERTER.convert(heightTag)); } } catch (final Exception e) { logger.warn("Invalid height {} for building part id {}", heightTag, getSource().getIdentifier()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/buildings/ComplexBuilding.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.buildings; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.tags.BuildingLevelsTag; import org.openstreetmap.atlas.tags.BuildingMinLevelTag; import org.openstreetmap.atlas.tags.BuildingTag; import org.openstreetmap.atlas.tags.MinHeightTag; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A complex building, that can be made of Parts, and have holes (MultiPolygon). It can also be very * simple and be made of only one {@link Area}. * * @author matthieun */ public class ComplexBuilding extends ComplexEntity { private static final long serialVersionUID = 5351464852316720525L; private static final Logger logger = LoggerFactory.getLogger(ComplexBuilding.class); private static final RelationOrAreaToMultiPolygonConverter RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final JtsMultiPolygonToMultiPolygonConverter MULTIPOLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final HeightConverter HEIGHT_CONVERTER = new HeightConverter(); private final Set buildingParts; private MultiPolygon outline = null; private AtlasEntity outlineSource; private final Set containedOSMIDs; protected ComplexBuilding(final AtlasEntity source) { super(source); this.buildingParts = new HashSet<>(); this.containedOSMIDs = new HashSet<>(); try { this.populateBuildingPartsAndOutline(); } catch (final Exception e) { setInvalidReason("Unable to create complex building", e); logger.warn("Unable to create complex building from {}", source, e); return; } } /** * @return The building's base height */ public Optional baseHeight() { return MinHeightTag.get(this.getSource()); } public boolean containsOSMIdentifier(final long identifier) { return this.containedOSMIDs.contains(identifier); } @Override public List getAllInvalidations() { final List returnValue = new ArrayList<>(); if (!isValid()) { getError().ifPresent(returnValue::add); this.buildingParts.stream().filter(part -> !part.isValid()) .map(ComplexEntity::getAllInvalidations).flatMap(List::stream) .forEach(returnValue::add); } return returnValue; } public Set getBuildingParts() { return this.buildingParts; } /** * @return The outline of the building. */ public Optional getOutline() { return Optional.ofNullable(this.outline); } /** * @return The {@link AtlasEntity} representing the outline of the building */ public AtlasEntity getOutlineSource() { return this.outlineSource; } @Override public boolean isValid() { if (super.isValid()) { for (final BuildingPart part : this.buildingParts) { if (!part.isValid()) { return false; } } return true; } return false; } public Optional levels() { return BuildingLevelsTag.get(this.getSource()); } public Optional minimumLevel() { return BuildingMinLevelTag.get(this.getSource()); } @Override public String toString() { final StringBuilder parts = new StringBuilder(); for (final BuildingPart part : this.buildingParts) { parts.append(part); } return String.format("[ComplexBuilding:\n\tOutline = %s,\n\tParts = %s]", this.outline == null ? "MISSING" : this.outline.toReadableString(), parts.toString()); } /** * @return The building's top height */ public Optional topHeight() { Map tags = getSource().getTags(); String heightTag = tags.get("height"); try { if (heightTag != null) { return Optional.of(HEIGHT_CONVERTER.convert(heightTag)); } tags = this.outlineSource.getTags(); heightTag = tags.get("height"); if (heightTag != null) { return Optional.of(HEIGHT_CONVERTER.convert(heightTag)); } } catch (final Exception e) { logger.warn("Invalid height {} for building id {}", heightTag, getSource().getIdentifier()); } return Optional.empty(); } protected void populateBuildingPartsAndOutline() { this.containedOSMIDs.add(getOsmIdentifier()); final AtlasEntity source = getSource(); if (source instanceof Area) { // Simple case, yay! this.outline = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER.convert(source); this.outlineSource = source; } else if (source instanceof Relation) { final Relation relation = (Relation) source; final String type = relation.tag(RelationTypeTag.KEY); // Two cases here. The relation can be a multipolygon (in case there are just holes and // no parts) or a building relation, in case there are building parts. final Optional geom = relation.asMultiPolygon(); if (RelationTypeTag.MULTIPOLYGON_TYPE.equals(type) && geom.isPresent()) { // 1. Multipolygon. Relatively easy, there will be no building parts. this.outline = MULTIPOLYGON_CONVERTER.convert(geom.get()); this.outlineSource = relation; } else if (BuildingTag.KEY.equals(type)) { // 2. This relation is of an OSM 3D building. It should contain a member Area that // is the outline, tagged as a building. It should also contain zero to many // building:part=yes areas. // 2.a. Loop through the roles and find the outline for (final RelationMember member : relation.members()) { this.containedOSMIDs.add(member.getEntity().getOsmIdentifier()); if (RelationTypeTag.BUILDING_ROLE_OUTLINE.equals(member.getRole())) { this.outline = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER .convert(member.getEntity()); this.outlineSource = member.getEntity(); } } if (this.outline == null) { throw new CoreException( "Building part relation does not contain a building outline member"); } for (final RelationMember member : relation.members()) { this.containedOSMIDs.add(member.getEntity().getOsmIdentifier()); if (RelationTypeTag.BUILDING_ROLE_PART.equals(member.getRole())) { this.buildingParts.add(new BuildingPart(member.getEntity())); } } } else { throw new CoreException( "A building relation can only be of type=multipolygon or type=building"); } } else { throw new CoreException( "A building can only be made of a Relation or an Area. This was a {}", source.getClass().getName()); } if (this.outline.outers().isEmpty()) { throw new CoreException("A building cannot have no geometry."); } try { // By fetching the surface we calculate the area: if that area is negative we throw an // exception this.outline.surface(); } catch (final IllegalArgumentException oops) { throw new CoreException("Negative surface area", oops); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/buildings/ComplexBuildingFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.buildings; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.BuildingTag; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * {@link Finder} for {@link ComplexBuilding}s * * @author matthieun */ public class ComplexBuildingFinder implements Finder { @Override public Iterable find(final Atlas atlas) { // 1. Get all the simple buildings (Areas) that are not outlines. final Iterable simpleBuildings = atlas .areas(area -> isBuilding(area) && isSimple(area)); final Iterable simpleEntities = Iterables.translate(simpleBuildings, ComplexBuilding::new); // 2. Get all the complex buildings final Iterable complexBuildings = atlas .relations(relation -> isBuilding(relation) && isSimple(relation)); final Iterable complexEntities = Iterables.translate(complexBuildings, ComplexBuilding::new); // 3. Combine them in a multi iterable. return new MultiIterable<>(simpleEntities, complexEntities); } private boolean hasChildAreaAsBuilding(final Relation relation) { for (final RelationMember member : relation.members()) { final AtlasEntity entity = member.getEntity(); final String role = member.getRole(); if (entity instanceof Area && RelationTypeTag.MULTIPOLYGON_ROLE_OUTER.equals(role) && isBuilding((Area) entity)) { return true; } } return false; } private boolean isBuilding(final Area area) { return BuildingTag.isBuilding(area); } private boolean isBuilding(final Relation relation) { final String type = relation.tag(RelationTypeTag.KEY); /* * If we have a multipolygon relation, then it can be a building if it has the building tag * itself, or if one outer member of the relation has a building tag (this one is not * recommended in OSM, but many occurrences happen) */ return BuildingTag.KEY.equals(type) || RelationTypeTag.MULTIPOLYGON_TYPE.equals(type) && (BuildingTag.isBuilding(relation) || hasChildAreaAsBuilding(relation)); } private boolean isSimple(final AtlasEntity entity) { for (final Relation relation : entity.relations()) { if (isBuilding(relation)) { return false; } } return true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/buildings/HeightConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.buildings; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * Example use: Building Height * * @deprecated use {@link org.openstreetmap.atlas.tags.HeightTag} {@code get()} or * {@link org.openstreetmap.atlas.tags.annotations.extraction.AltitudeExtractor} * instead. * @author matthieun */ @Deprecated public class HeightConverter implements StringConverter { private static final String METERS_SUFFIX = " m"; @Override public Distance convert(final String object) { try { if (object.endsWith(METERS_SUFFIX)) { return Distance.meters( Double.valueOf(object.substring(0, object.lastIndexOf(METERS_SUFFIX)))); } if (object.contains("\'") || object.contains("\"")) { final StringList split = StringList.split(object, "\'"); if (split.size() == 2) { return Distance.feetAndInches(Double.valueOf(split.get(0)), Double .valueOf(split.get(1).substring(0, split.get(1).lastIndexOf("\"")))); } else if (split.size() == 1) { return Distance.inches(Double .valueOf(split.get(1).substring(0, split.get(1).lastIndexOf("\"")))); } else { throw new CoreException("Invalid Feet & Inches height value: {}", object); } } return Distance.meters(Double.valueOf(object)); } catch (final Exception e) { throw new CoreException("Cannot parse height {}", object, e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/buildings/converters/ComplexBuildingToGeojsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.buildings.converters; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.ComplexBuildingFinder; import org.openstreetmap.atlas.geography.geojson.GeoJsonSaver; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Get all the building outlines in an Atlas to a GeoJson file. * * @author matthieun */ public class ComplexBuildingToGeojsonConverter extends Command { public static final Switch ATLAS_FOLDER = new Switch<>("atlasFolder", "Folder containing the Atlas files to transcribe", value -> new AtlasResourceLoader().load(new File(value)), Optionality.REQUIRED); public static final Switch OUTPUT = new Switch<>("output", "The output GeoJson file", value -> new File(value), Optionality.REQUIRED); public static void main(final String[] args) { new ComplexBuildingToGeojsonConverter().run(args); } @Override protected int onRun(final CommandMap command) { final Atlas atlas = (Atlas) command.get(ATLAS_FOLDER); final File output = (File) command.get(OUTPUT); final Iterable shapes = Iterables.translateMulti( new ComplexBuildingFinder().find(atlas), building -> building.getOutline().get().outers()); GeoJsonSaver.save(shapes, output); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(ATLAS_FOLDER, OUTPUT); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/highwayarea/ComplexHighwayArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.highwayarea; import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; import java.util.TreeSet; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.utilities.scalars.Surface; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.MoreObjects; /** * A complex entity of highway areas. * * @author mgostintsev * @author isabellehillberg * @author cstaylor */ public class ComplexHighwayArea extends ComplexEntity { private static final long serialVersionUID = 441824709133762710L; private static final Logger logger = LoggerFactory.getLogger(ComplexHighwayArea.class); private PolyLine boundary; private final NavigableSet visitedEdgeIdentifiers = new TreeSet<>(); protected ComplexHighwayArea(final ComplexHighwayAreaHelper helper) { super(helper.getSourceEdge()); try { if (helper.getException() != null) { throw helper.getException(); } this.boundary = helper.getBoundary(); this.visitedEdgeIdentifiers.addAll(helper.getVisitedEdgeIdentifiers()); if (isSelfIntersecting()) { throw new CoreException("Self-intersecting geometry"); } if (isZeroSized()) { throw new CoreException("Zero-sized area"); } } catch (final Exception oops) { logger.warn("Unable to create ComplexHighwayArea from {}", helper.getSourceEdge(), oops); setInvalidReason("Couldn't create ComplexHighwayArea", oops); } } protected ComplexHighwayArea(final Edge edge) { this(new ComplexHighwayAreaHelper(edge)); } public PolyLine getHighwayAreaBoundary() { return this.boundary; } /** * Returns the atlas identifiers of all edges that were navigated when creating this highway * area * * @return a sorted list of atlas edge identifiers that we visited when creating this highway * area */ public NavigableSet getVisitedEdgeIdentifiers() { return this.visitedEdgeIdentifiers; } public boolean isSelfIntersecting() { return boundaryAsPolygon().selfIntersects(); } public boolean isZeroSized() { return boundaryAsPolygon().surface().equals(Surface.MINIMUM); } @Override public String toString() { return MoreObjects.toStringHelper(this).add("osm", getSource().getOsmIdentifier()) .add("atlas", getSource().getIdentifier()).add("boundary", this.boundary) .toString(); } private Polygon boundaryAsPolygon() { if (!isValid()) { throw new CoreException("Highway Area is invalid {}", getOsmIdentifier()); } final List locations = new ArrayList<>(); locations.addAll(this.boundary); return new Polygon(locations); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/highwayarea/ComplexHighwayAreaFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.highwayarea; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.AreaTag; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A simple finder for a {@link ComplexHighwayArea} * * @author isabellehillberg */ public class ComplexHighwayAreaFinder implements Finder { private static Optional toEntity(final Edge edge) { final ComplexHighwayArea complexHighwayAreaEntity = new ComplexHighwayArea(edge); return Optional.of(complexHighwayAreaEntity); } private static boolean validEdge(final Edge edge) { return Validators.isNotOfType(edge, HighwayTag.class, HighwayTag.NO) && Validators.isOfType(edge, AreaTag.class, AreaTag.YES) && edge.isMainEdge(); } @Override public Iterable find(final Atlas atlas) { final Set visitedEdgeIdentifiers = new HashSet<>(); return Iterables.stream(atlas.edges()) .flatMap(edges -> processEntity(edges, visitedEdgeIdentifiers)); } private List processEntity(final Edge edge, final Set visitedEdgeIdentifiers) { final List returnValue = new ArrayList<>(); if (!visitedEdgeIdentifiers.contains(edge.getIdentifier())) { Stream.of(edge).filter(ComplexHighwayAreaFinder::validEdge) .map(ComplexHighwayAreaFinder::toEntity).filter(Optional::isPresent) .map(Optional::get).forEach(area -> { visitedEdgeIdentifiers.addAll(area.getVisitedEdgeIdentifiers()); returnValue.add(area); }); } return returnValue; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/highwayarea/ComplexHighwayAreaHelper.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.highwayarea; import java.util.NavigableSet; import java.util.Optional; import java.util.TreeSet; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Route; /** * Since the constructor for ComplexHighwayArea must pass the source edge immediately we need to do * all of the processing of routes and which edges are the lowest ordered ones outside of that * class. * * @author cstaylor */ class ComplexHighwayAreaHelper { private final NavigableSet visitedEdgeIdentifiers = new TreeSet<>(); private PolyLine boundary; private Edge sourceEdge; private CoreException oops; ComplexHighwayAreaHelper(final Edge edge) { this.sourceEdge = edge; buildHighwayAreaBoundary(Route.forEdge(edge)).ifPresent(route -> { this.boundary = route.asPolyLine(); StreamSupport.stream(route.spliterator(), false).map(Edge::getIdentifier) .forEach(this.visitedEdgeIdentifiers::add); this.sourceEdge = edge.getAtlas().edge(this.visitedEdgeIdentifiers.first()); }); if (this.boundary == null) { this.oops = new CoreException("Unable to build boundary for edge {}", edge.getOsmIdentifier()); } } PolyLine getBoundary() { return this.boundary; } CoreException getException() { return this.oops; } Edge getSourceEdge() { return this.sourceEdge; } NavigableSet getVisitedEdgeIdentifiers() { return this.visitedEdgeIdentifiers; } private Optional buildHighwayAreaBoundary(final Route boundary) { for (final Edge edge : boundary.end().end().connectedEdges()) { if (canAddEdgeToBoundary(edge, boundary)) { final Route extendedBoundary = boundary.append(edge); if (extendedBoundary.end().end().getLocation() .equals(extendedBoundary.start().start().getLocation())) { return Optional.of(extendedBoundary); } else { return buildHighwayAreaBoundary(extendedBoundary); } } } return Optional.empty(); } // 1. Traversing in one direction, don't add any reverse edges // 2. There are some overlapping areas (bad data) which represent the same entity. To avoid // adding incorrect edges, only add edges with the same OSM identifier. // 3. The end location of the boundary matches the start location of the candidate edge. // 4. No duplicate edges. private boolean canAddEdgeToBoundary(final Edge edge, final Route boundary) { return edge.getIdentifier() != -boundary.end().getIdentifier() && edge.getOsmIdentifier() == boundary.end().getOsmIdentifier() && boundary.end().end().getLocation().equals(edge.start().getLocation()) && !boundary.includes(edge); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/islands/ComplexIsland.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.islands; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * In cases of lakes/reservoirs with islands, they are modeled as relations of multi-polygon type. * The outer polygons are usually the lakes. The inner polygons are islands. See * http://www.openstreetmap.org/relation/2314241 * * @author Sid */ public class ComplexIsland extends ComplexEntity { private static final long serialVersionUID = 7840944233946510730L; private static final RelationOrAreaToMultiPolygonConverter RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final JtsMultiPolygonToMultiPolygonConverter MULTIPOLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final Logger logger = LoggerFactory.getLogger(ComplexIsland.class); private MultiPolygon multiPolygon; public ComplexIsland(final AtlasEntity source) { super(source); try { populateGeometry(); } catch (final Exception e) { logger.warn("Unable to create complex islands from {}", source, e); setInvalidReason("Unable to create complex islands", e); } } public MultiPolygon getGeometry() { return this.multiPolygon; } @Override public String toString() { return "Island : " + getSource(); } private void populateGeometry() { final AtlasEntity source = getSource(); if (source instanceof Relation) { final Relation relation = (Relation) source; final String type = relation.tag(RelationTypeTag.KEY); final Optional geom = relation.asMultiPolygon(); if (RelationTypeTag.MULTIPOLYGON_TYPE.equals(type) && geom.isPresent()) { this.multiPolygon = MULTIPOLYGON_CONVERTER.convert(geom.get()); return; } } else if (source instanceof Area) { this.multiPolygon = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER.convert(source); return; } throw new CoreException("Geometry is not set for {}", source); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/islands/ComplexIslandFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.islands; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.geography.atlas.items.complex.WaterIslandConfigurationReader; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author Sid * @author sbhalekar */ public class ComplexIslandFinder implements Finder { private final WaterIslandConfigurationReader islandConfigurationReader; public ComplexIslandFinder() { this.islandConfigurationReader = new DefaultIslandConfigurationReader("islands.json"); } public ComplexIslandFinder(final Resource resource) { this.islandConfigurationReader = new DefaultIslandConfigurationReader(resource); } /** * Use the configuration reader and convert all the allowed atlas entities into * {@link ComplexIsland} * * @param atlas * The {@link Atlas} to browse. * @return an {@link Iterable} of the {@link ComplexIsland}s in the given {@link Atlas} */ @Override public Iterable find(final Atlas atlas) { return StreamSupport .stream(Iterables .translate(atlas.entities(), this.islandConfigurationReader::convert) .spliterator(), false) .filter(Optional::isPresent).map(Optional::get) .filter(object -> object instanceof ComplexIsland) .map(object -> (ComplexIsland) object).collect(Collectors.toList()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/islands/DefaultIslandConfigurationReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.islands; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.WaterIslandConfigurationReader; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.configuration.ConfiguredFilter; import org.openstreetmap.atlas.utilities.configuration.StandardConfiguration; /** * Class used by {@link ComplexIslandFinder} to read the island configuration and map * {@link AtlasEntity} to {@link ComplexIsland} * * @author sbhalekar */ public class DefaultIslandConfigurationReader extends WaterIslandConfigurationReader { public DefaultIslandConfigurationReader(final Resource resource) { super(resource); } public DefaultIslandConfigurationReader(final String resourceName) { this(new InputStreamResource( () -> DefaultIslandConfigurationReader.class.getResourceAsStream(resourceName))); } @Override protected Optional createComplexEntity(final AtlasEntity atlasEntity) { return this.getConfigurationMapper().test(atlasEntity) ? Optional.of(new ComplexIsland(atlasEntity)) : Optional.empty(); } @Override protected ConfiguredFilter readConfiguration(final Resource configurationResource) { return ConfiguredFilter.getDefaultFilter(new StandardConfiguration(configurationResource)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/landcover/ComplexLandCover.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.landcover; import java.io.InputStreamReader; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonParser; /** * @author sbhalekar * @author samg */ public final class ComplexLandCover extends ComplexEntity { private static final Logger logger = LoggerFactory.getLogger(ComplexLandCover.class); private static final long serialVersionUID = 220683230343177634L; private static final RelationOrAreaToMultiPolygonConverter MULTIPOLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter(); private static final String LAND_COVER_RESOURCE = "land-cover-tag-filter.json"; // The default LandCover tags private static List defaultTaggableFilter; private final MultiPolygon multiPolygon; /** * This method creates a {@link ComplexLandCover} for the specified {@link AtlasEntity} if it * meets the requirements for Complex LandCover relation. The LandCover tags are checked against * the default tags. * * @param source * The {@link AtlasEntity} for which the ComplexEntity is created * @return {@link ComplexLandCover} if created, else return empty. */ public static Optional getComplexLandCover(final AtlasEntity source) { if (defaultTaggableFilter == null) { computeDefaultFilter(); } return getComplexLandCover(source, ComplexLandCover::hasLandCoverTag); } /** * This method creates a {@link ComplexLandCover} for the specified {@link AtlasEntity} and * {@link TaggableFilter}. The land cover tags are checked against the landCoverFilter param as * well as the default tags. * * @param source * The {@link AtlasEntity} for which the ComplexEntity is created * @param landCoverFilter * The {@link TaggableFilter} of land cover tags against which the relation is * checked for land cover tags * @return {@link ComplexLandCover} if created, else return empty. */ public static Optional getComplexLandCover(final AtlasEntity source, final Predicate landCoverFilter) { try { return ((source instanceof Relation || source instanceof Area) && landCoverFilter.test(source)) ? Optional.of(new ComplexLandCover(source)) : Optional.empty(); } catch (final Exception exception) { logger.warn("Unable to create complex land cover relations from {}", source, exception); return Optional.empty(); } } private static void computeDefaultFilter() { try (InputStreamReader reader = new InputStreamReader( ComplexLandCover.class.getResourceAsStream(LAND_COVER_RESOURCE))) { final JsonElement element = new JsonParser().parse(reader); final JsonArray filters = element.getAsJsonObject().get("filters").getAsJsonArray(); defaultTaggableFilter = StreamSupport.stream(filters.spliterator(), false) .map(jsonElement -> TaggableFilter.forDefinition(jsonElement.getAsString())) .collect(Collectors.toList()); } catch (final Exception exception) { throw new CoreException( "There was a problem parsing aoi-tag-filter.json. Check if the JSON file has valid structure.", exception); } } /** * Checks for land cover tags in the object * * @param source * {@link AtlasEntity} that needs to be checked for land cover tags * @return {@code true} if the source has land cover tags */ private static boolean hasLandCoverTag(final Taggable source) { return defaultTaggableFilter.stream() .anyMatch(taggableFilter -> taggableFilter.test(source)); } private ComplexLandCover(final AtlasEntity source) { super(source); try { this.multiPolygon = MULTIPOLYGON_CONVERTER.convert(source); } catch (final Exception exception) { setInvalidReason("Unable to convert the AtlasEntity to MultiPolygon", exception); throw new CoreException("Unable to convert the AtlasEntity {} to MultiPolygon", source.getOsmIdentifier(), exception); } } @Override public boolean equals(final Object other) { return other instanceof ComplexLandCover && super.equals(other); } public MultiPolygon getGeometry() { return this.multiPolygon; } @Override public int hashCode() { return super.hashCode(); } @Override public String toString() { return this.getClass().getName() + " " + getSource(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/landcover/ComplexLandCoverFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.landcover; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * {@link ComplexLandCover} finder. * * @author samg */ public class ComplexLandCoverFinder implements Finder { /** * Finds all relations and areas that are candidates for {@link ComplexLandCover} and converts * them into {@link ComplexLandCover}. * * @param atlas * The {@link Atlas} to browse. * @return {@link Iterables} of {@link ComplexLandCover}. */ @Override public Iterable find(final Atlas atlas) { final Iterable iterableOfComplexLandCoverRelations = StreamSupport .stream(atlas.relations().spliterator(), true) .map(ComplexLandCover::getComplexLandCover).filter(Optional::isPresent) .map(Optional::get).collect(Collectors.toList()); final Iterable iterableOfComplexLandCoverAreas = StreamSupport .stream(atlas.areas().spliterator(), true) .map(ComplexLandCover::getComplexLandCover).filter(Optional::isPresent) .map(Optional::get).collect(Collectors.toList()); return new MultiIterable<>(iterableOfComplexLandCoverRelations, iterableOfComplexLandCoverAreas); } /** * Finds all relations and areas that are candidates for {@link ComplexLandCover} with given * land cover tags and converts them into {@link ComplexLandCover}. * * @param atlas * Atlas to build the ComplexLandCover * @param landCoverFilter * {@link TaggableFilter} land cover taggable filter * @return {@link Iterables} of {@link ComplexLandCover}. */ public Iterable find(final Atlas atlas, final TaggableFilter landCoverFilter) { final Iterable iterableOfComplexLandCoverRelations = StreamSupport .stream(atlas.relations().spliterator(), true) .map(relation -> ComplexLandCover.getComplexLandCover(relation, landCoverFilter)) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); final Iterable iterableOfComplexLandCoverAreas = StreamSupport .stream(atlas.areas().spliterator(), true) .map(area -> ComplexLandCover.getComplexLandCover(area, landCoverFilter)) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); return new MultiIterable<>(iterableOfComplexLandCoverRelations, iterableOfComplexLandCoverAreas); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/restriction/ComplexTurnRestriction.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.restriction; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.geography.atlas.items.TurnRestriction; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A Complex turn restriction from one atlas. * * @author matthieun * @author cstaylor */ public class ComplexTurnRestriction extends ComplexEntity { private static final Logger logger = LoggerFactory.getLogger(ComplexTurnRestriction.class); private static final long serialVersionUID = 8558201688502883714L; private TurnRestriction turnRestriction; protected ComplexTurnRestriction(final AtlasEntity source, final Predicate validEdge) { super(source); try { this.turnRestriction = TurnRestriction.from((Relation) source) .orElseThrow(() -> new CoreException( "{} is not a turn restriction according to Atlas", source.getIdentifier())); final Route route = this.turnRestriction.route(); final int routeLength = route.size(); if (routeLength < 2) { throw new CoreException("Must have at least two edges in the route"); } final long filteredLength = StreamSupport.stream(route.spliterator(), false) .filter(validEdge).count(); if (filteredLength < routeLength) { throw new CoreException("{} invalid edges", routeLength - filteredLength); } } catch (final Exception oops) { logger.trace("Unable to create ComplexTurnRestriction from {}", source, oops); setInvalidReason("Couldn't create ComplexTurnRestriction", oops); } } @Override public Rectangle bounds() { return this.route().bounds(); } @Override public List getAllInvalidations() { final List returnValue = new ArrayList<>(); if (!isValid()) { returnValue.add(new ComplexEntityError(this, "turn restriction is null")); } return returnValue; } public TurnRestriction getTurnRestriction() { return this.turnRestriction; } /** * Proxy for TurnRestriction.route() * * @return The Route represented by this {@link ComplexTurnRestriction} */ public Route route() { if (this.turnRestriction != null) { return this.turnRestriction.route(); } else { return null; } } @Override public String toString() { return "[ComplexTurnRestriction: " + this.turnRestriction + "]"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/restriction/ComplexTurnRestrictionFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.restriction; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.tags.TurnRestrictionTag; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A simple finder for a {@link ComplexTurnRestriction} * * @author matthieun */ public class ComplexTurnRestrictionFinder implements Finder { private final Predicate acceptableEdges; public ComplexTurnRestrictionFinder() { this(x -> true); } public ComplexTurnRestrictionFinder(final Predicate acceptableEdges) { this.acceptableEdges = acceptableEdges == null ? x -> true : acceptableEdges; } @Override public Iterable find(final Atlas atlas) { return Iterables.translate(atlas.relations(TurnRestrictionTag::isRestriction), relation -> new ComplexTurnRestriction(relation, this.acceptableEdges)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/restriction/converters/AtlasTurnRestrictionsToGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.restriction.converters; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.complex.restriction.ComplexTurnRestrictionFinder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * @author matthieun */ public class AtlasTurnRestrictionsToGeoJsonConverter implements Converter { @Override public GeoJsonObject convert(final Atlas atlas) { return new GeoJsonBuilder() .create(Iterables.translate(new ComplexTurnRestrictionFinder().find(atlas), turnRestriction -> turnRestriction.getTurnRestriction().asGeoJson())); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/roundabout/ComplexRoundabout.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.roundabout; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.walker.SimpleEdgeWalker; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.tags.JunctionTag; /** * A {@link ComplexEntity} representation of a roundabout. To form a valid {@link ComplexRoundabout} * the source {@link Edge} and all connected Edges tagged as a roundabout must meet the following * criteria: be one way; be car navigable; together form a single closed {@link Route} that has the * appropriate directionality based on the country. Adapted from the MalformedRoundaboutCheck in * Atlas-Checks. * * @author bbreithaupt * @author savannahostrowski */ public class ComplexRoundabout extends ComplexEntity { /** * An enum of RoundaboutDirections */ public enum RoundaboutDirection { CLOCKWISE, COUNTERCLOCKWISE, // Handles the case where we were unable to get any information about the roundabout's // direction. UNKNOWN } protected static final String WRONG_WAY_INVALIDATION = "This roundabout is going the wrong direction, or has been improperly tagged as a roundabout."; protected static final String INCOMPLETE_ROUTE_INVALIDATION = "This roundabout does not form a single, one-way, complete, car navigable route."; private static final String EXCEPTION_MESSAGE = "Exception thrown while trying to build a ComplexRoundabout"; // Country default source: // https://en.wikipedia.org/wiki/List_of_countries_with_left-hand_traffic private static final List LEFT_DRIVING_COUNTRIES_DEFAULT = Arrays.asList("AIA", "ATG", "AUS", "BGD", "BHS", "BMU", "BRB", "BRN", "BTN", "BWA", "CCK", "COK", "CXR", "CYM", "CYP", "DMA", "FJI", "FLK", "GBR", "GGY", "GRD", "GUY", "HKG", "IDN", "IMN", "IND", "IRL", "JAM", "JEY", "JPN", "KEN", "KIR", "KNA", "LCA", "LKA", "LSO", "MAC", "MDV", "MLT", "MOZ", "MSR", "MUS", "MWI", "MYS", "NAM", "NFK", "NIU", "NPL", "NRU", "NZL", "PAK", "PCN", "PNG", "SGP", "SGS", "SHN", "SLB", "SUR", "SWZ", "SYC", "TCA", "THA", "TKL", "TLS", "TON", "TTO", "TUV", "TZA", "UGA", "VCT", "VGB", "VIR", "WSM", "ZAF", "ZMB", "ZWE"); private static final long serialVersionUID = 2512054399729675784L; private final List invalidationReasons = new ArrayList<>(); private Set roundaboutEdgeSet; private Route roundaboutRoute; /** * This method returns a RoundaboutDirection enum which indicates direction of the flow of * traffic based on the cross product of two adjacent edges. This method leverages the * right-hand rule as it relates to the directionality of two vectors. * * @see "https://en.wikipedia.org/wiki/Right-hand_rule" * @param roundaboutEdges * A list of Edges in a roundabout * @return CLOCKWISE or COUNTERCLOCKWISE if the majority of the edges have positive or negative * cross products respectively, and UNKNOWN if all edge cross products are 0 or if the * roundabout's geometry is malformed */ private static RoundaboutDirection findRoundaboutDirection(final Route roundaboutEdges) { int clockwiseCount = 0; int counterClockwiseCount = 0; for (int index = 0; index < roundaboutEdges.size(); index++) { // Get the Edges to use in the cross product final Edge edge1 = roundaboutEdges.get(index); // We mod the roundabout edges here so that we can get the last pair of edges in the // Roundabout correctly final Edge edge2 = roundaboutEdges.get((index + 1) % roundaboutEdges.size()); // Get the cross product and then the direction of the roundabout final double crossProduct = getCrossProduct(edge1, edge2); final RoundaboutDirection direction; if (crossProduct < 0) { direction = RoundaboutDirection.COUNTERCLOCKWISE; } else if (crossProduct > 0) { direction = RoundaboutDirection.CLOCKWISE; } else { direction = RoundaboutDirection.UNKNOWN; } // If the direction is UNKNOWN then we continue to the next iteration because we do not // Have any new information about the roundabout's direction if (direction.equals(RoundaboutDirection.UNKNOWN)) { continue; } if (direction.equals(RoundaboutDirection.CLOCKWISE)) { clockwiseCount += 1; } if (direction.equals(RoundaboutDirection.COUNTERCLOCKWISE)) { counterClockwiseCount += 1; } } // Return the Enum for whatever has the highest count if (clockwiseCount > counterClockwiseCount) { return RoundaboutDirection.CLOCKWISE; } else if (clockwiseCount < counterClockwiseCount) { return RoundaboutDirection.COUNTERCLOCKWISE; } else { return RoundaboutDirection.UNKNOWN; } } /** * This method returns the cross product between two adjacent edges. * * @see "https://en.wikipedia.org/wiki/Cross_product" * @param edge1 * An Edge entity in the roundabout * @param edge2 * An Edge entity in the roundabout adjacent to edge1 * @return A double corresponding to the cross product between two edges */ private static double getCrossProduct(final Edge edge1, final Edge edge2) { // Get the nodes' latitudes and longitudes to use in deriving the vectors final double node1Y = edge1.start().getLocation().getLatitude().asDegrees(); final double node1X = edge1.start().getLocation().getLongitude().asDegrees(); final double node2Y = edge1.end().getLocation().getLatitude().asDegrees(); final double node2X = edge1.end().getLocation().getLongitude().asDegrees(); final double node3Y = edge2.end().getLocation().getLatitude().asDegrees(); final double node3X = edge2.end().getLocation().getLongitude().asDegrees(); // Get the vectors from node 2 to 1, and node 2 to 3 final double vector1X = node2X - node1X; final double vector1Y = node2Y - node1Y; final double vector2X = node2X - node3X; final double vector2Y = node2Y - node3Y; // The cross product tells us the direction of the orthogonal vector, which is // Directly related to the direction of rotation/traffic return (vector1X * vector2Y) - (vector1Y * vector2X); } public ComplexRoundabout(final Edge source) { this(source, LEFT_DRIVING_COUNTRIES_DEFAULT); } public ComplexRoundabout(final Edge source, final List leftDrivingCountries) { super(source); try { if (!( // Make sure that the source has an iso_country_code source.getTag(ISOCountryTag.KEY).isPresent() // Make sure that the source is an instances of a roundabout && JunctionTag.isRoundabout(source) // Make sure that we are looking at a main edge && (source.isMainEdge()))) { final ComplexEntityError sourceError = new ComplexEntityError(this, String.format("Invalid source Edge (%s) for a roundabout.", source.getIdentifier()), new CoreException(EXCEPTION_MESSAGE)); this.invalidationReasons.add(sourceError); throw sourceError.getException(); } final String isoCountryCode = source.tag(ISOCountryTag.KEY).toUpperCase(); // Get all edges in the roundabout this.roundaboutEdgeSet = new SimpleEdgeWalker(source, this.isRoundaboutEdge()) .collectEdges(); // Try to build a Route from the edges try { this.roundaboutRoute = Route.fromNonArrangedEdgeSet(this.roundaboutEdgeSet, false); } // If a Route cannot be formed, invalidate the roundabout. catch (final CoreException badRoundabout) { final ComplexEntityError routeError = new ComplexEntityError(this, INCOMPLETE_ROUTE_INVALIDATION, new CoreException(EXCEPTION_MESSAGE)); this.invalidationReasons.add(routeError); throw routeError.getException(); } // Invalidate the roundabout if any of the edges are not car navigable or main Edges, // or the route does not form a closed loop. if (this.roundaboutEdgeSet.stream() .anyMatch(roundaboutEdge -> !HighwayTag.isCarNavigableHighway(roundaboutEdge) || !roundaboutEdge.isMainEdge()) || !this.roundaboutRoute.start().inEdges().contains(this.roundaboutRoute.end())) { this.invalidationReasons.add(new ComplexEntityError(this, INCOMPLETE_ROUTE_INVALIDATION, new CoreException(EXCEPTION_MESSAGE))); } // Get the direction of the roundabout final RoundaboutDirection direction = findRoundaboutDirection(this.roundaboutRoute); // Determine if the roundabout is in a left or right driving country final boolean isLeftDriving = leftDrivingCountries.contains(isoCountryCode); // If the roundabout traffic is clockwise in a right-driving country, or // If the roundabout traffic is counterclockwise in a left-driving country if (direction.equals(RoundaboutDirection.CLOCKWISE) && !isLeftDriving || direction.equals(RoundaboutDirection.COUNTERCLOCKWISE) && isLeftDriving) { this.invalidationReasons.add(new ComplexEntityError(this, WRONG_WAY_INVALIDATION, new CoreException(EXCEPTION_MESSAGE))); } // If there is an invalidation logged, invalidate the roundabout if (!this.invalidationReasons.isEmpty()) { throw this.invalidationReasons.get(0).getException(); } } catch (final Throwable exception) { setInvalidReason(exception.getMessage(), exception); } } @Override public Rectangle bounds() { return this.roundaboutRoute.bounds(); } @Override public boolean equals(final Object other) { if (other instanceof ComplexRoundabout) { return this.roundaboutEdgeSet .equals(((ComplexRoundabout) other).getRoundaboutEdgeSet()); } return false; } @Override public List getAllInvalidations() { return this.invalidationReasons; } public Set getRoundaboutEdgeSet() { return this.roundaboutEdgeSet; } public Route getRoundaboutRoute() { return this.roundaboutRoute; } @Override public int hashCode() { return super.hashCode(); } @Override public String toString() { return String.format("Roundabout of Edges: %s", this.roundaboutEdgeSet); } /** * Function for {@link SimpleEdgeWalker} that gathers connected edges that are part of a * roundabout. * * @return {@link Function} for {@link SimpleEdgeWalker} */ private Function> isRoundaboutEdge() { return edge -> edge.connectedEdges().stream().filter(JunctionTag::isRoundabout); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/roundabout/ComplexRoundaboutFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.roundabout; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * {@link Finder} for {@link ComplexRoundabout}s. * * @author bbreithaupt */ public class ComplexRoundaboutFinder implements Finder { private final Set checkedIds = new HashSet<>(); @Override public Iterable find(final Atlas atlas) { final List complexRoundabouts = new ArrayList<>(); Iterables.stream(atlas.edges()).forEach(edge -> { if (!this.checkedIds.contains(edge.getIdentifier())) { final ComplexRoundabout complexRoundabout = new ComplexRoundabout(edge); if (complexRoundabout.getRoundaboutEdgeSet() != null) { this.checkedIds.addAll(complexRoundabout.getRoundaboutEdgeSet().stream() .map(Edge::getIdentifier).collect(Collectors.toSet())); } if (complexRoundabout.isValid()) { complexRoundabouts.add(complexRoundabout); } } }); return complexRoundabouts; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/ComplexWaterEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.slf4j.Logger; /** * @author Sid */ public abstract class ComplexWaterEntity extends ComplexEntity { private static final long serialVersionUID = 7835788819725148174L; private final WaterType waterType; public ComplexWaterEntity(final AtlasEntity source, final WaterType waterType) { super(source); this.waterType = waterType; try { populateGeometry(); } catch (final Exception e) { getLogger().warn("Unable to create complex water entity from {}", source, e); setInvalidReason("Unable to create complex water entity", e); } } @Override public boolean equals(final Object other) { if (other instanceof ComplexWaterEntity) { final ComplexWaterEntity that = (ComplexWaterEntity) other; return new EqualsBuilder().append(this.waterType, that.waterType) .append(this.getSource(), that.getSource()).build(); } return false; } public WaterType getWaterType() { return this.waterType; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getSource()).append(this.waterType).build(); } @Override public String toString() { return this.getClass().getName() + " " + this.getWaterType() + " " + getSource(); } protected abstract Logger getLogger(); /** * This function populates the geometry */ protected abstract void populateGeometry(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/ComplexWaterbody.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water; import java.util.Optional; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.RelationOrAreaToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A waterbody usually refers to a area of (standing) water and typically has polygonal geometry. * This contrasts with waterways (usually flowing) like streams which typically have linear geometry * * @author Sid */ public class ComplexWaterbody extends ComplexWaterEntity { private static final long serialVersionUID = -666543090371777011L; private static final RelationOrAreaToMultiPolygonConverter RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER = new RelationOrAreaToMultiPolygonConverter( true); private static final JtsMultiPolygonToMultiPolygonConverter MULTIPOLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); private static final Logger logger = LoggerFactory.getLogger(ComplexWaterbody.class); private MultiPolygon geometry; public ComplexWaterbody(final AtlasEntity source, final WaterType type) { super(source, type); } @Override public boolean equals(final Object other) { if (other instanceof ComplexWaterbody) { final ComplexWaterbody that = (ComplexWaterbody) other; return new EqualsBuilder().append(this.getWaterType(), that.getWaterType()) .append(this.getSource(), that.getSource()) .append(this.getGeometry().toWkt(), that.getGeometry().toWkt()).build(); } return false; } public MultiPolygon getGeometry() { return this.geometry; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getSource()).append(this.getWaterType()) .append(this.getGeometry().toWkb()).build(); } @Override protected Logger getLogger() { return logger; } @Override protected void populateGeometry() { final AtlasEntity source = getSource(); if (source instanceof Area) { this.geometry = RELATION_OR_AREA_TO_MULTI_POLYGON_CONVERTER.convert(source); return; } else if (source instanceof Relation) { final Relation relation = (Relation) source; final Optional geom = relation.asMultiPolygon(); final String type = relation.tag(RelationTypeTag.KEY); if (RelationTypeTag.MULTIPOLYGON_TYPE.equals(type) && geom.isPresent()) { this.geometry = MULTIPOLYGON_CONVERTER.convert(geom.get()); return; } } throw new CoreException("Geometry is not set for {}", source); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/ComplexWaterway.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A waterway (usually flowing ex : rivers, streams) typically has linear geometry. This contrasts * with waterbody which usually refers to an area of (standing) water and typically has polygonal * geometry * * @author Sid */ public class ComplexWaterway extends ComplexWaterEntity { private static final long serialVersionUID = -5567739097914423531L; private static final Logger logger = LoggerFactory.getLogger(ComplexWaterway.class); private PolyLine geometry; public ComplexWaterway(final AtlasEntity source, final WaterType type) { super(source, type); } @Override public boolean equals(final Object other) { if (other instanceof ComplexWaterbody) { final ComplexWaterbody that = (ComplexWaterbody) other; return new EqualsBuilder().append(this.getWaterType(), that.getWaterType()) .append(this.getSource(), that.getSource()) .append(this.getGeometry().toWkt(), that.getGeometry().toWkt()).build(); } return false; } public PolyLine getGeometry() { return this.geometry; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getSource()).append(this.getWaterType()) .append(this.getGeometry().toWkb()).build(); } @Override protected Logger getLogger() { return logger; } @Override protected void populateGeometry() { final AtlasEntity source = getSource(); /* * We currently don't process waterway relations. So if it is a way and it is part of * relation where it is a side stream, main stream or tributary, we process them. */ if (source instanceof Line) { final Line line = (Line) source; this.geometry = line.asPolyLine(); return; } throw new CoreException("Geometry is not set for {}", source); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/WaterType.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water; import org.openstreetmap.atlas.exception.CoreException; /** * Type of supported water bodies * * @author sbhalekar */ public enum WaterType { LAKE, RIVER, CANAL, CREEK, DITCH, LAGOON, POND, POOL, RESERVOIR, SEA, WETLAND, HARBOUR, UNKNOWN; /** * Convert a string to {@link WaterType} * * @param type * String representation of WaterType * @return {@link WaterType} */ public static WaterType from(final String type) { for (final WaterType waterBodyType : WaterType.values()) { if (waterBodyType.name().equalsIgnoreCase(type)) { return waterBodyType; } } throw new CoreException("Unknown water type {}", type); } @Override public String toString() { return this.name().toLowerCase(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/finder/ComplexWaterEntityFinder.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water.finder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.Finder; import org.openstreetmap.atlas.geography.atlas.items.complex.water.ComplexWaterEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.water.WaterType; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Finder to find all {@link ComplexWaterEntity} from and {@link Atlas} by using multiple * configuration readers for each {@link WaterType} * * @author sbhalekar */ public class ComplexWaterEntityFinder implements Finder { private static final Predicate RELATION_FILTER = relation -> Validators.isOfType( relation, RelationTypeTag.class, RelationTypeTag.MULTIPOLYGON, RelationTypeTag.BOUNDARY, RelationTypeTag.WATERWAY); private static final Logger logger = LoggerFactory.getLogger(ComplexWaterEntityFinder.class); private final List waterConfigurationReaders; public ComplexWaterEntityFinder() { this.waterConfigurationReaders = new ArrayList<>(); // read in the default configuration files with default mappings for each water body type this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("lake.json", WaterType.LAKE)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("river.json", WaterType.RIVER)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("lagoon.json", WaterType.LAGOON)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("wetland.json", WaterType.WETLAND)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("reservoir.json", WaterType.RESERVOIR)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("pool.json", WaterType.POOL)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("pond.json", WaterType.POND)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("harbour.json", WaterType.HARBOUR)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("canal.json", WaterType.CANAL)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("creek.json", WaterType.CREEK)); this.waterConfigurationReaders .add(new DefaultWaterConfigurationReader("ditch.json", WaterType.DITCH)); } public ComplexWaterEntityFinder(final WaterConfigurationReader... waterConfigurationReaders) { // use the passed in configuration readers for the filters this.waterConfigurationReaders = Arrays.asList(waterConfigurationReaders); } @Override public Iterable find(final Atlas atlas) { final Stream lineStream = StreamSupport.stream(atlas.lines().spliterator(), false); final Stream areaStream = StreamSupport.stream(atlas.areas().spliterator(), false); final Stream relationStream = StreamSupport .stream(atlas.relations(RELATION_FILTER).spliterator(), false); return Stream.concat(Stream.concat(lineStream, areaStream), relationStream) .map(this::processEntity).filter(Optional::isPresent).map(Optional::get) .collect(Collectors.toList()); } public List getWaterConfigurationReaders() { return this.waterConfigurationReaders; } /** * Convert {@link AtlasEntity} to an Optional {@link ComplexWaterEntity}. Sometimes an * {@link AtlasEntity} might not pass any of the filters in the configuration files. In that * case return empty optional * * @param atlasEntity * {@link AtlasEntity} which needs to be converted * @return Optional {@link ComplexWaterEntity} */ public Optional processEntity(final AtlasEntity atlasEntity) { // pass the atlas entity through all the filters in the configuration files and try to // create a complex water entity for each passed filter final List complexWaterEntities = this.waterConfigurationReaders .stream() .map(waterConfigurationReader -> waterConfigurationReader.convert(atlasEntity)) .filter(Optional::isPresent).map(Optional::get) .filter(object -> object instanceof ComplexWaterEntity) .map(object -> (ComplexWaterEntity) object).collect(Collectors.toList()); if (complexWaterEntities.isEmpty()) { logger.trace("AtlasEntity: {} did not match any water type filters", atlasEntity); return Optional.empty(); } else if (complexWaterEntities.size() > 1) { // Each AtlasEntity should pass only one WaterType configuration. If is passes more than // one means the taggable filters specified in the configurations has an overlap. Due to // ambiguous taggable filters specified this method would return empty optional final String matchedWaterBodies = complexWaterEntities.stream() .map(ComplexWaterEntity::getWaterType).map(WaterType::toString) .collect(Collectors.joining(",")); logger.error("Skipping AtlasEnity : {} as it got mapped to multiple types: {}", atlasEntity, matchedWaterBodies); return Optional.empty(); } // return the ComplexWaterEntity matched with the only water type configuration return Optional.of(complexWaterEntities.get(0)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/finder/DefaultWaterConfigurationReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water.finder; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.complex.water.ComplexWaterEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.water.ComplexWaterbody; import org.openstreetmap.atlas.geography.atlas.items.complex.water.ComplexWaterway; import org.openstreetmap.atlas.geography.atlas.items.complex.water.WaterType; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.configuration.ConfiguredFilter; import org.openstreetmap.atlas.utilities.configuration.StandardConfiguration; /** * Configuration reader which would read a configuration file and only convert the default * configuration filter into a {@link ConfiguredFilter} * * @author sbhalekar */ public class DefaultWaterConfigurationReader extends WaterConfigurationReader { public DefaultWaterConfigurationReader(final Resource resource, final WaterType waterType) { super(resource, waterType); } public DefaultWaterConfigurationReader(final String resourceFileName, final WaterType waterType) { this(new InputStreamResource( () -> DefaultWaterConfigurationReader.class.getResourceAsStream(resourceFileName)), waterType); } @Override public ConfiguredFilter readConfiguration(final Resource configurationResource) { return ConfiguredFilter.getDefaultFilter(new StandardConfiguration(configurationResource)); } @Override protected Optional createComplexEntity(final AtlasEntity atlasEntity) { final WaterType waterBodyType = this.getWaterBodyType(); Optional complexWaterEntity = Optional.empty(); if (this.getConfigurationMapper().test(atlasEntity)) { if (atlasEntity instanceof Relation || atlasEntity instanceof Area) { complexWaterEntity = Optional.of(new ComplexWaterbody(atlasEntity, waterBodyType)); } else if (atlasEntity instanceof Line) { complexWaterEntity = Optional.of(new ComplexWaterway(atlasEntity, waterBodyType)); } } return complexWaterEntity; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex/water/finder/WaterConfigurationReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.items.complex.water.finder; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity; import org.openstreetmap.atlas.geography.atlas.items.complex.WaterIslandConfigurationReader; import org.openstreetmap.atlas.geography.atlas.items.complex.water.WaterType; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Class which ties a water configuration reader to {@link WaterType} * * @param * Type of objects which will map {@link AtlasEntity} to any {@link ComplexEntity} * @author sbhalekar */ public abstract class WaterConfigurationReader extends WaterIslandConfigurationReader { private final WaterType waterType; public WaterConfigurationReader(final Resource configurationResource, final WaterType waterType) { super(configurationResource); this.waterType = waterType; } protected WaterType getWaterBodyType() { return this.waterType; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; import javax.annotation.Nullable; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A lightweight area. * * @author Taylor Smock */ public class LightArea extends Area implements LightLineItem { private static final byte HASH_BYTE = 31; private final long identifier; private final long[] relationIdentifiers; private final Location[] locations; /** * Create a new area from another area * * @param from * The area to clone * @return A new LightArea */ static LightArea from(final Area from) { return new LightArea(from); } /** * Create a new LightArea with just an identifier * * @param identifier * The identifier */ LightArea(final long identifier) { this(identifier, EMPTY_LOCATION_ARRAY); } /** * Create a new LightArea with just an identifier and points * * @param identifier * The identifier * @param points * The points of the area */ LightArea(final long identifier, final Location... points) { super(new EmptyAtlas()); this.identifier = identifier; this.relationIdentifiers = EMPTY_LONG_ARRAY; this.locations = points.length > 0 ? points.clone() : points; } /** * Create a new LightArea from another Area * * @param from * The area to copy from */ LightArea(final Area from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.relationIdentifiers = from.relations().stream().mapToLong(Relation::getIdentifier) .toArray(); this.locations = from.asPolygon().toArray(EMPTY_LOCATION_ARRAY); } @Override @Nullable public PolyLine asPolyLine() { if (this.locations.length > 0) { return new PolyLine(this.locations); } return null; } @Override @Nullable public Polygon asPolygon() { if (this.locations.length > 0) { return new Polygon(this.locations); } return null; } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } if (!super.equals(other)) { return false; } final var lightArea = (LightArea) other; return this.identifier == lightArea.identifier && Arrays.equals(this.relationIdentifiers, lightArea.relationIdentifiers) && Arrays.equals(this.locations, lightArea.locations); } @Nullable @Override public Iterable getGeometry() { if (this.locations.length > 0) { return Iterables.asList(this.locations); } return null; } @Override public long getIdentifier() { return this.identifier; } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); result = HASH_BYTE * result + Long.hashCode(this.identifier); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.locations); return result; } /** * Please note that the relations returned from this method should *only* be used for * identifiers. * * @see Area#relations() * @return A set of identifier only relations */ @Override public Set relations() { return LongStream.of(this.relationIdentifiers).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * A lightweight edge with basic information * * @author Taylor Smock */ public class LightEdge extends Edge implements LightLineItem { private static final byte HASH_BYTE = 31; private final long identifier; private final long[] relationIdentifiers; private final long startNodeIdentifier; private final long endNodeIdentifier; private final Location[] pointLocations; /** * Create a new LightEdge from another Edge * * @param from * The edge to copy from * @return A new LightEdge */ static LightEdge from(final Edge from) { return new LightEdge(from); } /** * Create a LightEdge with just an identifier * * @param identifier * The identifier */ LightEdge(final long identifier) { this(identifier, EMPTY_LOCATION_ARRAY); } /** * Create a new LightEdge with an identifier and locations * * @param identifier * The identifier * @param points * The location points */ LightEdge(final long identifier, final Location... points) { super(new EmptyAtlas()); this.identifier = identifier; this.relationIdentifiers = EMPTY_LONG_ARRAY; this.startNodeIdentifier = 0; this.endNodeIdentifier = 0; this.pointLocations = points.length > 0 ? points.clone() : points; } /** * Create a new LightEdge from another Edge * * @param from * The edge to copy from */ LightEdge(final Edge from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.relationIdentifiers = from.relations().stream().mapToLong(Relation::getIdentifier) .toArray(); this.startNodeIdentifier = from.start().getIdentifier(); this.endNodeIdentifier = from.end().getIdentifier(); this.pointLocations = from.asPolyLine().toArray(EMPTY_LOCATION_ARRAY); } @Override public PolyLine asPolyLine() { if (this.pointLocations.length > 0) { return new PolyLine(this.pointLocations); } return null; } @Override public Node end() { if (this.pointLocations.length > 0) { return new LightNode(this.endNodeIdentifier, this.pointLocations[this.pointLocations.length - 1]); } return null; } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } if (!super.equals(other)) { return false; } final var lightEdge = (LightEdge) other; return this.identifier == lightEdge.identifier && this.startNodeIdentifier == lightEdge.startNodeIdentifier && this.endNodeIdentifier == lightEdge.endNodeIdentifier && Arrays.equals(this.relationIdentifiers, lightEdge.relationIdentifiers) && Arrays.equals(this.pointLocations, lightEdge.pointLocations); } @Override public long getIdentifier() { return this.identifier; } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); result = HASH_BYTE * result + Long.hashCode(this.identifier); result = HASH_BYTE * result + Long.hashCode(this.startNodeIdentifier); result = HASH_BYTE * result + Long.hashCode(this.endNodeIdentifier); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.pointLocations); return result; } @Override public Set relations() { return LongStream.of(this.getRelationIdentifiers()).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } @Override public Node start() { if (this.pointLocations.length > 0) { return new LightNode(this.startNodeIdentifier, this.pointLocations[0]); } return null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightEntity.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import javax.annotation.Nullable; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * A lightweight Atlas entity. In this case, light weight refers to size in memory. These are * primarily useful for raw identifier uses. First generation lightweight entities may also have * geometry. * * @param * The primitive type * @author Taylor Smock */ public interface LightEntity> { /** A common empty long array to avoid additional empty arrays (memory) */ long[] EMPTY_LONG_ARRAY = {}; /** A common empty location array to avoid additional empty arrays (memory) */ Location[] EMPTY_LOCATION_ARRAY = {}; /** * Create a {@link LightEntity} from a given {@link AtlasEntity} reference. The * {@link LightEntity}'s fields will match the fields of the reference. The returned * {@link LightEntity} may or may not be full, i.e. some of its associated fields will be * {@code null}. Currently no tags are saved in light entities. The only items guaranteed * to exist is the id of the entity. First generation entities also have geometry, but any * second generation entities do not. For example, the {@link AtlasEntity#relations()} method * only returns {@link LightRelation}s with an identifier and no geometry. For this reason, * {@link LightEntity}s should only be used with code that expects {@link LightEntity}s. * * @param reference * the reference to copy * @return the full entity */ static AtlasEntity from(final AtlasEntity reference) { final ItemType type = reference.getType(); switch (type) { case NODE: return LightNode.from((Node) reference); case EDGE: return LightEdge.from((Edge) reference); case AREA: return LightArea.from((Area) reference); case LINE: return LightLine.from((Line) reference); case POINT: return LightPoint.from((Point) reference); case RELATION: return LightRelation.from((Relation) reference); default: throw new CoreException("Unknown ItemType {}", type); } } /** * Get the geometry of the entity * * @return The geometry */ @Nullable Iterable getGeometry(); /** * Get relation identifiers * * @return The relation identifiers */ long[] getRelationIdentifiers(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A lightweight line * * @author Taylor Smock */ public class LightLine extends Line implements LightLineItem { private static final byte HASH_BYTE = 31; private final long identifier; private final long[] relationIdentifiers; private final Location[] locations; /** * Create a new LightLine from another line * * @param from * The line to copy from * @return A new LightLine */ static LightLine from(final Line from) { return new LightLine(from); } /** * Create a new LightLine with just an identifier * * @param identifier * The identifier */ LightLine(final long identifier) { this(identifier, EMPTY_LOCATION_ARRAY); } /** * Create a new LightLine with the specified points * * @param identifier * The identifier for the Line * @param points * The points for the line */ LightLine(final long identifier, final Location... points) { super(new EmptyAtlas()); this.identifier = identifier; this.relationIdentifiers = EMPTY_LONG_ARRAY; this.locations = points.length > 0 ? points.clone() : points; } /** * Create a new LightLine from another line * * @param from * The line to copy from */ LightLine(final Line from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.relationIdentifiers = from.relations().stream().mapToLong(Relation::getIdentifier) .toArray(); this.locations = Iterables.stream(from.asPolyLine()).collectToList() .toArray(Location[]::new); } @Override public PolyLine asPolyLine() { if (this.locations.length > 0) { return new PolyLine(this.locations); } return null; } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } final var lightLine = (LightLine) other; return this.identifier == lightLine.identifier && Arrays.equals(this.relationIdentifiers, lightLine.relationIdentifiers) && Arrays.equals(this.locations, lightLine.locations); } @Override public long getIdentifier() { return this.identifier; } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); result = HASH_BYTE * result + Long.hashCode(this.identifier); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.locations); return result; } /** * Please note that the relations returned from this method should *only* be used for * identifiers. * * @see Line#relations() * @return A set of identifier only relations */ @Override public Set relations() { return LongStream.of(this.getRelationIdentifiers()).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightLineItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import javax.annotation.Nullable; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; /** * A common interface for line items * * @param * The line item type * @author Taylor Smock */ public interface LightLineItem> extends LightEntity { PolyLine asPolyLine(); @Nullable default Iterable getGeometry() { return this.asPolyLine(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightLocationItem.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Collections; import org.openstreetmap.atlas.geography.Location; /** * A light location item * * @param * The type of location item * @author Taylor Smock */ public interface LightLocationItem> extends LightEntity> { @Override default Iterable getGeometry() { return Collections.singleton(this.getLocation()); } Location getLocation(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Arrays; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * A lightweight node with only basic information * * @author Taylor Smock */ public class LightNode extends Node implements LightLocationItem { private static final byte HASH_BYTE = 31; private final long identifier; private final Location location; private final long[] inEdgeIdentifiers; private final long[] outEdgeIdentifiers; private final long[] relationIdentifiers; /** * Create a new LightNode from a Node * * @param node * The node to copy from * @return A LightNode */ static LightNode from(final Node node) { return new LightNode(node); } /** * Create a LightNode with just an identifier * * @param identifier * The identifier */ LightNode(final long identifier) { this(identifier, null); } /** * Create a LightNode from an identifier and a location * * @param identifier * The identifier * @param location * The location */ LightNode(final long identifier, final Location location) { super(new EmptyAtlas()); this.identifier = identifier; this.location = location; this.inEdgeIdentifiers = EMPTY_LONG_ARRAY; this.outEdgeIdentifiers = EMPTY_LONG_ARRAY; this.relationIdentifiers = EMPTY_LONG_ARRAY; } /** * Create a new LightNode from another Node * * @param from * The ndoe to copy from */ LightNode(final Node from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.location = from.getLocation(); this.inEdgeIdentifiers = from.inEdges().stream().map(Edge::getIdentifier).distinct() .mapToLong(Long::longValue).toArray(); this.outEdgeIdentifiers = from.outEdges().stream().map(Edge::getIdentifier).distinct() .mapToLong(Long::longValue).toArray(); this.relationIdentifiers = from.relations().stream().map(Relation::getIdentifier).distinct() .mapToLong(Long::longValue).toArray(); } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } final var lightNode = (LightNode) other; if (this.location != null && lightNode.location != null) { return this.identifier == lightNode.identifier && this.location.equals(lightNode.location) && Arrays.equals(this.inEdgeIdentifiers, lightNode.inEdgeIdentifiers) && Arrays.equals(this.outEdgeIdentifiers, lightNode.outEdgeIdentifiers) && Arrays.equals(this.relationIdentifiers, lightNode.relationIdentifiers); } else if (this.location == null && lightNode.location == null) { return this.identifier == lightNode.identifier && Arrays.equals(this.inEdgeIdentifiers, lightNode.inEdgeIdentifiers) && Arrays.equals(this.outEdgeIdentifiers, lightNode.outEdgeIdentifiers) && Arrays.equals(this.relationIdentifiers, lightNode.relationIdentifiers); } return false; } @Override public long getIdentifier() { return this.identifier; } /** * Get the identifiers for in edges * * @return The identifiers for in edges */ public long[] getInEdgeIdentifiers() { return this.inEdgeIdentifiers.clone(); } @Override public Location getLocation() { return this.location; } /** * Get the identifiers for out edges * * @return The identifiers for out edges */ public long[] getOutEdgeIdentifiers() { return this.outEdgeIdentifiers.clone(); } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); result = HASH_BYTE * result + Long.hashCode(this.identifier); if (this.location != null) { result = HASH_BYTE * result + this.location.hashCode(); } result = HASH_BYTE * result + Arrays.hashCode(this.inEdgeIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.outEdgeIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); return result; } /** * Please note that the edges returned from this method should *only* be used for identifiers. * * @see Node#inEdges() * @return A set of identifier only edges */ @Override public SortedSet inEdges() { return LongStream.of(this.inEdgeIdentifiers).mapToObj(LightEdge::new) .collect(Collectors.toCollection(TreeSet::new)); } /** * Please note that the edges returned from this method should *only* be used for identifiers. * * @see Node#outEdges() * @return A set of identifier only edges */ @Override public SortedSet outEdges() { return LongStream.of(this.outEdgeIdentifiers).mapToObj(LightEdge::new) .collect(Collectors.toCollection(TreeSet::new)); } /** * Please note that the relations returned from this method should *only* be used for * identifiers. * * @see Node#relations() * @return A set of identifier only relations */ @Override public Set relations() { return LongStream.of(this.getRelationIdentifiers()).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightPoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.Arrays; import java.util.Collections; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * A lightweight Point. If it doesn't need to be stored in a field, it isn't. * * @author Taylor Smock */ public class LightPoint extends Point implements LightLocationItem { private static final byte HASH_BYTE = 31; private final long identifier; private final Location location; private final long[] relationIdentifiers; static LightPoint from(final Point point) { return new LightPoint(point); } /** * Create a new light point. * * @param identifier * The identifier for the new point * @param location * The location of the point * @param relationIdentifiers * Any relations for the point */ public LightPoint(final Long identifier, final Location location, final Set relationIdentifiers) { super(new EmptyAtlas()); this.identifier = identifier; this.location = location; this.relationIdentifiers = relationIdentifiers.stream().mapToLong(Long::longValue) .toArray(); } /** * A basic point with just an identifier * * @param identifier * The identifier for the point */ LightPoint(final long identifier) { this(identifier, null); } /** * Create a new LightPoint with an id and location * * @param identifier * The identifier * @param location * The location */ LightPoint(final long identifier, final Location location) { this(identifier, location, Collections.emptySet()); } /** * Create a lightweight point from another point * * @param from * The point to copy information from */ LightPoint(final Point from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.location = from.getLocation(); this.relationIdentifiers = from.relations().stream().mapToLong(Relation::getIdentifier) .toArray(); } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } final var lightPoint = (LightPoint) other; if (this.location != null && lightPoint.location != null) { return this.identifier == lightPoint.identifier && this.location.equals(lightPoint.location) && Arrays.equals(this.relationIdentifiers, lightPoint.relationIdentifiers); } else if (this.location == null && lightPoint.location == null) { return this.identifier == lightPoint.identifier && Arrays.equals(this.relationIdentifiers, lightPoint.relationIdentifiers); } return false; } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return this.location; } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); if (this.location != null) { result = HASH_BYTE * result + this.location.hashCode(); } result = HASH_BYTE * result + Long.hashCode(this.identifier); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); return result; } /** * Please note that the relations returned from this method should *only* be used for * identifiers. * * @see Point#relations() * @return A set of identifier only relations */ @Override public Set relations() { return LongStream.of(this.getRelationIdentifiers()).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/lightweight/LightRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.lightweight; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; import javax.annotation.Nullable; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.complete.EmptyAtlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import com.google.common.collect.Iterables; /** * A minimal relation class. Anything that does not need to be stored in memory isn't stored in * memory. * * @author Taylor Smock */ public class LightRelation extends Relation implements LightEntity { private static final byte HASH_BYTE = 31; private static final String[] EMPTY_STRING_ARRAY = {}; private static final ItemType[] EMPTY_ITEM_TYPE_ARRAY = {}; private static final Location[][] EMPTY_LOCATIONS_ARRAY = {}; private final long identifier; private final long[] relationIdentifiers; private final long[] memberIdentifiers; private final ItemType[] memberTypes; private final String[] memberRoles; private final Location[][] memberLocations; /** * Create a new LightRelation from a relation * * @param relation * The relation to copy * @return The generated light relation */ static LightRelation from(final Relation relation) { return new LightRelation(relation); } /** * Create a new LightRelation with just an identifier * * @param identifier * The relation identifier */ LightRelation(final long identifier) { super(new EmptyAtlas()); this.identifier = identifier; this.relationIdentifiers = EMPTY_LONG_ARRAY; this.memberIdentifiers = EMPTY_LONG_ARRAY; this.memberRoles = EMPTY_STRING_ARRAY; this.memberTypes = EMPTY_ITEM_TYPE_ARRAY; this.memberLocations = EMPTY_LOCATIONS_ARRAY; } /** * Create a new LightRelation with basic information from another relation * * @param from * The relation to copy from */ LightRelation(final Relation from) { super(new EmptyAtlas()); this.identifier = from.getIdentifier(); this.relationIdentifiers = from.relations().stream().mapToLong(Relation::getIdentifier) .toArray(); this.memberIdentifiers = from.allKnownOsmMembers().stream() .mapToLong(member -> member.getEntity().getIdentifier()).toArray(); this.memberTypes = from.allKnownOsmMembers().stream() .map(member -> member.getEntity().getType()).toArray(ItemType[]::new); this.memberRoles = from.allKnownOsmMembers().stream().map(RelationMember::getRole) .toArray(String[]::new); this.memberLocations = from.allKnownOsmMembers().stream().map(RelationMember::getEntity) .map(entity -> { if (entity instanceof AtlasItem) { return Iterables.toArray(((AtlasItem) entity).getRawGeometry(), Location.class); } else if (entity instanceof Relation) { // Don't store sub relation entity at this time -- we aren't storing the // necessary information for // sub relations anyway (we would need to store the sub relation member // information). return EMPTY_LOCATION_ARRAY; } else { throw new CoreException("{0} type not understood", entity.getType()); } }).toArray(Location[][]::new); } @Override public RelationMemberList allKnownOsmMembers() { return this.members(); } @Override public List allRelationsWithSameOsmIdentifier() { return Collections.emptyList(); } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || this.getClass() != other.getClass()) { return false; } final var that = (LightRelation) other; return this.identifier == that.identifier && Arrays.equals(this.relationIdentifiers, that.relationIdentifiers) && Arrays.equals(this.memberIdentifiers, that.memberIdentifiers) && Arrays.equals(this.memberTypes, that.memberTypes) && Arrays.equals(this.memberRoles, that.memberRoles) && Arrays.deepEquals(this.memberLocations, that.memberLocations); } @Nullable @Override public Iterable getGeometry() { return Stream.of(this.memberLocations).flatMap(Stream::of).collect(Collectors.toList()); } @Override public long getIdentifier() { return this.identifier; } @Override public long[] getRelationIdentifiers() { return this.relationIdentifiers.clone(); } @Override public int hashCode() { int result = super.hashCode(); result = HASH_BYTE * result + Long.hashCode(this.identifier); result = HASH_BYTE * result + Arrays.hashCode(this.relationIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.memberIdentifiers); result = HASH_BYTE * result + Arrays.hashCode(this.memberTypes); result = HASH_BYTE * result + Arrays.hashCode(this.memberRoles); result = HASH_BYTE * result + Arrays.deepHashCode(this.memberLocations); return result; } /** * Please note that the entities returned from this method should *only* be used for * identifiers. * * @see Relation#members() * @return A set of identifier only entities */ @Override public RelationMemberList members() { final List relationMemberList = new ArrayList<>( this.memberIdentifiers.length); for (var index = 0; index < this.memberIdentifiers.length; index++) { final AtlasEntity entity; final long memberIdentifier = this.memberIdentifiers[index]; final ItemType type = this.memberTypes[index]; if (type == ItemType.RELATION) { entity = new LightRelation(memberIdentifier); } else if (type == ItemType.AREA) { entity = new LightArea(memberIdentifier, this.memberLocations[index]); } else if (type == ItemType.EDGE) { entity = new LightEdge(memberIdentifier, this.memberLocations[index]); } else if (type == ItemType.LINE) { entity = new LightLine(memberIdentifier, this.memberLocations[index]); } else if (type == ItemType.NODE) { entity = new LightNode(memberIdentifier, this.memberLocations[index][0]); } else if (type == ItemType.POINT) { entity = new LightPoint(memberIdentifier, this.memberLocations[index][0]); } else { throw new CoreException("{0} is not a known type", this.memberTypes[index]); } relationMemberList .add(new RelationMember(this.memberRoles[index], entity, this.getIdentifier())); } return new RelationMemberList(relationMemberList); } @Override public Long osmRelationIdentifier() { return this.getOsmIdentifier(); } /** * Please note that the relations returned from this method should *only* be used for * identifiers. * * @see Relation#relations() * @return A set of identifier only relations */ @Override public Set relations() { return LongStream.of(this.getRelationIdentifiers()).mapToObj(LightRelation::new) .collect(Collectors.toSet()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.common.collect.Sets; /** * {@link Area} made from a {@link MultiAtlas}. * * @author matthieun */ public class MultiArea extends Area { private static final long serialVersionUID = 4710025391581335160L; // Not index! private final long identifier; private SubAreaList subAreaList; protected MultiArea(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public Polygon asPolygon() { return getRepresentativeSubArea().asPolygon(); } @Override public long getIdentifier() { return this.identifier; } public SubAreaList getSubAreas() { if (this.subAreaList == null) { this.subAreaList = multiAtlas().subAreas(this.identifier); } return this.subAreaList; } @Override public Map getTags() { return this.getRepresentativeSubArea().getTags(); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Area subArea : getSubAreas().getSubAreas()) { final Set currentSubAreaParentRelations = multiAtlas() .multifyRelations(subArea); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubAreaParentRelations); } return unionOfAllParentRelations; } private Area getRepresentativeSubArea() { return getSubAreas().getSubAreas().get(0); } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.AbstractAtlas; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.index.RTree; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.maps.LongToIntegerMultiMap; import org.openstreetmap.atlas.utilities.maps.LongToLongMap; import org.openstreetmap.atlas.utilities.maps.LongToLongMultiMap; import org.openstreetmap.atlas.utilities.scalars.Ratio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; /** * {@link Atlas} that is backed by multiple {@link Atlas}es stitched on the fly. * * @author matthieun */ public class MultiAtlas extends AbstractAtlas { private static final long serialVersionUID = -5917117808042670700L; private static final Logger logger = LoggerFactory.getLogger(MultiAtlas.class); private static final double ARRAY_SIZE_MULTIPLIER = 1.1; private final List atlases; private final AtlasMetaData metaData; private final long numberOfEdges; private final long numberOfNodes; private final long numberOfAreas; private final long numberOfLines; private final long numberOfPoints; private final long numberOfRelations; private final RTree atlasSpatialIndex; // The identifiers of the Nodes, and the Atlases we can find them into. Only shared nodes, used // for connectivity will be referenced in more than one Atlas (for real time stitching of the // navigable network). private final LongToIntegerMultiMap nodeIdentifierToAtlasIndices; private final LongToIntegerMultiMap edgeIdentifierToAtlasIndices; private final LongToIntegerMultiMap areaIdentifierToAtlasIndices; private final LongToIntegerMultiMap lineIdentifierToAtlasIndices; private final LongToIntegerMultiMap pointIdentifierToAtlasIndices; private final LongToIntegerMultiMap relationIdentifierToAtlasIndices; // Re-build relations with the same OSM identifier private final LongToLongMultiMap relationOsmIdentifierToRelationIdentifiers; private final LongToLongMap relationIdentifierToRelationOsmIdentifier; private final int subArraySize; private final long maximumSize; private final int nodeMemoryBlockSize; private final int edgeMemoryBlockSize; private final int areaMemoryBlockSize; private final int lineMemoryBlockSize; private final int pointMemoryBlockSize; private final int relationMemoryBlockSize; private final int nodeHashSize; private final int edgeHashSize; private final int areaHashSize; private final int lineHashSize; private final int pointHashSize; private final int relationHashSize; // Custom fixers for crossing edges and overlapping nodes private final MultiAtlasOverlappingNodesFixer nodesFixer; /** * Load a {@link MultiAtlas} from a serialized resource * * @param resource * The {@link Resource} to read from * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas load(final Resource resource) { try (ObjectInputStream input = new ObjectInputStream(resource.read())) { final MultiAtlas result = (MultiAtlas) input.readObject(); return result; } catch (final Exception e) { throw new CoreException("Could not load Atlas from {}", e, resource); } } /** * Load a {@link MultiAtlas} from an {@link Iterable} of {@link PackedAtlas} serialized * resources * * @param resources * The {@link Resource}s to read from (which each contain a serialized * {@link PackedAtlas}). * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas loadFromPackedAtlas(final Iterable resources) { return loadFromPackedAtlas(resources, false); } /** * Load a {@link MultiAtlas} from an {@link Iterable} of {@link PackedAtlas} serialized * resources * * @param resources * The {@link Resource}s to read from (which each contain a serialized * {@link PackedAtlas}). * @param lotsOfOverlap * If this is true, then the builder will start with small arrays and re-size a lot, * but won't waste memory because of the overlap of the sub-{@link Atlas}es. However, * if this is false, the builder will blindly sum all the items of all the * {@link Atlas}es regardless of overlap, and will hence allocate potentially more * memory than necessary. In that case though, the arrays will never resize, and the * load time will be faster. * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas loadFromPackedAtlas(final Iterable resources, final boolean lotsOfOverlap) { if (Iterables.size(resources) == 0) { throw new CoreException("Can't create an atlas from zero resources"); } return new MultiAtlas(Iterables.translate(resources, resource -> { try { return PackedAtlas.load(resource); } catch (final Exception exception) { throw new CoreException("Failed to load an atlas from {} with name {}", resource.getClass().getName(), resource.getName(), exception); } }), lotsOfOverlap); } /** * Load a {@link MultiAtlas} from an {@link Iterable} of {@link PackedAtlas} serialized * resources * * @param resources * The {@link Resource}s to read from (which each contain a serialized * {@link PackedAtlas}). * @param lotsOfOverlap * If this is true, then the builder will start with small arrays and re-size a lot, * but won't waste memory because of the overlap of the sub-{@link Atlas}es. However, * if this is false, the builder will blindly sum all the items of all the * {@link Atlas}es regardless of overlap, and will hence allocate potentially more * memory than necessary. In that case though, the arrays will never resize, and the * load time will be faster. * @param filter * The {@link Predicate} to use when loading from {@link PackedAtlas} * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas loadFromPackedAtlas(final Iterable resources, final boolean lotsOfOverlap, final Predicate filter) { if (Iterables.size(resources) == 0) { throw new CoreException("Can't create an atlas from zero resources"); } final Iterable validAtlases = Iterables .translate(Iterables.stream(Iterables.translate(resources, resource -> { final Atlas atlas; try { atlas = PackedAtlas.load(resource); } catch (final Exception exception) { throw new CoreException("Failed to load an atlas from {} with name {}", resource.getClass().getName(), resource.getName(), exception); } return atlas.subAtlas(filter, AtlasCutType.SOFT_CUT); })).filter(Optional::isPresent), Optional::get); if (Iterables.size(validAtlases) == 0) { throw new CoreException("Provided entity filter will result in an empty Atlas"); } return new MultiAtlas(validAtlases, lotsOfOverlap); } /** * Load a {@link MultiAtlas} from an {@link Iterable} of {@link PackedAtlas} serialized * resources * * @param resources * The {@link Resource}s to read from (which each contain a serialized * {@link PackedAtlas}). * @param filter * The {@link Predicate} to use when loading from {@link PackedAtlas} * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas loadFromPackedAtlas(final Iterable resources, final Predicate filter) { return loadFromPackedAtlas(resources, false, filter); } /** * Load a {@link MultiAtlas} from {@link PackedAtlas} serialized resources * * @param resources * The {@link Resource}s to read from (which each contain a serialized * {@link PackedAtlas}). * @return The deserialized {@link MultiAtlas} */ public static MultiAtlas loadFromPackedAtlas(final Resource... resources) { return loadFromPackedAtlas(Iterables.iterable(resources), false); } /** * Create an {@link Atlas} from stitching many other {@link Atlas} * * @param atlases * The {@link Atlas}es to stitch together */ public MultiAtlas(final Atlas... atlases) { this(Iterables.iterable(atlases), false); } /** * Create an {@link Atlas} from stitching many other {@link Atlas} * * @param atlases * The {@link Atlas}es to stitch together */ public MultiAtlas(final Iterable atlases) { this(atlases, false); } /** * Create an {@link Atlas} from stitching many other {@link Atlas} * * @param atlases * The {@link Atlas}es to stitch together * @param lotsOfOverlap * If this is true, then the builder will start with small arrays and re-size a lot, * but won't waste memory because of the overlap of the sub-{@link Atlas}es. However, * if this is false, the builder will blindly sum all the items of all the * {@link Atlas}es regardless of overlap, and will hence allocate potentially more * memory than necessary. In that case though, the arrays will never resize, and the * load time will be faster. */ public MultiAtlas(final Iterable atlases, final boolean lotsOfOverlap) { this(Iterables.asList(atlases), lotsOfOverlap); } /** * Create an {@link Atlas} from stitching many other {@link Atlas} * * @param atlases * The {@link Atlas}es to stitch together */ public MultiAtlas(final List atlases) { this(atlases, false); } /** * Create an {@link Atlas} from stitching many other {@link Atlas} * * @param atlases * The {@link Atlas}es to stitch together * @param lotsOfOverlap * If this is true, then the builder will start with small arrays and re-size a lot, * but won't waste memory because of the overlap of the sub-{@link Atlas}es. However, * if this is false, the builder will blindly sum all the items of all the * {@link Atlas}es regardless of overlap, and will hence allocate potentially more * memory than necessary. In that case though, the arrays will never resize, and the * load time will be faster. */ public MultiAtlas(final List atlases, final boolean lotsOfOverlap) { this(atlases, lotsOfOverlap, true); } public MultiAtlas(final boolean fixNodesOnOppositeAntiMeridians, final Atlas... atlases) { this(Iterables.iterable(atlases), false, fixNodesOnOppositeAntiMeridians); } public MultiAtlas(final Iterable atlases, final boolean lotsOfOverlap, final boolean fixNodesOnOppositeAntiMeridians) { this(Iterables.asList(atlases), lotsOfOverlap, fixNodesOnOppositeAntiMeridians); } public MultiAtlas(final List atlases, final boolean lotsOfOverlap, final boolean fixNodesOnOppositeAntiMeridians) { if (atlases.isEmpty()) { throw new CoreException("An Atlas is Located, and therefore cannot be empty."); } this.atlases = atlases; this.atlasSpatialIndex = newPackedAtlasSpatialIndex(); long numberOfNodes; long numberOfEdges; long numberOfAreas; long numberOfLines; long numberOfPoints; long numberOfRelations; if (lotsOfOverlap) { // We do not know in advance how much overlap there will be. We choose to re-size // instead of wasting memory numberOfEdges = DEFAULT_NUMBER_OF_ITEMS; numberOfNodes = DEFAULT_NUMBER_OF_ITEMS; numberOfAreas = DEFAULT_NUMBER_OF_ITEMS; numberOfLines = DEFAULT_NUMBER_OF_ITEMS; numberOfPoints = DEFAULT_NUMBER_OF_ITEMS; numberOfRelations = DEFAULT_NUMBER_OF_ITEMS; } else { // Re-sizing arrays is prohibitive, so even if we do not know how much overlap there // will be, we choose to waste memory instead. numberOfNodes = Iterables.count(this.atlases, atlas -> atlas.numberOfNodes()); numberOfEdges = Iterables.count(this.atlases, atlas -> atlas.numberOfEdges()); numberOfAreas = Iterables.count(this.atlases, atlas -> atlas.numberOfAreas()); numberOfLines = Iterables.count(this.atlases, atlas -> atlas.numberOfLines()); numberOfPoints = Iterables.count(this.atlases, atlas -> atlas.numberOfPoints()); numberOfRelations = Iterables.count(this.atlases, atlas -> atlas.numberOfRelations()); } if (atlases.size() > 1) { numberOfNodes = Math.round(numberOfNodes * ARRAY_SIZE_MULTIPLIER); numberOfEdges = Math.round(numberOfEdges * ARRAY_SIZE_MULTIPLIER); numberOfAreas = Math.round(numberOfAreas * ARRAY_SIZE_MULTIPLIER); numberOfLines = Math.round(numberOfLines * ARRAY_SIZE_MULTIPLIER); numberOfPoints = Math.round(numberOfPoints * ARRAY_SIZE_MULTIPLIER); numberOfRelations = Math.round(numberOfRelations * ARRAY_SIZE_MULTIPLIER); } int index = 0; for (final Atlas atlas : this.atlases) { this.atlasSpatialIndex.add(atlas.bounds(), index); index++; } this.subArraySize = Integer.MAX_VALUE; this.maximumSize = Long.MAX_VALUE; this.nodeMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfNodes % Integer.MAX_VALUE); this.edgeMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfEdges % Integer.MAX_VALUE); this.areaMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfAreas % Integer.MAX_VALUE); this.lineMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfLines % Integer.MAX_VALUE); this.pointMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfPoints % Integer.MAX_VALUE); this.relationMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, numberOfRelations % Integer.MAX_VALUE); this.nodeHashSize = (int) Math .max(Math.min(numberOfNodes / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.edgeHashSize = (int) Math .max(Math.min(numberOfEdges / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.areaHashSize = (int) Math .max(Math.min(numberOfAreas / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.lineHashSize = (int) Math .max(Math.min(numberOfLines / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.pointHashSize = (int) Math .max(Math.min(numberOfPoints / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.relationHashSize = (int) Math .max(Math.min(numberOfRelations / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.nodeIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - nodeIdentifierToAtlasIndices", this.maximumSize, this.nodeHashSize, this.nodeMemoryBlockSize, this.subArraySize, this.nodeMemoryBlockSize, this.subArraySize); this.edgeIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - edgeIdentifierToAtlasIndex", this.maximumSize, this.edgeHashSize, this.edgeMemoryBlockSize, this.subArraySize, this.edgeMemoryBlockSize, this.subArraySize); this.areaIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - areaIdentifierToAtlasIndex", this.maximumSize, this.areaHashSize, this.areaMemoryBlockSize, this.subArraySize, this.areaMemoryBlockSize, this.subArraySize); this.lineIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - lineIdentifierToAtlasIndex", this.maximumSize, this.lineHashSize, this.lineMemoryBlockSize, this.subArraySize, this.lineMemoryBlockSize, this.subArraySize); this.pointIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - pointIdentifierToAtlasIndex", this.maximumSize, this.pointHashSize, this.pointMemoryBlockSize, this.subArraySize, this.pointMemoryBlockSize, this.subArraySize); this.relationIdentifierToAtlasIndices = new LongToIntegerMultiMap( "MultiAtlas - relationIdentifierToAtlasIndices", this.maximumSize, this.relationHashSize, this.relationMemoryBlockSize, this.subArraySize, this.relationMemoryBlockSize, this.subArraySize); this.relationOsmIdentifierToRelationIdentifiers = new LongToLongMultiMap( "MultiAtlas - relationOsmIdentifierToRelationIdentifier", this.maximumSize, this.relationHashSize, this.relationMemoryBlockSize, this.subArraySize, this.relationMemoryBlockSize, this.subArraySize); this.relationIdentifierToRelationOsmIdentifier = new LongToLongMap( "MultiAtlas - relationIdentifierToRelationOsmIdentifier", this.maximumSize, this.relationHashSize, this.relationMemoryBlockSize, this.subArraySize, this.relationMemoryBlockSize, this.subArraySize); // Populate the pointers int atlasIndex = 0; for (final Atlas atlas : this.atlases) { populateReferences(atlas, atlasIndex); atlasIndex++; } // Build the spatial indices this.getAsNewNodeSpatialIndex(); this.getAsNewEdgeSpatialIndex(); this.getAsNewAreaSpatialIndex(); this.getAsNewLineSpatialIndex(); this.getAsNewPointSpatialIndex(); this.getAsNewRelationSpatialIndex(); this.nodeIdentifierToAtlasIndices .forEach(identifier -> this.getNodeSpatialIndex().add(node(identifier))); this.edgeIdentifierToAtlasIndices .forEach(identifier -> this.getEdgeSpatialIndex().add(edge(identifier))); this.areaIdentifierToAtlasIndices .forEach(identifier -> this.getAreaSpatialIndex().add(area(identifier))); this.lineIdentifierToAtlasIndices .forEach(identifier -> this.getLineSpatialIndex().add(line(identifier))); this.pointIdentifierToAtlasIndices .forEach(identifier -> this.getPointSpatialIndex().add(point(identifier))); this.relationIdentifierToAtlasIndices.forEach(identifier -> { final Relation relation = relation(identifier); if (!relation.members().isEmpty() && relation.bounds() != null) { // The relation is not empty, hence it is located this.getRelationSpatialIndex().add(relation); } final long osmIdentifier = relation.osmRelationIdentifier(); this.relationOsmIdentifierToRelationIdentifiers.add(osmIdentifier, identifier); this.relationIdentifierToRelationOsmIdentifier.put(identifier, osmIdentifier); }); // Find the overlapping nodes. Main to alternate has a one to many relationship. A main // cannot be an alternate and vice versa this.nodesFixer = new MultiAtlasOverlappingNodesFixer(this, fixNodesOnOppositeAntiMeridians); this.nodesFixer.aggregateSameLocationNodes(); // At this point de-duplication has been done already. this.numberOfEdges = this.edgeIdentifierToAtlasIndices.size(); this.numberOfNodes = this.nodeIdentifierToAtlasIndices.size(); this.numberOfAreas = this.areaIdentifierToAtlasIndices.size(); this.numberOfLines = this.lineIdentifierToAtlasIndices.size(); this.numberOfPoints = this.pointIdentifierToAtlasIndices.size(); this.numberOfRelations = this.relationIdentifierToAtlasIndices.size(); if (!lotsOfOverlap) { // In case we have small overlap and the arrays are larger than necessary, resize is // worth it if the arrays are more than twice as big as needed. final Ratio trimRatio = Ratio.HALF; this.nodeIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.edgeIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.areaIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.lineIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.pointIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.relationIdentifierToAtlasIndices.trimIfLessFilledThan(trimRatio); this.relationOsmIdentifierToRelationIdentifiers.trimIfLessFilledThan(trimRatio); this.relationIdentifierToRelationOsmIdentifier.trimIfLessFilledThan(trimRatio); } // Update the meta data. this.metaData = mergeMetaData(); } @Override public Area area(final long identifier) { if (this.areaIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiArea(this, identifier); } return null; } @Override public Iterable areas() { return Iterables.translate(this.areaIdentifierToAtlasIndices, this::area); } /** * Get all the Atlas intersecting some bounds * * @param bounds * The bounds * @return All the Atlas intersecting the bounds */ public Set atlasIntersecting(final Polygon bounds) { if (!(bounds instanceof Rectangle)) { throw new UnsupportedOperationException("Non-Rectangle Polygons not supported yet."); } return Iterables.asSet(Iterables.translate(this.atlasSpatialIndex.get(bounds.bounds()), atlasIndex -> this.atlases.get(atlasIndex))); } @Override public Rectangle bounds() { return Rectangle.forLocated(this.atlases); } @Override public Edge edge(final long identifier) { if (this.edgeIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiEdge(this, identifier); } return null; } @Override public Iterable edges() { return Iterables.translate(this.edgeIdentifierToAtlasIndices, this::edge); } @Override public Line line(final long identifier) { if (this.lineIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiLine(this, identifier); } return null; } @Override public Iterable lines() { return Iterables.translate(this.lineIdentifierToAtlasIndices, this::line); } @Override public AtlasMetaData metaData() { return this.metaData; } @Override public Node node(final long identifier) { if (this.nodeIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiNode(this, identifier); } return null; } @Override public Iterable nodes() { // Use the identifier here to avoid listing duplicated nodes twice. return Iterables.translate(this.nodeIdentifierToAtlasIndices, this::node); } @Override public long numberOfAreas() { return this.numberOfAreas; } @Override public long numberOfEdges() { return this.numberOfEdges; } @Override public long numberOfLines() { return this.numberOfLines; } @Override public long numberOfNodes() { return this.numberOfNodes; } @Override public long numberOfPoints() { return this.numberOfPoints; } @Override public long numberOfRelations() { return this.numberOfRelations; } public int numberOfSubAtlas() { return this.atlases.size(); } @Override public Point point(final long identifier) { if (this.pointIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiPoint(this, identifier); } return null; } @Override public Iterable points() { return Iterables.translate(this.pointIdentifierToAtlasIndices, this::point); } @Override public Relation relation(final long identifier) { if (this.relationIdentifierToAtlasIndices.containsKey(identifier)) { return new MultiRelation(this, identifier); } return null; } @Override public Iterable relations() { return Iterables.translate(this.relationIdentifierToAtlasIndices, this::relation); } @Override public void save(final WritableResource writableResource) { throw new CoreException( "A MultiAtlas has to be cloned to a {} before it can be saved. Consider using {}", PackedAtlas.class.getName(), PackedAtlasCloner.class.getName()); } /** * Get the List of {@link Atlas} that comprise this {@link MultiAtlas}. * * @return The list of {@link Atlas} */ public List subAtlases() { return ImmutableList.copyOf(this.atlases); } protected List getAtlases() { return this.atlases; } protected int getEdgeHashSize() { return this.edgeHashSize; } protected LongToIntegerMultiMap getEdgeIdentifierToAtlasIndices() { return this.edgeIdentifierToAtlasIndices; } protected int getEdgeMemoryBlockSize() { return this.edgeMemoryBlockSize; } protected long getMaximumSize() { return this.maximumSize; } protected LongToIntegerMultiMap getNodeIdentifierToAtlasIndices() { return this.nodeIdentifierToAtlasIndices; } protected int getSubArraySize() { return this.subArraySize; } /** * In case there is a main node overlapping this node, get the main node. * * @param identifier * The node identifier to query * @return The identifier of the main node that has the exact same location */ protected Optional mainNode(final Long identifier) { return this.nodesFixer.mainNode(identifier); } @Deprecated protected Optional masterNode(final Long identifier) { return mainNode(identifier); } /** * Take all the relations an {@link AtlasEntity} belongs to, and replace them with the * corresponding relations linking back to this {@link MultiAtlas}. * * @param entity * The {@link AtlasEntity} * @return The relations of this entity, as viewed from this multi atlas. */ protected Set multifyRelations(final AtlasEntity entity) { final Set subRelations = entity.relations(); final Set result = new HashSet<>(); for (final Relation relation : subRelations) { result.add(relation(relation.getIdentifier())); } return result; } /** * In case this is a main node, get all the overlapping nodes. * * @param identifier * The node identifier to query * @return The identifiers of the overlapping nodes that has the exact same location */ protected Set overlappingNodes(final Long identifier) { return this.nodesFixer.overlappingNodes(identifier); } protected void populateReferences(final Atlas atlas, final int atlasIndex) { for (final Node node : atlas.nodes()) { this.nodeIdentifierToAtlasIndices.add(node.getIdentifier(), atlasIndex); } for (final Edge edge : atlas.edges()) { this.edgeIdentifierToAtlasIndices.add(edge.getIdentifier(), atlasIndex); } for (final Area area : atlas.areas()) { this.areaIdentifierToAtlasIndices.add(area.getIdentifier(), atlasIndex); } for (final Line line : atlas.lines()) { this.lineIdentifierToAtlasIndices.add(line.getIdentifier(), atlasIndex); } for (final Point point : atlas.points()) { this.pointIdentifierToAtlasIndices.add(point.getIdentifier(), atlasIndex); } for (final Relation relation : atlas.relations()) { this.relationIdentifierToAtlasIndices.add(relation.getIdentifier(), atlasIndex); } } protected List relationAllRelationsWithSameOsmIdentifier(final long identifier) { final List result = new ArrayList<>(); final long osmIdentifier = this.relationIdentifierToRelationOsmIdentifier.get(identifier); for (final long candidateIdentifier : this.relationOsmIdentifierToRelationIdentifiers .get(osmIdentifier)) { result.add(relation(candidateIdentifier)); } return result; } /** * Get the {@link Area} from the {@link Atlas} that has this identifier. * * @param identifier * The identifier to query * @return The {@link Area}s that have this identifier */ SubAreaList subAreas(final long identifier) { final List subAreas = new ArrayList<>(); for (final int index : this.areaIdentifierToAtlasIndices.get(identifier)) { if (index != -1) { subAreas.add(this.atlases.get(index).area(identifier)); } } return new SubAreaList(subAreas); } /** * Get the {@link Edge}s from the {@link Atlas} that has this identifier. * * @param identifier * The identifier to query * @return The {@link Edge}s that have this identifier */ SubEdgeList subEdge(final long identifier) { final List subEdges = new ArrayList<>(); for (final int index : this.edgeIdentifierToAtlasIndices.get(identifier)) { if (index != -1) { subEdges.add(this.atlases.get(index).edge(identifier)); } } return new SubEdgeList(subEdges); } /** * Get the {@link Line}s from the {@link Atlas} that has this identifier. * * @param identifier * The identifier to query * @return The {@link Line}s that have this identifier */ SubLineList subLines(final long identifier) { final List subLines = new ArrayList<>(); for (final int index : this.lineIdentifierToAtlasIndices.get(identifier)) { if (index != -1) { subLines.add(this.atlases.get(index).line(identifier)); } } return new SubLineList(subLines); } /** * Get the nodes from different Atlases that have this identifier. * * @param identifier * The identifier to query * @return The {@link Node}s that have this identifier */ SubNodeList subNodes(final long identifier) { final List subNodes = new ArrayList<>(); for (final int index : this.nodeIdentifierToAtlasIndices.get(identifier)) { // Add all the sub nodes that come from regular sub atlas if (index != -1) { subNodes.add(this.atlases.get(index).node(identifier)); } } return new SubNodeList(subNodes); } /** * Get the {@link Point}s from the {@link Atlas} that has this identifier. * * @param identifier * The identifier to query * @return The {@link Point}s that have this identifier */ SubPointList subPoints(final long identifier) { final List subPoints = new ArrayList<>(); for (final int index : this.pointIdentifierToAtlasIndices.get(identifier)) { if (index != -1) { subPoints.add(this.atlases.get(index).point(identifier)); } } return new SubPointList(subPoints); } /** * Get the {@link Relation}s from the {@link Atlas} that have this identifier. * * @param identifier * The identifier to query * @return The {@link Relation}s that have this identifier */ SubRelationList subRelations(final long identifier) { final List subRelations = new ArrayList<>(); for (final int index : this.relationIdentifierToAtlasIndices.get(identifier)) { if (index != -1) { subRelations.add(this.atlases.get(index).relation(identifier)); } } return new SubRelationList(subRelations); } private AtlasMetaData mergeMetaData() { final AtlasSize size = new AtlasSize(this.numberOfEdges, this.numberOfNodes, this.numberOfAreas, this.numberOfLines, this.numberOfPoints, this.numberOfRelations); String codeVersion = null; String dataVersion = null; String shardName = null; final Map tags = Maps.hashMap(); // countries final StringList countries = new StringList(this.atlases.stream().map(Atlas::metaData) .map(AtlasMetaData::getCountry).filter(Optional::isPresent).map(Optional::get) .flatMap(value -> StringList.split(value, ",").stream()).distinct() .collect(Collectors.toList())); // code version final List codeVersions = this.atlases.stream().map(Atlas::metaData) .map(AtlasMetaData::getCodeVersion).filter(Optional::isPresent).map(Optional::get) .distinct().collect(Collectors.toList()); if (codeVersions.size() > 1) { logger.warn("Two sub atlas files have different code versions: {}", codeVersions); } if (codeVersions.size() > 0) { codeVersion = codeVersions.get(0); } // data version final List dataVersions = this.atlases.stream().map(Atlas::metaData) .map(AtlasMetaData::getDataVersion).filter(Optional::isPresent).map(Optional::get) .distinct().collect(Collectors.toList()); if (dataVersions.size() > 1) { logger.warn("Two sub atlas files have different data versions: {}", dataVersions); } if (dataVersions.size() > 0) { dataVersion = dataVersions.get(0); } // shard name final List shardNames = this.atlases.stream().map(Atlas::metaData) .map(AtlasMetaData::getShardName).filter(Optional::isPresent).map(Optional::get) .distinct().collect(Collectors.toList()); if (shardNames.size() > 1) { logger.warn("Two sub atlas files have different shard names: {}", shardNames); } if (shardNames.size() > 0) { shardName = shardNames.get(0); } // tags this.atlases.stream().map(Atlas::metaData).map(AtlasMetaData::getTags) .flatMap(map -> map.entrySet().stream()).forEach(entry -> { final String key = entry.getKey(); final String value = entry.getValue(); if (tags.containsKey(key)) { final String overridenValue = tags.get(key); if (overridenValue != null && !overridenValue.equals(value)) { logger.trace( "AtlasMetaData has conflicting values for the same key:" + " key = {}, and values = [{}, {}]. 2nd one is kept.", key, overridenValue, value); } } tags.put(key, value); }); return new AtlasMetaData(size, true, codeVersion, dataVersion, countries.isEmpty() ? null : countries.join(","), shardName, tags); } private RTree newPackedAtlasSpatialIndex() { return new RTree<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiAtlasLoaderCommand.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.statistic.storeless.CounterWithStatistic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class MultiAtlasLoaderCommand extends Command { private static final Logger logger = LoggerFactory.getLogger(MultiAtlasLoaderCommand.class); private static final Switch FOLDER = new Switch<>("folder", "Folder containing the atlas files", path -> new AtlasResourceLoader().load(new File(path)), Optionality.REQUIRED); private static final Switch OUTPUT = new Switch<>("output", "output atlas file", StringConverter.IDENTITY, Optionality.REQUIRED); public static void main(final String[] args) { new MultiAtlasLoaderCommand().run(args); } @Override protected int onRun(final CommandMap command) { final CounterWithStatistic statistics = new CounterWithStatistic(logger); final Atlas multi = (Atlas) command.get(FOLDER); final File output = new File((String) command.get(OUTPUT)); statistics.summary(); final PackedAtlas packed = PackedAtlas.cloneFrom(multi); packed.save(output); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(FOLDER, OUTPUT); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiAtlasOverlappingNodesFixer.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.exception.AtlasIntegrityException; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.maps.MultiMapWithSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Merge nodes that exactly overlap. * * @author matthieun * @author samg */ public class MultiAtlasOverlappingNodesFixer implements Serializable { private static final long serialVersionUID = -8926124646524609913L; private static final Logger logger = LoggerFactory .getLogger(MultiAtlasOverlappingNodesFixer.class); private final MultiAtlas parent; private final boolean fixNodesOnOppositeAntiMeridians; // The overlapping nodes... Those maps should be tiny private final Map overlappingNodeIdentifierToMainNodeIdentifier = new HashMap<>(); private final MultiMapWithSet mainNodeIdentifierToOverlappingNodeIdentifier = new MultiMapWithSet<>(); protected MultiAtlasOverlappingNodesFixer(final MultiAtlas parent, final boolean fixNodesOnOppositeAntiMeridians) { this.parent = parent; this.fixNodesOnOppositeAntiMeridians = fixNodesOnOppositeAntiMeridians; } /** * This is to build maps of nodes that are at the same location, and to pick the main node based * on point identifier */ protected void aggregateSameLocationNodes() { this.parent.getNodeIdentifierToAtlasIndices().forEach(identifier -> { // if this identifier is in the map, then skip. otherwise, poll all // overlapping nodes here, pick a main node based on identifier, // and update the map if (!this.overlappingNodeIdentifierToMainNodeIdentifier.containsKey(identifier)) { final Node current = this.parent.node(identifier); final SortedSet overlapping = new TreeSet<>((final Node a, final Node b) -> { if (a.getIdentifier() < b.getIdentifier()) { return -1; } else if (a.getIdentifier() > b.getIdentifier()) { return 1; } return 0; }); overlapping.addAll(nodesOverlapping(current)); if (overlapping.size() > 1) { final Node main = overlapping.first(); final long mainIdentifier = main.getIdentifier(); overlapping.remove(main); overlapping.forEach(node -> { final long overlappingIdentifier = node.getIdentifier(); this.overlappingNodeIdentifierToMainNodeIdentifier .put(overlappingIdentifier, mainIdentifier); this.mainNodeIdentifierToOverlappingNodeIdentifier.add(mainIdentifier, overlappingIdentifier); }); warnIfNodesHaveDifferentTags(main, overlapping); } } }); } /** * In case there is a main node overlapping this node, get the main node. * * @param identifier * The node identifier to query * @return The identifier of the main node that has the exact same location */ protected Optional mainNode(final Long identifier) { return Optional .ofNullable(this.overlappingNodeIdentifierToMainNodeIdentifier.get(identifier)); } @Deprecated protected Optional masterNode(final Long identifier) { return mainNode(identifier); } /** * In case this node is a main, get all the overlapping nodes. * * @param identifier * The node identifier to query * @return The identifiers of the overlapping nodes that has the exact same location */ protected Set overlappingNodes(final Long identifier) { if (this.mainNodeIdentifierToOverlappingNodeIdentifier.containsKey(identifier)) { return this.mainNodeIdentifierToOverlappingNodeIdentifier.get(identifier); } else { return new HashSet<>(); } } /** * @param main * The node to check for * @return all the nodes that have the same location, including itself */ private Set nodesOverlapping(final Node main) { final List bounds = new ArrayList<>(); // Make sure that the AntiMeridian case is taken care of final Location mainLocation = main.getLocation(); // This will be true for both the minimum antimeridian and the maximum. if (this.fixNodesOnOppositeAntiMeridians && (Longitude.ANTIMERIDIAN_WEST.equals(mainLocation.getLongitude()) || Longitude.ANTIMERIDIAN_EAST.equals(mainLocation.getLongitude()))) { final Location antimeridianMinimum = new Location(mainLocation.getLatitude(), Longitude.ANTIMERIDIAN_WEST); final Location antimeridianMaximum = new Location(mainLocation.getLatitude(), Longitude.ANTIMERIDIAN_EAST); bounds.add(Rectangle.forCorners(antimeridianMinimum, antimeridianMinimum)); bounds.add(Rectangle.forCorners(antimeridianMaximum, antimeridianMaximum)); } else { bounds.add(main.bounds()); } final Set others = new HashSet<>(); for (final Rectangle bound : bounds) { others.addAll(Iterables.asSet(this.parent.nodesWithin(bound))); } if (others.isEmpty()) { throw new AtlasIntegrityException("A node has to overlap itself at least! {}", main); } return others; } /** * Print a warning for nodes that have the same location but different tags. * * @param main * The main node * @param overlapping * The overlapping nodes */ private void warnIfNodesHaveDifferentTags(final Node main, final Set overlapping) { final Map tags = main.getTags(); for (final Node node : overlapping) { if (!tags.equals(node.getTags())) { logger.warn("Nodes overlap but have different tags: {} and {}", main.getIdentifier(), node.getIdentifier()); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.common.collect.Sets; /** * {@link Edge} made from a {@link MultiAtlas}. * * @author matthieun */ public class MultiEdge extends Edge { private static final long serialVersionUID = -3986525201031430336L; // Not index! private final long identifier; private SubEdgeList subEdgeList; protected MultiEdge(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public PolyLine asPolyLine() { return this.getRepresentativeSubEdge().asPolyLine(); } @Override public Node end() { return new MultiNode(multiAtlas(), getMainNodeIdentifier(this.getRepresentativeSubEdge().end().getIdentifier())); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { return this.getRepresentativeSubEdge().getTags(); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Edge subEdge : getSubEdges().getSubEdges()) { final Set currentSubEdgeParentRelations = multiAtlas() .multifyRelations(subEdge); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubEdgeParentRelations); } return unionOfAllParentRelations; } @Override public Node start() { return new MultiNode(multiAtlas(), getMainNodeIdentifier(this.getRepresentativeSubEdge().start().getIdentifier())); } /** * In case there is another node that overlaps this one, and the other one is the main, get the * other one * * @param identifier * The node identifier * @return The main node identifier if any, or identity */ private Long getMainNodeIdentifier(final long identifier) { final Optional mainNodeIdentifier = multiAtlas().mainNode(identifier); if (mainNodeIdentifier.isPresent()) { return mainNodeIdentifier.get(); } else { return identifier; } } private Edge getRepresentativeSubEdge() { return getSubEdges().getSubEdges().get(0); } private SubEdgeList getSubEdges() { if (this.subEdgeList == null) { this.subEdgeList = this.multiAtlas().subEdge(this.identifier); } return this.subEdgeList; } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.common.collect.Sets; /** * {@link Area} made from a {@link MultiAtlas}. * * @author matthieun */ public class MultiLine extends Line { private static final long serialVersionUID = 4833193008195471987L; // Not index! private final long identifier; private SubLineList subLineList; protected MultiLine(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public PolyLine asPolyLine() { return getRepresentativeSubLine().asPolyLine(); } @Override public long getIdentifier() { return this.identifier; } public SubLineList getSubLines() { if (this.subLineList == null) { this.subLineList = multiAtlas().subLines(this.identifier); } return this.subLineList; } @Override public Map getTags() { return this.getRepresentativeSubLine().getTags(); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Line subLine : getSubLines().getSubLines()) { final Set currentSubLineParentRelations = multiAtlas() .multifyRelations(subLine); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubLineParentRelations); } return unionOfAllParentRelations; } private Line getRepresentativeSubLine() { return getSubLines().getSubLines().get(0); } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Function; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; /** * {@link Node} made from a {@link MultiAtlas} * * @author matthieun */ public class MultiNode extends Node { private static final long serialVersionUID = 4280290265432052817L; private static final Logger logger = LoggerFactory.getLogger(MultiNode.class); private final long identifier; private SubNodeList subNodes; MultiNode(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return getRepresentativeSubNode().getLocation(); } @Override public Map getTags() { return getRepresentativeSubNode().getTags(); } @Override public SortedSet inEdges() { return attachedEdgesFromOverlappingNodes(Node::inEdges); } @Override public SortedSet outEdges() { return attachedEdgesFromOverlappingNodes(Node::outEdges); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Node subNode : getSubNodes().getSubNodes()) { final Set currentSubNodeParentRelations = multiAtlas() .multifyRelations(subNode); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubNodeParentRelations); } return unionOfAllParentRelations; } /** * Get all the attached edges from one single node identifier, directly coming from the * sub-Atlases... * * @param getConnectedEdges * The function that decides what side the edges are to be taken (in or out) * @return All the attached edges from one single node identifier */ private SortedSet attachedEdges(final Function> getConnectedEdges) { final Set subResult = new HashSet<>(); if (getSubNodes().size() == 1) { // The node is in one single sub-atlas only, hence not fixed. getConnectedEdges.apply(getRepresentativeSubNode()) .forEach(edge -> subResult.add(edge.getIdentifier())); } else { // The node is in many sub-atlases. Collect all the edges from the different Atlases. final SubNodeList subNodes = getSubNodes(); for (final Node node : subNodes.getSubNodes()) { for (final Edge connectedEdge : getConnectedEdges.apply(node)) { subResult.add(connectedEdge.getIdentifier()); } } } // Return MultiEdges so they still have a reference to the MultiAtlas final SortedSet result = new TreeSet<>(); for (final Long subEdgeIdentifier : subResult) { final Edge multiEdge = multiAtlas().edge(subEdgeIdentifier); if (multiEdge == null) { // This can happen sometimes when a node has another overlapping node in a sub-atlas // which gets all the connected edges, and that edge has been corrected for way // sectioning issues at the border, but the edge has been marked as "removed" from // the other node and not this one. Log it, and do not include the null. final List missingEdgeAtlasNames = new ArrayList<>(); final List atlases = ((MultiAtlas) getAtlas()).getAtlases(); for (int index = 0; index < atlases.size(); index++) { final Atlas subAtlas = atlases.get(index); final Edge subEdge = subAtlas.edge(subEdgeIdentifier); if (subEdge != null) { missingEdgeAtlasNames.add(subAtlas.getName()); } } logger.warn("Some edge got lost in translation, and is not in the MultiAtlas. " + "The node below probably has another node at the exact same location!\n\t" + "Node: {}\n\t" + "Edge connected: {}\n\t" + "From SubAtlas: {}", this.identifier, subEdgeIdentifier, missingEdgeAtlasNames); } else { result.add(multiEdge); } } return result; } /** * Get all the attached edges from one node, including those coming from overlapping nodes. * * @param getConnectedEdges * The function that decides what side the edges are to be taken (in or out) * @return All the attached edges from one node, including those coming from overlapping edges */ private SortedSet attachedEdgesFromOverlappingNodes( final Function> getConnectedEdges) { final Set alternateNodes = multiAtlas().overlappingNodes(getIdentifier()); final Optional mainNode = multiAtlas().mainNode(this.identifier); if (!alternateNodes.isEmpty()) { // This Multi-Node is an overlapping main node. Return all the in/out edges of this // node, plus those of the other alternate nodes. final SortedSet result = attachedEdges(getConnectedEdges); alternateNodes.forEach(alternateIdentifier -> result .addAll(((MultiNode) multiAtlas().node(alternateIdentifier)) .attachedEdges(getConnectedEdges))); return result; } else if (mainNode.isPresent()) { // This Multi-Node is an overlapping alternate node. Return no edges. All its edges will // be directed to the main node. Alternate nodes are then rendered useless (even though // they are still present) to mimic the behavior of the PackedAtlas. return new TreeSet<>(); } else { // This Multi-Node is not an overlapping node. Return the standard set of in/out edges. return attachedEdges(getConnectedEdges); } } private Node getRepresentativeSubNode() { return getSubNodes().getSubNodes().get(0); } private SubNodeList getSubNodes() { if (this.subNodes == null) { this.subNodes = multiAtlas().subNodes(this.identifier); } return this.subNodes; } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiPoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import com.google.common.collect.Sets; /** * {@link Point} made from a {@link MultiAtlas}. * * @author matthieun */ public class MultiPoint extends Point { private static final long serialVersionUID = 209103872813085178L; // Not index! private final long identifier; private SubPointList subPoints; protected MultiPoint(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public long getIdentifier() { return this.identifier; } @Override public Location getLocation() { return getRepresentativeSubPoint().getLocation(); } public SubPointList getSubPoints() { if (this.subPoints == null) { this.subPoints = multiAtlas().subPoints(this.identifier); } return this.subPoints; } @Override public Map getTags() { return this.getRepresentativeSubPoint().getTags(); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Point subPoint : getSubPoints().getSubPoints()) { final Set currentSubPointParentRelations = multiAtlas() .multifyRelations(subPoint); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubPointParentRelations); } return unionOfAllParentRelations; } private Point getRepresentativeSubPoint() { return getSubPoints().getSubPoints().get(0); } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/MultiRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import com.google.common.collect.Sets; /** * {@link Relation} made from a {@link MultiAtlas}. * * @author matthieun */ public class MultiRelation extends Relation { private static final long serialVersionUID = 2377231271257992525L; // Not index! private final long identifier; private SubRelationList subRelations; protected MultiRelation(final MultiAtlas atlas, final long identifier) { super(atlas); this.identifier = identifier; } @Override public RelationMemberList allKnownOsmMembers() { final List members = new ArrayList<>(); for (final Relation candidate : multiAtlas() .relationAllRelationsWithSameOsmIdentifier(this.identifier)) { candidate.members().forEach(relationMember -> members.add(relationMember)); } return new RelationMemberList(members); } @Override public List allRelationsWithSameOsmIdentifier() { return multiAtlas().relationAllRelationsWithSameOsmIdentifier(this.identifier); } @Override public Optional asMultiPolygon() { return getSingleSubRelation().asMultiPolygon(); } @Override public long getIdentifier() { return this.identifier; } @Override public Map getTags() { // They all should have the same tags return getSingleSubRelation().getTags(); } @Override public RelationMemberList members() { // Use a TreeSet to make sure all the members are always in a deterministic order. // RelationMember(s) are always ordered by member identifier. final Set members = new TreeSet<>(); for (final Relation subRelation : getSubRelations().getSubRelations()) { final RelationMemberList subMembers = subRelation.members(); for (final RelationMember subMember : subMembers) { final AtlasEntity nonMulti = subMember.getEntity(); final long nonMultiIdentifier = nonMulti.getIdentifier(); AtlasEntity multiEntity = null; if (nonMulti instanceof Node) { multiEntity = multiAtlas().node(nonMultiIdentifier); } else if (nonMulti instanceof Edge) { multiEntity = multiAtlas().edge(nonMultiIdentifier); } else if (nonMulti instanceof Area) { multiEntity = multiAtlas().area(nonMultiIdentifier); } else if (nonMulti instanceof Line) { multiEntity = multiAtlas().line(nonMultiIdentifier); } else if (nonMulti instanceof Point) { multiEntity = multiAtlas().point(nonMultiIdentifier); } else if (nonMulti instanceof Relation) { multiEntity = multiAtlas().relation(nonMultiIdentifier); } else { throw new CoreException("Could not find the proper type for {}", nonMulti); } if (multiEntity != null) { members.add(new RelationMember(subMember.getRole(), multiEntity, subMember.getRelationIdentifier())); } } } if (members.isEmpty()) { throw new CoreException( "This should not happen: MultiRelation {} has no members. Its sub relations are {}.", getIdentifier(), getSubRelations()); } return new RelationMemberList(members); } @Override public Long osmRelationIdentifier() { return getSingleSubRelation().osmRelationIdentifier(); } @Override public Set relations() { Set unionOfAllParentRelations = new HashSet<>(); for (final Relation subRelations : getSubRelations().getSubRelations()) { final Set currentSubRelationParentRelations = multiAtlas() .multifyRelations(subRelations); unionOfAllParentRelations = Sets.union(unionOfAllParentRelations, currentSubRelationParentRelations); } return unionOfAllParentRelations; } private Relation getSingleSubRelation() { return getSubRelations().getSubRelations().get(0); } private SubRelationList getSubRelations() { if (this.subRelations == null) { this.subRelations = this.multiAtlas().subRelations(this.identifier); } return this.subRelations; } private MultiAtlas multiAtlas() { return (MultiAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/README.md ================================================ # `MultiAtlas` This is a stitched `Atlas`. It is made of multiple sub-`Atlas` (which themselves can be `PackedAtlas` or `MultiAtlas`) and takes care of conflating and stitching at the intersection points. ## Flyweight `MultiAtlas` When creating a `MultiAtlas` from multiple other `Atlas` objects, no data is copied. The `MultiAtlas` builds array references to remember for example where each available `Edge` is relative to the sub `Atlas`, but does not copy the `Edge` data. When an `Edge` is requested, it creates a `MultiEdge` that contains only its identifier and the `MultiAtlas` it belongs to. When a user asks for the `Node`s connected to that `Edge` for example, the `MultiEdge` relays the query to its `MultiAtlas` that relays the query to the appropriate sub-`Atlas` which returns the result to be relayed back to the user. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubAreaList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Used by the {@link MultiArea} to hold multiple versions of identical {@link Area}s. This is in * case one of the {@link Area}s has a parent {@link Relation} that was not contained in one of the * sub-{@link Atlas}es of the containing {@link MultiAtlas}. * * @author lcram */ public class SubAreaList implements Iterable, Serializable { private static final long serialVersionUID = -1413359659676228024L; private final List subAreas; SubAreaList(final List subAreas) { if (subAreas == null) { throw new CoreException("Cannot have a null list of sub areas."); } this.subAreas = subAreas; } @Override public Iterator iterator() { return this.subAreas.iterator(); } public int size() { return this.subAreas.size(); } List getSubAreas() { return this.subAreas; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubEdgeList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Used by the {@link MultiEdge} to hold multiple versions of identical {@link Edge}s. This is in * case one of the {@link Edge}s has a parent {@link Relation} that was not contained in one of the * sub-{@link Atlas}es of the containing {@link MultiAtlas}. * * @author lcram * @author matthieun */ public class SubEdgeList implements Iterable, Serializable { private static final long serialVersionUID = 4338093791628259315L; private final List subEdges; SubEdgeList(final List subEdges) { if (subEdges == null) { throw new CoreException("Cannot have a null list of sub edges."); } this.subEdges = subEdges; } public List getSubEdges() { return this.subEdges; } @Override public Iterator iterator() { return this.subEdges.iterator(); } public int size() { return this.subEdges.size(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubLineList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Used by the {@link MultiLine} to hold multiple versions of identical {@link Line}s. This is in * case one of the {@link Line}s has a parent {@link Relation} that was not contained in one of the * sub-{@link Atlas}es of the containing {@link MultiAtlas}. * * @author lcram */ public class SubLineList implements Iterable, Serializable { private static final long serialVersionUID = -1413359659676228024L; private final List subLines; SubLineList(final List subLines) { if (subLines == null) { throw new CoreException("Cannot have a null list of sub lines."); } this.subLines = subLines; } @Override public Iterator iterator() { return this.subLines.iterator(); } public int size() { return this.subLines.size(); } List getSubLines() { return this.subLines; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubNodeList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Used by the {@link MultiNode} to hold multiple versions of identical {@link Node}s. This is in * case one of the {@link Node}s has a parent {@link Relation} that was not contained in one of the * sub-{@link Atlas}es of the containing {@link MultiAtlas}. * * @author matthieun */ public class SubNodeList implements Iterable, Serializable { private static final long serialVersionUID = -1413359659676228024L; private final List subNodes; SubNodeList(final List subNodes) { if (subNodes == null) { throw new CoreException("Cannot have a null list of sub nodes."); } this.subNodes = subNodes; } @Override public Iterator iterator() { return this.subNodes.iterator(); } public int size() { return this.subNodes.size(); } List getSubNodes() { return this.subNodes; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubPointList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * Used by the {@link MultiPoint} to hold multiple versions of identical {@link Point}s. This is in * case one of the {@link Point}s has a parent {@link Relation} that was not contained in one of the * sub-{@link Atlas}es of the containing {@link MultiAtlas}. * * @author lcram */ public class SubPointList implements Iterable, Serializable { private static final long serialVersionUID = -1413359659676228024L; private final List subPoints; SubPointList(final List subPoints) { if (subPoints == null) { throw new CoreException("Cannot have a null list of sub points."); } this.subPoints = subPoints; } @Override public Iterator iterator() { return this.subPoints.iterator(); } public int size() { return this.subPoints.size(); } List getSubPoints() { return this.subPoints; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/multi/SubRelationList.java ================================================ package org.openstreetmap.atlas.geography.atlas.multi; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.collections.StringList; /** * @author matthieun */ public class SubRelationList implements Iterable, Serializable { private static final long serialVersionUID = 8408824588171850810L; private final List subRelations; SubRelationList(final List subRelations) { if (subRelations == null) { throw new CoreException("Cannot have a null list of sub relations."); } this.subRelations = subRelations; } @Override public Iterator iterator() { return this.subRelations.iterator(); } public int size() { return this.subRelations.size(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); final StringList relations = new StringList(); this.subRelations.forEach(relation -> relations.add(relation.toString())); builder.append("[SubRelations: "); builder.append(relations.join(System.lineSeparator())); builder.append("]"); return builder.toString(); } List getSubRelations() { return this.subRelations; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedArea.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Area} from a {@link PackedAtlas} * * @author matthieun */ public class PackedArea extends Area { private static final long serialVersionUID = 4578525310383858728L; private final long index; protected PackedArea(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public Polygon asPolygon() { return packedAtlas().areaPolygon(this.index); } @Override public long getIdentifier() { return packedAtlas().areaIdentifier(this.index); } @Override public Map getTags() { return packedAtlas().areaTags(this.index); } @Override public Set relations() { return packedAtlas().areaRelations(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlas.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Consumer; import java.util.function.Supplier; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.AbstractAtlas; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.exception.AtlasIntegrityException; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; import org.openstreetmap.atlas.streaming.compression.Compressor; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.arrays.ByteArrayOfArrays; import org.openstreetmap.atlas.utilities.arrays.IntegerArrayOfArrays; import org.openstreetmap.atlas.utilities.arrays.LongArray; import org.openstreetmap.atlas.utilities.arrays.LongArrayOfArrays; import org.openstreetmap.atlas.utilities.arrays.PolyLineArray; import org.openstreetmap.atlas.utilities.arrays.PolygonArray; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.compression.IntegerDictionary; import org.openstreetmap.atlas.utilities.maps.LongToLongMap; import org.openstreetmap.atlas.utilities.maps.LongToLongMultiMap; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Atlas that packs data in large arrays * * @author matthieun */ public final class PackedAtlas extends AbstractAtlas { /** * Serialization format settings for an {@link Atlas}. While the serialization interface for * saving is well-defined by the {@link Atlas}, the actual serialization mechanics - as well as * the interface for loading - are left up to the discretion of the implementing {@link Atlas} * subclass. * * @author lcram */ public enum AtlasSerializationFormat { PROTOBUF, JAVA } // Keep track of the field names for reflection code in the Serializer. protected static final String FIELD_PREFIX = "FIELD_"; protected static final String FIELD_BOUNDS = "bounds"; protected static final String FIELD_LOGGER = "logger"; protected static final String FIELD_SERIAL_VERSION_UID = "serialVersionUID"; protected static final String FIELD_SERIALIZER = "serializer"; protected static final String FIELD_SAVE_SERIALIZATION_FORMAT = "saveSerializationFormat"; protected static final String FIELD_LOAD_SERIALIZATION_FORMAT = "loadSerializationFormat"; protected static final String FIELD_CONTAINS_ENHANCED_RELATION_GEOMETRY = "containsEnhancedRelationGeometry"; protected static final String FIELD_META_DATA = "metaData"; protected static final String FIELD_DICTIONARY = "dictionary"; private transient Object fieldDictionaryLock = new Object(); protected static final String FIELD_EDGE_IDENTIFIERS = "edgeIdentifiers"; private transient Object fieldEdgeIdentifiersLock = new Object(); protected static final String FIELD_NODE_IDENTIFIERS = "nodeIdentifiers"; private transient Object fieldNodeIdentifiersLock = new Object(); protected static final String FIELD_AREA_IDENTIFIERS = "areaIdentifiers"; private transient Object fieldAreaIdentifiersLock = new Object(); protected static final String FIELD_LINE_IDENTIFIERS = "lineIdentifiers"; private transient Object fieldLineIdentifiersLock = new Object(); protected static final String FIELD_POINT_IDENTIFIERS = "pointIdentifiers"; private transient Object fieldPointIdentifiersLock = new Object(); protected static final String FIELD_RELATION_IDENTIFIERS = "relationIdentifiers"; private transient Object fieldRelationIdentifiersLock = new Object(); protected static final String FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX = "edgeIdentifierToEdgeArrayIndex"; private transient Object fieldEdgeIdentifierToEdgeArrayIndexLock = new Object(); protected static final String FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX = "nodeIdentifierToNodeArrayIndex"; private transient Object fieldNodeIdentifierToNodeArrayIndexLock = new Object(); protected static final String FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX = "areaIdentifierToAreaArrayIndex"; private transient Object fieldAreaIdentifierToAreaArrayIndexLock = new Object(); protected static final String FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX = "lineIdentifierToLineArrayIndex"; private transient Object fieldLineIdentifierToLineArrayIndexLock = new Object(); protected static final String FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX = "pointIdentifierToPointArrayIndex"; private transient Object fieldPointIdentifierToPointArrayIndexLock = new Object(); protected static final String FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX = "relationIdentifierToRelationArrayIndex"; private transient Object fieldRelationIdentifierToRelationArrayIndexLock = new Object(); protected static final String FIELD_NODE_LOCATIONS = "nodeLocations"; private transient Object fieldNodeLocationsLock = new Object(); protected static final String FIELD_NODE_IN_EDGES_INDICES = "nodeInEdgesIndices"; private transient Object fieldNodeInEdgesIndicesLock = new Object(); protected static final String FIELD_NODE_OUT_EDGES_INDICES = "nodeOutEdgesIndices"; private transient Object fieldNodeOutEdgesIndicesLock = new Object(); protected static final String FIELD_NODE_TAGS = "nodeTags"; private transient Object fieldNodeTagsLock = new Object(); protected static final String FIELD_NODE_INDEX_TO_RELATION_INDICES = "nodeIndexToRelationIndices"; private transient Object fieldNodeIndexToRelationIndicesLock = new Object(); protected static final String FIELD_EDGE_START_NODE_INDEX = "edgeStartNodeIndex"; private transient Object fieldEdgeStartNodeIndexLock = new Object(); protected static final String FIELD_EDGE_END_NODE_INDEX = "edgeEndNodeIndex"; private transient Object fieldEdgeEndNodeIndexLock = new Object(); protected static final String FIELD_EDGE_POLY_LINES = "edgePolyLines"; private transient Object fieldEdgePolyLinesLock = new Object(); protected static final String FIELD_EDGE_TAGS = "edgeTags"; private transient Object fieldEdgeTagsLock = new Object(); protected static final String FIELD_EDGE_INDEX_TO_RELATION_INDICES = "edgeIndexToRelationIndices"; private transient Object fieldEdgeIndexToRelationIndicesLock = new Object(); protected static final String FIELD_AREA_POLYGONS = "areaPolygons"; private transient Object fieldAreaPolygonsLock = new Object(); protected static final String FIELD_AREA_TAGS = "areaTags"; private transient Object fieldAreaTagsLock = new Object(); protected static final String FIELD_AREA_INDEX_TO_RELATION_INDICES = "areaIndexToRelationIndices"; private transient Object fieldAreaIndexToRelationIndicesLock = new Object(); protected static final String FIELD_LINE_POLYLINES = "linePolyLines"; private transient Object fieldLinePolyLinesLock = new Object(); protected static final String FIELD_LINE_TAGS = "lineTags"; private transient Object fieldLineTagsLock = new Object(); protected static final String FIELD_LINE_INDEX_TO_RELATION_INDICES = "lineIndexToRelationIndices"; private transient Object fieldLindIndexToRelationIndicesLock = new Object(); protected static final String FIELD_POINT_LOCATIONS = "pointLocations"; private transient Object fieldPointLocationsLock = new Object(); protected static final String FIELD_POINT_TAGS = "pointTags"; private transient Object fieldPointTagsLock = new Object(); protected static final String FIELD_POINT_INDEX_TO_RELATION_INDICES = "pointIndexToRelationIndices"; private transient Object fieldPointIndexToRelationIndicesLock = new Object(); protected static final String FIELD_RELATION_MEMBERS_INDICES = "relationMemberIndices"; private transient Object fieldRelationMembersIndicesLock = new Object(); protected static final String FIELD_RELATION_MEMBER_TYPES = "relationMemberTypes"; private transient Object fieldRelationMemberTypesLock = new Object(); protected static final String FIELD_RELATION_MEMBER_ROLES = "relationMemberRoles"; private transient Object fieldRelationMemberRolesLock = new Object(); protected static final String FIELD_RELATION_TAGS = "relationTags"; private transient Object fieldRelationTagsLock = new Object(); protected static final String FIELD_RELATION_INDEX_TO_RELATION_INDICES = "relationIndexToRelationIndices"; private transient Object fieldRelationIndexToRelationIndicesLock = new Object(); protected static final String FIELD_RELATION_OSM_IDENTIFIER_TO_RELATION_IDENTIFIERS = "relationOsmIdentifierToRelationIdentifiers"; private transient Object fieldRelationOsmIdentifierToRelationIdentifiersLock = new Object(); protected static final String FIELD_RELATION_OSM_IDENTIFIERS = "relationOsmIdentifiers"; private transient Object fieldRelationOsmIdentifiersLock = new Object(); protected static final String FIELD_RELATION_GEOMETRIES = "relationGeometries"; private transient Object fieldRelationGeometriesLock = new Object(); protected static final String FIELD_BUILT_RELATION_GEOMETRIES = "builtRelationGeometries"; private static final long serialVersionUID = -7582554057580336684L; private static final Logger logger = LoggerFactory.getLogger(PackedAtlas.class); // Serializer. private transient PackedAtlasSerializer serializer; // Serialization formats for saving/loading this PackedAtlas private AtlasSerializationFormat saveSerializationFormat = AtlasSerializationFormat.PROTOBUF; private AtlasSerializationFormat loadSerializationFormat = AtlasSerializationFormat.PROTOBUF; private boolean containsEnhancedRelationGeometry = false; // Meta-Data private AtlasMetaData metaData = new AtlasMetaData(); // Dictionary private final IntegerDictionary dictionary; // The OSM (and way-sectioned) edge and node indices private final LongArray edgeIdentifiers; private final LongArray nodeIdentifiers; private final LongArray areaIdentifiers; private final LongArray lineIdentifiers; private final LongArray pointIdentifiers; private final LongArray relationIdentifiers; // The maps from edge index to index in the arrays above, and in the attributes private final LongToLongMap edgeIdentifierToEdgeArrayIndex; private final LongToLongMap nodeIdentifierToNodeArrayIndex; private final LongToLongMap areaIdentifierToAreaArrayIndex; private final LongToLongMap lineIdentifierToLineArrayIndex; private final LongToLongMap pointIdentifierToPointArrayIndex; private final LongToLongMap relationIdentifierToRelationArrayIndex; // Node attributes private final LongArray nodeLocations; private final LongArrayOfArrays nodeInEdgesIndices; private final LongArrayOfArrays nodeOutEdgesIndices; private final PackedTagStore nodeTags; private final LongToLongMultiMap nodeIndexToRelationIndices; // Edge attributes private final LongArray edgeStartNodeIndex; private final LongArray edgeEndNodeIndex; private final PolyLineArray edgePolyLines; private final PackedTagStore edgeTags; private final LongToLongMultiMap edgeIndexToRelationIndices; // Areas attributes private final PolygonArray areaPolygons; private final PackedTagStore areaTags; private final LongToLongMultiMap areaIndexToRelationIndices; // Line attributes private final PolyLineArray linePolyLines; private final PackedTagStore lineTags; private final LongToLongMultiMap lineIndexToRelationIndices; // Point attributes private final LongArray pointLocations; private final PackedTagStore pointTags; private final LongToLongMultiMap pointIndexToRelationIndices; // Relation attributes private final LongArrayOfArrays relationMemberIndices; private final ByteArrayOfArrays relationMemberTypes; private final IntegerArrayOfArrays relationMemberRoles; private final PackedTagStore relationTags; private final LongToLongMultiMap relationIndexToRelationIndices; private final LongToLongMultiMap relationOsmIdentifierToRelationIdentifiers; private final LongArray relationOsmIdentifiers; private ByteArrayOfArrays relationGeometries; private transient Map builtRelationGeometries = new HashMap<>(); // Bounds of the Atlas private Rectangle bounds; /** * Clone an {@link Atlas} into a {@link PackedAtlas} * * @param other * The {@link Atlas} to clone * @return The cloned {@link PackedAtlas} */ public static PackedAtlas cloneFrom(final Atlas other) { return new PackedAtlasCloner().cloneFrom(other); } /** * Load a {@link PackedAtlas} from a zip entry resource * * @param resource * The {@link Resource} to read from * @return The deserialized {@link PackedAtlas} */ public static PackedAtlas load(final Resource resource) { final PackedAtlas result = PackedAtlasSerializer.load(resource); result.setName(resource.getName()); return result; } /** * This constructor is used only by the serializer. */ protected PackedAtlas() { this.metaData = null; this.dictionary = null; this.edgeIdentifiers = null; this.nodeIdentifiers = null; this.areaIdentifiers = null; this.lineIdentifiers = null; this.pointIdentifiers = null; this.relationIdentifiers = null; this.edgeIdentifierToEdgeArrayIndex = null; this.nodeIdentifierToNodeArrayIndex = null; this.areaIdentifierToAreaArrayIndex = null; this.lineIdentifierToLineArrayIndex = null; this.pointIdentifierToPointArrayIndex = null; this.relationIdentifierToRelationArrayIndex = null; this.nodeLocations = null; this.nodeInEdgesIndices = null; this.nodeOutEdgesIndices = null; this.nodeTags = null; this.nodeIndexToRelationIndices = null; this.edgeStartNodeIndex = null; this.edgeEndNodeIndex = null; this.edgePolyLines = null; this.edgeTags = null; this.edgeIndexToRelationIndices = null; this.areaPolygons = null; this.areaTags = null; this.areaIndexToRelationIndices = null; this.linePolyLines = null; this.lineTags = null; this.lineIndexToRelationIndices = null; this.pointLocations = null; this.pointTags = null; this.pointIndexToRelationIndices = null; this.relationMemberIndices = null; this.relationMemberTypes = null; this.relationMemberRoles = null; this.relationTags = null; this.relationIndexToRelationIndices = null; this.relationOsmIdentifierToRelationIdentifiers = null; this.relationOsmIdentifiers = null; this.relationGeometries = null; } /** * Construct an Atlas * * @param estimates * The size estimates */ protected PackedAtlas(final AtlasSize estimates) { this(estimates, false); } protected PackedAtlas(final AtlasSize estimates, final boolean containsEnhancedRelationGeometry) { final long edgeNumberEstimate = estimates.getEdgeNumber(); final long nodeNumberEstimate = estimates.getNodeNumber(); final long areaNumberEstimate = estimates.getAreaNumber(); final long lineNumberEstimate = estimates.getLineNumber(); final long pointNumberEstimate = estimates.getPointNumber(); final long relationNumberEstimate = estimates.getRelationNumber(); final int subArraySize = Integer.MAX_VALUE; final long maximumSize = Long.MAX_VALUE; final int edgeMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, edgeNumberEstimate % Integer.MAX_VALUE); final int nodeMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, nodeNumberEstimate % Integer.MAX_VALUE); final int areaMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, areaNumberEstimate % Integer.MAX_VALUE); final int lineMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, lineNumberEstimate % Integer.MAX_VALUE); final int pointMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, pointNumberEstimate % Integer.MAX_VALUE); final int relationMemoryBlockSize = (int) Math.max(DEFAULT_NUMBER_OF_ITEMS, relationNumberEstimate % Integer.MAX_VALUE); final int edgeHashSize = (int) Math .max(Math.min(edgeNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); final int nodeHashSize = (int) Math .max(Math.min(nodeNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); final int areaHashSize = (int) Math .max(Math.min(areaNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); final int lineHashSize = (int) Math .max(Math.min(lineNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); final int pointHashSize = (int) Math .max(Math.min(pointNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); final int relationHashSize = (int) Math .max(Math.min(relationNumberEstimate / HASH_MODULO_RATIO, Integer.MAX_VALUE), 1); this.dictionary = new IntegerDictionary<>(); this.edgeIdentifiers = new LongArray(maximumSize, edgeMemoryBlockSize, subArraySize); this.nodeIdentifiers = new LongArray(maximumSize, nodeMemoryBlockSize, subArraySize); this.areaIdentifiers = new LongArray(maximumSize, areaMemoryBlockSize, subArraySize); this.lineIdentifiers = new LongArray(maximumSize, lineMemoryBlockSize, subArraySize); this.pointIdentifiers = new LongArray(maximumSize, pointMemoryBlockSize, subArraySize); this.relationIdentifiers = new LongArray(maximumSize, relationMemoryBlockSize, subArraySize); this.edgeIdentifierToEdgeArrayIndex = new LongToLongMap( "PackedAtlas - edgeIdentifierToEdgeArrayIndex", maximumSize, edgeHashSize, edgeMemoryBlockSize, subArraySize, edgeMemoryBlockSize, subArraySize); this.nodeIdentifierToNodeArrayIndex = new LongToLongMap( "PackedAtlas - nodeIdentifierToNodeArrayIndex", maximumSize, nodeHashSize, nodeMemoryBlockSize, subArraySize, nodeMemoryBlockSize, subArraySize); this.areaIdentifierToAreaArrayIndex = new LongToLongMap( "PackedAtlas - areaIdentifierToAreaArrayIndex", maximumSize, areaHashSize, areaMemoryBlockSize, subArraySize, areaMemoryBlockSize, subArraySize); this.lineIdentifierToLineArrayIndex = new LongToLongMap( "PackedAtlas - lineIdentifierToLineArrayIndex", maximumSize, lineHashSize, lineMemoryBlockSize, subArraySize, lineMemoryBlockSize, subArraySize); this.pointIdentifierToPointArrayIndex = new LongToLongMap( "PackedAtlas - pointIdentifierToPointArrayIndex", maximumSize, pointHashSize, pointMemoryBlockSize, subArraySize, pointMemoryBlockSize, subArraySize); this.relationIdentifierToRelationArrayIndex = new LongToLongMap( "PackedAtlas - relationIdentifierToRelationArrayIndex", maximumSize, relationHashSize, relationMemoryBlockSize, subArraySize, relationMemoryBlockSize, subArraySize); this.nodeInEdgesIndices = new LongArrayOfArrays(subArraySize, nodeMemoryBlockSize, subArraySize); this.nodeOutEdgesIndices = new LongArrayOfArrays(subArraySize, nodeMemoryBlockSize, subArraySize); this.nodeLocations = new LongArray(maximumSize, nodeMemoryBlockSize, subArraySize); this.nodeTags = new PackedTagStore(maximumSize, nodeMemoryBlockSize, subArraySize, dictionary()); this.nodeIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - nodeIndexToRelationIndices", maximumSize, nodeHashSize, nodeMemoryBlockSize, subArraySize, nodeMemoryBlockSize, nodeHashSize); this.edgeStartNodeIndex = new LongArray(maximumSize, edgeMemoryBlockSize, subArraySize); this.edgeEndNodeIndex = new LongArray(maximumSize, edgeMemoryBlockSize, subArraySize); this.edgePolyLines = new PolyLineArray(maximumSize, edgeMemoryBlockSize, subArraySize); this.edgeTags = new PackedTagStore(maximumSize, edgeMemoryBlockSize, subArraySize, dictionary()); this.edgeIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - edgeIndexToRelationIndices", maximumSize, edgeHashSize, edgeMemoryBlockSize, subArraySize, edgeMemoryBlockSize, edgeHashSize); this.areaPolygons = new PolygonArray(maximumSize, areaMemoryBlockSize, subArraySize); this.areaTags = new PackedTagStore(maximumSize, areaMemoryBlockSize, subArraySize, dictionary()); this.areaIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - areaIndexToRelationIndices", maximumSize, areaHashSize, areaMemoryBlockSize, subArraySize, areaMemoryBlockSize, areaHashSize); this.linePolyLines = new PolyLineArray(maximumSize, lineMemoryBlockSize, subArraySize); this.lineTags = new PackedTagStore(maximumSize, lineMemoryBlockSize, subArraySize, dictionary()); this.lineIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - lineIndexToRelationIndices", maximumSize, lineHashSize, lineMemoryBlockSize, subArraySize, lineMemoryBlockSize, lineHashSize); this.pointLocations = new LongArray(maximumSize, pointMemoryBlockSize, subArraySize); this.pointTags = new PackedTagStore(maximumSize, pointMemoryBlockSize, subArraySize, dictionary()); this.pointIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - pointIndexToRelationIndices", maximumSize, pointHashSize, pointMemoryBlockSize, subArraySize, pointMemoryBlockSize, pointHashSize); this.relationMemberIndices = new LongArrayOfArrays(maximumSize, relationMemoryBlockSize, subArraySize); this.relationMemberTypes = new ByteArrayOfArrays(maximumSize, relationMemoryBlockSize, subArraySize); this.relationMemberRoles = new IntegerArrayOfArrays(maximumSize, relationMemoryBlockSize, subArraySize); this.relationTags = new PackedTagStore(maximumSize, relationMemoryBlockSize, subArraySize, dictionary()); this.relationIndexToRelationIndices = new LongToLongMultiMap( "PackedAtlas - relationIndexToRelationIndices", maximumSize, relationHashSize, relationMemoryBlockSize, subArraySize, relationMemoryBlockSize, relationHashSize); this.relationOsmIdentifierToRelationIdentifiers = new LongToLongMultiMap( "PackedAtlas - relationOsmIdentifierToRelationIdentifier", maximumSize, relationHashSize, relationMemoryBlockSize, subArraySize, relationMemoryBlockSize, subArraySize); this.relationOsmIdentifiers = new LongArray(maximumSize, relationMemoryBlockSize, subArraySize); this.edgeIdentifiers.setName("PackedAtlas - edgeIdentifiers"); this.edgeStartNodeIndex.setName("PackedAtlas - edgeStartNodeIndex"); this.edgeEndNodeIndex.setName("PackedAtlas - edgeEndNodeIndex"); this.edgePolyLines.setName("PackedAtlas - edgePolyLines"); this.nodeIdentifiers.setName("PackedAtlas - nodeIdentifiers"); this.nodeInEdgesIndices.setName("PackedAtlas - nodeInEdgesIndices"); this.nodeOutEdgesIndices.setName("PackedAtlas - nodeOutEdgesIndices"); this.nodeLocations.setName("PackedAtlas - nodeLocations"); this.areaIdentifiers.setName("PackedAtlas - areaIdentifiers"); this.areaPolygons.setName("PackedAtlas - areaPolygons"); this.lineIdentifiers.setName("PackedAtlas - lineIdentifiers"); this.linePolyLines.setName("PackedAtlas - linePolyLines"); this.pointIdentifiers.setName("PackedAtlas - pointIdentifiers"); this.pointLocations.setName("PackedAtlas - pointLocations"); this.relationIdentifiers.setName("PackedAtlas - relationIdentifiers"); this.relationMemberIndices.setName("PackedAtlas - relationMemberIndices"); this.relationMemberTypes.setName("PackedAtlas - relationMemberTypes"); this.relationMemberRoles.setName("PackedAtlas - relationMemberRoles"); this.relationOsmIdentifiers.setName("PackedAtlas - relationOsmIdentifiers"); if (containsEnhancedRelationGeometry) { this.containsEnhancedRelationGeometry = containsEnhancedRelationGeometry; this.relationGeometries = new ByteArrayOfArrays(maximumSize, relationMemoryBlockSize, subArraySize); this.relationGeometries.setName("PackedAtlas - relationGeometries"); } } @Override public Area area(final long identifier) { if (this.areaIdentifierToAreaArrayIndex().containsKey(identifier)) { return new PackedArea(this, this.areaIdentifierToAreaArrayIndex().get(identifier)); } return null; } @Override public Iterable areas() { return Iterables.indexBasedIterable(this.areaIdentifiers().size(), index -> new PackedArea(this, index)); } @Override public Rectangle bounds() { if (this.bounds == null) { final Iterable boundedEntities = Iterables.filter(this, entity -> entity.bounds() != null); this.bounds = Rectangle.forLocated(boundedEntities); } return this.bounds; } public boolean containsEnhancedRelationGeometry() { return this.containsEnhancedRelationGeometry; } @Override public Edge edge(final long identifier) { if (this.edgeIdentifierToEdgeArrayIndex().containsKey(identifier)) { return new PackedEdge(this, this.edgeIdentifierToEdgeArrayIndex().get(identifier)); } return null; } @Override public Iterable edges() { return Iterables.indexBasedIterable(this.edgeIdentifiers().size(), index -> new PackedEdge(this, index)); } /** * Get the serialization format used for saving this {@link PackedAtlas}. By default use Java * serialization. * * @return The serialization format setting */ public AtlasSerializationFormat getSaveSerializationFormat() { return this.saveSerializationFormat; } /** * Get the serialization format with which this atlas was loaded. * * @return the format */ public AtlasSerializationFormat getSerializationFormat() { return this.loadSerializationFormat; } @Override public Line line(final long identifier) { if (this.lineIdentifierToLineArrayIndex().containsKey(identifier)) { return new PackedLine(this, this.lineIdentifierToLineArrayIndex().get(identifier)); } return null; } @Override public Iterable lines() { return Iterables.indexBasedIterable(this.lineIdentifiers().size(), index -> new PackedLine(this, index)); } @Override public AtlasMetaData metaData() { if (this.metaData == null) { this.serializer.deserializeIfNeeded(FIELD_META_DATA); } return this.metaData; } @Override public Node node(final long identifier) { if (this.nodeIdentifierToNodeArrayIndex().containsKey(identifier)) { return new PackedNode(this, this.nodeIdentifierToNodeArrayIndex().get(identifier)); } return null; } @Override public Iterable nodes() { return Iterables.indexBasedIterable(this.nodeIdentifiers().size(), index -> new PackedNode(this, index)); } @Override public long numberOfAreas() { return this.areaIdentifiers().size(); } @Override public long numberOfEdges() { return this.edgeIdentifiers().size(); } @Override public long numberOfLines() { return this.lineIdentifiers().size(); } @Override public long numberOfNodes() { return this.nodeIdentifiers().size(); } @Override public long numberOfPoints() { return this.pointIdentifiers().size(); } @Override public long numberOfRelations() { return this.relationIdentifiers().size(); } @Override public Point point(final long identifier) { if (this.pointIdentifierToPointArrayIndex().containsKey(identifier)) { return new PackedPoint(this, this.pointIdentifierToPointArrayIndex().get(identifier)); } return null; } @Override public Iterable points() { return Iterables.indexBasedIterable(this.pointIdentifiers().size(), index -> new PackedPoint(this, index)); } @Override public Relation relation(final long identifier) { if (this.relationIdentifierToRelationArrayIndex().containsKey(identifier)) { return new PackedRelation(this, this.relationIdentifierToRelationArrayIndex().get(identifier)); } return null; } @Override public Iterable relations() { return Iterables.indexBasedIterable(this.relationIdentifiers().size(), index -> new PackedRelation(this, index)); } @Override public void save(final WritableResource writableResource) { new PackedAtlasSerializer(this, writableResource).save(); } /** * Set the serialization format for saving this {@link PackedAtlas}. * * @param format * The format to use */ public void setSaveSerializationFormat(final AtlasSerializationFormat format) { this.saveSerializationFormat = format; if (this.saveSerializationFormat != AtlasSerializationFormat.PROTOBUF) { this.containsEnhancedRelationGeometry = false; } } /** * Trim this Atlas' arrays with the proper size. WARNING! This could potentially temporarily * double the amount of memory used by each array. */ public void trim() { logger.info("Trimming Atlas {} to save on space.", this.getName()); final Time start = Time.now(); this.edgeIdentifiers.trim(); this.nodeIdentifiers.trim(); this.areaIdentifiers.trim(); this.lineIdentifiers.trim(); this.pointIdentifiers.trim(); this.relationIdentifiers.trim(); this.edgeIdentifierToEdgeArrayIndex.trim(); this.nodeIdentifierToNodeArrayIndex.trim(); this.areaIdentifierToAreaArrayIndex.trim(); this.lineIdentifierToLineArrayIndex.trim(); this.pointIdentifierToPointArrayIndex.trim(); this.relationIdentifierToRelationArrayIndex.trim(); this.nodeLocations.trim(); this.nodeInEdgesIndices.trim(); this.nodeOutEdgesIndices.trim(); this.nodeTags.trim(); this.nodeIndexToRelationIndices.trim(); this.edgeStartNodeIndex.trim(); this.edgeEndNodeIndex.trim(); this.edgePolyLines.trim(); this.edgeTags.trim(); this.edgeIndexToRelationIndices.trim(); this.areaPolygons.trim(); this.areaTags.trim(); this.areaIndexToRelationIndices.trim(); this.linePolyLines.trim(); this.lineTags.trim(); this.lineIndexToRelationIndices.trim(); this.pointLocations.trim(); this.pointTags.trim(); this.pointIndexToRelationIndices.trim(); this.relationMemberIndices.trim(); this.relationMemberTypes.trim(); this.relationMemberRoles.trim(); this.relationTags.trim(); this.relationIndexToRelationIndices.trim(); this.relationOsmIdentifierToRelationIdentifiers.trim(); this.relationOsmIdentifiers.trim(); if (this.containsEnhancedRelationGeometry) { this.relationGeometries.trim(); } logger.info("Trimmed Atlas {} in {}.", this.getName(), start.elapsedSince()); } protected void addArea(final long areaIdentifier, final Polygon polygon, final Map tags) { synchronized (this.areaIdentifiers) { if (this.areaIdentifierToAreaArrayIndex.containsKey(areaIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.AREA, areaIdentifier); } final long index = this.areaIdentifiers.size(); this.areaIdentifiers.add(areaIdentifier); this.areaIdentifierToAreaArrayIndex.put(areaIdentifier, index); this.areaPolygons.add(polygon); this.getAsNewAreaSpatialIndex().add(new PackedArea(this, index)); // Tags updatePackedTagStore(this.areaTags, index, tags); } } protected void addEdge(final long edgeIdentifier, final long startNodeIdentifier, final long endNodeIdentifier, final PolyLine polyline, final Map tags) { synchronized (this.edgeIdentifiers) { if (this.edgeIdentifierToEdgeArrayIndex.containsKey(edgeIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.EDGE, edgeIdentifier); } final long index = this.edgeIdentifiers.size(); this.edgeIdentifiers.add(edgeIdentifier); this.edgeIdentifierToEdgeArrayIndex.put(edgeIdentifier, index); this.edgePolyLines.add(polyline); // Start Node final long startNodeIndex = this.nodeIdentifierToNodeArrayIndex .get(startNodeIdentifier); this.edgeStartNodeIndex.add(startNodeIndex); // End Node final long endNodeIndex = this.nodeIdentifierToNodeArrayIndex.get(endNodeIdentifier); this.edgeEndNodeIndex.add(endNodeIndex); // Node In edges updateNodeEdgesReference(endNodeIndex, this.nodeInEdgesIndices, index); // Node Out edges updateNodeEdgesReference(startNodeIndex, this.nodeOutEdgesIndices, index); // Spatial Index this.getAsNewEdgeSpatialIndex().add(new PackedEdge(this, index)); // Tags updatePackedTagStore(this.edgeTags, index, tags); } } protected void addLine(final long lineIdentifier, final PolyLine polyline, final Map tags) { synchronized (this.lineIdentifiers) { if (this.lineIdentifierToLineArrayIndex.containsKey(lineIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.LINE, lineIdentifier); } final long index = this.lineIdentifiers.size(); this.lineIdentifiers.add(lineIdentifier); this.lineIdentifierToLineArrayIndex.put(lineIdentifier, index); this.linePolyLines.add(polyline); this.getAsNewLineSpatialIndex().add(new PackedLine(this, index)); // Tags updatePackedTagStore(this.lineTags, index, tags); } } protected void addNode(final long nodeIdentifier, final Location location, final Map tags) { synchronized (this.nodeIdentifiers) { if (this.nodeIdentifierToNodeArrayIndex.containsKey(nodeIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.NODE, nodeIdentifier); } final long index = this.nodeIdentifiers.size(); this.nodeIdentifiers.add(nodeIdentifier); this.nodeIdentifierToNodeArrayIndex.put(nodeIdentifier, index); this.nodeLocations.add(location.asConcatenation()); // Fill the in/out edges arrays for later this.nodeInEdgesIndices.add(new long[0]); this.nodeOutEdgesIndices.add(new long[0]); this.getAsNewNodeSpatialIndex().add(new PackedNode(this, index)); // Tags updatePackedTagStore(this.nodeTags, index, tags); } } protected void addPoint(final long pointIdentifier, final Location location, final Map tags) { synchronized (this.pointIdentifiers) { if (this.pointIdentifierToPointArrayIndex.containsKey(pointIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.POINT, pointIdentifier); } final long index = this.pointIdentifiers.size(); this.pointIdentifiers.add(pointIdentifier); this.pointIdentifierToPointArrayIndex.put(pointIdentifier, index); this.pointLocations.add(location.asConcatenation()); this.getAsNewPointSpatialIndex().add(new PackedPoint(this, index)); // Tags updatePackedTagStore(this.pointTags, index, tags); } } /** * Add a relation to the {@link PackedAtlas}. WARNING: This method will throw * {@link AtlasIntegrityException}s in the following cases: *

    *
  • The identifiers list, types list and roles list do not have all the same length *
  • The identifiers list, types list and roles list are empty *
  • Some member identifiers are null *
* * @param relationIdentifier * The identifier of the relation * @param relationOsmIdentifier * The original OSM identifier of the relation. Can be the same as the identifier. * @param identifiers * The member identifiers. * @param types * The member types in the same order as the member identifiers * @param roles * The member roles in the same order as the member identifiers * @param tags * The relation's tags * @param geometry * the JTS geometry for the relation, if it is a multipolygon */ protected void addRelation(final long relationIdentifier, final long relationOsmIdentifier, final List identifiers, final List types, final List roles, final Map tags, final MultiPolygon geometry) { if (identifiers.size() != types.size() || types.size() != roles.size()) { throw new AtlasIntegrityException( "Different sizes for relation identifiers and types and roles."); } if (identifiers.isEmpty()) { throw new AtlasIntegrityException("Cannot add the relation {} with no members", relationIdentifier); } // Do not allow relations with some null members. if (identifiers.stream().anyMatch(Objects::isNull)) { throw new AtlasIntegrityException("Cannot have a relation with null members."); } synchronized (this.relationIdentifiers) { if (this.relationIdentifierToRelationArrayIndex.containsKey(relationIdentifier)) { throw new AtlasIntegrityException( PackedAtlasLogMessages.ALREADY_EXISTS_EXCEPTION_MESSAGE, ItemType.RELATION, relationIdentifier); } final long index = this.relationIdentifiers.size(); this.relationIdentifiers.add(relationIdentifier); this.relationIdentifierToRelationArrayIndex.put(relationIdentifier, index); this.relationOsmIdentifierToRelationIdentifiers.add(relationOsmIdentifier, relationIdentifier); this.relationOsmIdentifiers.add(relationOsmIdentifier); final long[] memberIndices = new long[identifiers.size()]; final byte[] typeValues = new byte[types.size()]; final int[] roleValues = new int[roles.size()]; for (int i = 0; i < identifiers.size(); i++) { final ItemType type = types.get(i); typeValues[i] = (byte) type.getValue(); roleValues[i] = this.dictionary.add(roles.get(i)); final Long memberIdentifier = identifiers.get(i); switch (type) { case NODE: addRelationMember("Node", index, memberIdentifier, i, memberIndices, this.nodeIdentifierToNodeArrayIndex, this.nodeIndexToRelationIndices); break; case EDGE: addRelationMember("Edge", index, memberIdentifier, i, memberIndices, this.edgeIdentifierToEdgeArrayIndex, this.edgeIndexToRelationIndices); break; case AREA: addRelationMember("Area", index, memberIdentifier, i, memberIndices, this.areaIdentifierToAreaArrayIndex, this.areaIndexToRelationIndices); break; case LINE: addRelationMember("Line", index, memberIdentifier, i, memberIndices, this.lineIdentifierToLineArrayIndex, this.lineIndexToRelationIndices); break; case POINT: addRelationMember("Point", index, memberIdentifier, i, memberIndices, this.pointIdentifierToPointArrayIndex, this.pointIndexToRelationIndices); break; case RELATION: addRelationMember("Relation", index, memberIdentifier, i, memberIndices, this.relationIdentifierToRelationArrayIndex, this.relationIndexToRelationIndices); break; default: throw new CoreException("Cannot recognize ItemType {}", type); } } if (geometry != null) { if (!this.containsEnhancedRelationGeometry) { throw new AtlasIntegrityException( "Could not add Relation {} with enhanced geometry because it was not enabled for this atlas", relationIdentifier); } final ByteArrayResource compressedGeom = new ByteArrayResource( geometry.toText().getBytes().length * (long) Byte.SIZE); compressedGeom.setCompressor(Compressor.GZIP); compressedGeom.writeAndClose(geometry.toText()); this.relationGeometries.add(compressedGeom.readBytesAndClose()); } else if (this.containsEnhancedRelationGeometry) { final ByteArrayResource compressedGeom = new ByteArrayResource(); compressedGeom.setCompressor(Compressor.GZIP); compressedGeom.writeAndClose(""); this.relationGeometries.add(compressedGeom.readBytesAndClose()); } this.relationMemberTypes.add(typeValues); this.relationMemberIndices.add(memberIndices); this.relationMemberRoles.add(roleValues); // Tags updatePackedTagStore(this.relationTags, index, tags); } } protected long areaIdentifier(final long index) { return this.areaIdentifiers().get(index); } protected Polygon areaPolygon(final long index) { return this.areaPolygons().get(index); } protected Set areaRelations(final long index) { return itemRelations(this.areaIndexToRelationIndices().get(index)); } protected Map areaTags(final long index) { return this.areaTags().keyValuePairs(index); } protected Node edgeEndNode(final long index) { return new PackedNode(this, this.edgeEndNodeIndex().get(index)); } protected long edgeIdentifier(final long index) { return this.edgeIdentifiers().get(index); } protected PolyLine edgePolyLine(final long index) { return this.edgePolyLines().get(index); } protected Set edgeRelations(final long index) { return itemRelations(this.edgeIndexToRelationIndices().get(index)); } protected Node edgeStartNode(final long index) { return new PackedNode(this, this.edgeStartNodeIndex().get(index)); } protected Map edgeTags(final long index) { return this.edgeTags().keyValuePairs(index); } /** * Get the serialization format used for loading this {@link PackedAtlas}. * * @return The load serialization format setting */ protected AtlasSerializationFormat getLoadSerializationFormat() { return this.loadSerializationFormat; } protected Optional getSerializer() { return Optional.ofNullable(this.serializer); } protected boolean isEmpty() { return this.nodeIdentifiers().isEmpty() && this.edgeIdentifiers().isEmpty() && this.areaIdentifiers().isEmpty() && this.lineIdentifiers().isEmpty() && this.pointIdentifiers().isEmpty() && this.relationIdentifiers().isEmpty(); } protected long lineIdentifier(final long index) { return this.lineIdentifiers().get(index); } protected PolyLine linePolyLine(final long index) { return this.linePolyLines().get(index); } protected Set lineRelations(final long index) { return itemRelations(this.lineIndexToRelationIndices().get(index)); } protected Map lineTags(final long index) { return this.lineTags().keyValuePairs(index); } protected long nodeIdentifier(final long index) { return this.nodeIdentifiers().get(index); } /** * In very rare cases, Way Slicing will return slightly non-deterministic cut locations in * different shards. This tolerance allows the PackedAtlasBuilder to identify very closeby nodes * and use them instead. * * @param location * The location to target * @param searchDistance * The distance to search for around the location * @param toleranceDistance * The maximum distance at which a node is accepted * @return The resulting node identifier */ protected Long nodeIdentifierForEnlargedLocation(final Location location, final Distance searchDistance, final Distance toleranceDistance) { final Rectangle locationBounds = location.bounds().expand(searchDistance); buildNodeSpatialIndexIfNecessary(); final SortedSet nodes = new TreeSet<>((node1, node2) -> { final Distance distance1 = location.distanceTo(node1.getLocation()); final Distance distance2 = location.distanceTo(node2.getLocation()); final double difference = distance2.asMillimeters() - distance1.asMillimeters(); if (difference > 0.0) { return 1; } else if (difference < 0.0) { return -1; } else { return 0; } }); this.getNodeSpatialIndex().get(locationBounds).forEach(nodes::add); for (final Node candidate : nodes) { final Distance distance = location.distanceTo(candidate.getLocation()); if (distance.isLessThanOrEqualTo(toleranceDistance)) { return candidate.getIdentifier(); } } return null; } /** * Return the identifier of the {@link Node}, if any, at a given {@link Location}. If there are * multiple {@link Node}s at the given location, the one with the lowest identifier is returned. * * @param location * the location to check * @return the {@link Node} if it exists, null otherwise */ protected Long nodeIdentifierForLocation(final Location location) { buildNodeSpatialIndexIfNecessary(); final Rectangle locationBounds = location.bounds(); final SortedSet nodesByAscendingIdentifier = new TreeSet<>((node1, node2) -> { if (node1.getIdentifier() < node2.getIdentifier()) { return -1; } else if (node1.getIdentifier() > node2.getIdentifier()) { return 1; } else { return 0; } }); this.getNodeSpatialIndex().get(locationBounds).forEach(nodesByAscendingIdentifier::add); if (!nodesByAscendingIdentifier.isEmpty()) { return nodesByAscendingIdentifier.first().getIdentifier(); } return null; } protected SortedSet nodeInEdges(final long index) { final SortedSet result = new TreeSet<>(); for (final long edgeIndex : this.nodeInEdgesIndices().get(index)) { result.add(new PackedEdge(this, edgeIndex)); } return result; } protected Location nodeLocation(final long index) { return new Location(this.nodeLocations().get(index)); } protected SortedSet nodeOutEdges(final long index) { final SortedSet result = new TreeSet<>(); for (final long edgeIndex : this.nodeOutEdgesIndices().get(index)) { result.add(new PackedEdge(this, edgeIndex)); } return result; } protected Set nodeRelations(final long index) { return itemRelations(this.nodeIndexToRelationIndices().get(index)); } protected Map nodeTags(final long index) { return this.nodeTags().keyValuePairs(index); } protected long pointIdentifier(final long index) { return this.pointIdentifiers().get(index); } protected Location pointLocation(final long index) { return new Location(this.pointLocations().get(index)); } protected Set pointRelations(final long index) { return itemRelations(this.pointIndexToRelationIndices().get(index)); } protected Map pointTags(final long index) { return this.pointTags().keyValuePairs(index); } protected RelationMemberList relationAllKnownOsmMembers(final long index) { final List result = new ArrayList<>(); for (final long candidateIdentifier : this.relationOsmIdentifierToRelationIdentifiers() .get(relationOsmIdentifier(index))) { final long candidateIndex = this.relationIdentifierToRelationArrayIndex() .get(candidateIdentifier); result.addAll(relationMembers(candidateIndex)); } return new RelationMemberList(result); } protected List relationAllRelationsWithSameOsmIdentifier(final long index) { final List result = new ArrayList<>(); for (final long candidateIdentifier : this.relationOsmIdentifierToRelationIdentifiers() .get(relationOsmIdentifier(index))) { final long candidateIndex = this.relationIdentifierToRelationArrayIndex() .get(candidateIdentifier); result.add(new PackedRelation(this, candidateIndex)); } return result; } protected MultiPolygon relationGeometry(final long index) { if (this.builtRelationGeometries == null) { this.builtRelationGeometries = new HashMap<>(); } if (this.builtRelationGeometries.containsKey(index)) { return this.builtRelationGeometries.get(index); } try { final ByteArrayResource compressed = new ByteArrayResource( this.relationGeometries().get(index).length * (long) Byte.SIZE); compressed.writeAndClose(this.relationGeometries().get(index)); compressed.setDecompressor(Decompressor.GZIP); final MultiPolygon geom = (MultiPolygon) new WKTReader() .read(new String(compressed.readBytesAndClose())); this.builtRelationGeometries.put(index, geom); return geom; } catch (final ParseException exc) { logger.warn("Couldn't deserialized relation geometry for relation {}", index, exc); return null; } } protected long relationIdentifier(final long index) { return this.relationIdentifiers().get(index); } /** * Fetch the {@link RelationMemberList} for a given index. Note that while OSM technically * allows duplicate {@link RelationMember}s, this method disallows duplicates. So a valid OSM * relation that looks like {[1L, 'role1', POINT], [1L, 'role1', POINT], [45L, 'area', AREA]} * would become {[1L, 'role1', POINT], [45L, 'area', AREA]}. * * @param index * the {@link Relation} array index * @return a fully constructed {@link RelationMemberList} for the {@link Relation} at the given * index */ protected RelationMemberList relationMembers(final long index) { final Set result = new TreeSet<>(); int arrayIndex = 0; for (final byte typeValue : this.relationMemberTypes().get(index)) { final ItemType type = ItemType.forValue(typeValue); final long memberIndex = this.relationMemberIndices().get(index)[arrayIndex]; final String role = this.dictionary() .word(this.relationMemberRoles().get(index)[arrayIndex]); final AtlasEntity entity; switch (type) { case NODE: entity = new PackedNode(this, memberIndex); break; case EDGE: entity = new PackedEdge(this, memberIndex); break; case AREA: entity = new PackedArea(this, memberIndex); break; case LINE: entity = new PackedLine(this, memberIndex); break; case POINT: entity = new PackedPoint(this, memberIndex); break; case RELATION: entity = new PackedRelation(this, memberIndex); break; default: throw new CoreException("Invalid member type {}", type); } result.add(new RelationMember(role, entity, relationIdentifier(index))); arrayIndex++; } return new RelationMemberList(result); } protected long relationOsmIdentifier(final long index) { return this.relationOsmIdentifiers().get(index); } protected Set relationRelations(final long index) { return itemRelations(this.relationIndexToRelationIndices().get(index)); } protected Map relationTags(final long index) { return this.relationTags().keyValuePairs(index); } /** * Set the serialization format for loading this {@link PackedAtlas}. * * @param loadFormat * The format to use */ protected void setLoadSerializationFormat(final AtlasSerializationFormat loadFormat) { this.loadSerializationFormat = loadFormat; } /** * This method is to be used by the {@link PackedAtlasBuilder} only * * @param metaData * The new MetaData */ protected void setMetaData(final AtlasMetaData metaData) { this.metaData = metaData; } @Override protected void setName(final String name) // NOSONAR { super.setName(name); } ByteArrayOfArrays enhancedRelationGeometries() { return relationGeometries(); } void setContainsEnhancedRelationGeometry(final boolean flag) { this.containsEnhancedRelationGeometry = flag; } /** * Add a {@link RelationMember} * * @param relationIndex * The index of the {@link Relation} in the {@link Relation} arrays. * @param memberIdentifier * The identifier of the member to add * @param relationMemberListIndex * The index of the member in the {@link Relation}'s member arrays. * @param relationMemberIndexArray * The array of the indices of the relation members. * @param memberIdentifierToArrayIndex * The member type's identifier to array index map * @param memberIndicesToRelationIndices * The member type's index to relation indices map. */ private void addRelationMember(final String type, final long relationIndex, final Long memberIdentifier, final int relationMemberListIndex, final long[] relationMemberIndexArray, final LongToLongMap memberIdentifierToArrayIndex, final LongToLongMultiMap memberIndicesToRelationIndices) { if (memberIdentifierToArrayIndex.containsKey(memberIdentifier)) { relationMemberIndexArray[relationMemberListIndex] = memberIdentifierToArrayIndex .get(memberIdentifier); memberIndicesToRelationIndices.add(relationMemberIndexArray[relationMemberListIndex], relationIndex); } else { throw new AtlasIntegrityException("The {} {} does not exist for relation {}.", type, memberIdentifier, this.relationIdentifiers.get(relationIndex)); } } private LongToLongMap areaIdentifierToAreaArrayIndex() { return deserializedIfNeeded(() -> this.areaIdentifierToAreaArrayIndex, this.fieldAreaIdentifierToAreaArrayIndexLock, FIELD_AREA_IDENTIFIER_TO_AREA_ARRAY_INDEX); } private LongArray areaIdentifiers() { return deserializedIfNeeded(() -> this.areaIdentifiers, this.fieldAreaIdentifiersLock, FIELD_AREA_IDENTIFIERS); } private LongToLongMultiMap areaIndexToRelationIndices() { return deserializedIfNeeded(() -> this.areaIndexToRelationIndices, this.fieldAreaIndexToRelationIndicesLock, FIELD_AREA_INDEX_TO_RELATION_INDICES); } private PolygonArray areaPolygons() { return deserializedIfNeeded(() -> this.areaPolygons, this.fieldAreaPolygonsLock, FIELD_AREA_POLYGONS); } private PackedTagStore areaTags() { return deserializedIfNeeded(() -> this.areaTags, tags -> tags.setDictionary(dictionary()), this.fieldAreaTagsLock, FIELD_AREA_TAGS); } private T deserializedIfNeeded(final Supplier supplier, final Consumer consumer, final Object lock, final String fieldName) { if (supplier.get() == null) { synchronized (lock) // NOSONAR { if (supplier.get() == null) { this.serializer.deserializeIfNeeded(fieldName); } } } if (consumer != null) { consumer.accept(supplier.get()); } return supplier.get(); } private T deserializedIfNeeded(final Supplier supplier, final Object lock, final String fieldName) { return deserializedIfNeeded(supplier, null, lock, fieldName); } private IntegerDictionary dictionary() { return deserializedIfNeeded(() -> this.dictionary, this.fieldDictionaryLock, FIELD_DICTIONARY); } private LongArray edgeEndNodeIndex() { return deserializedIfNeeded(() -> this.edgeEndNodeIndex, this.fieldEdgeEndNodeIndexLock, FIELD_EDGE_END_NODE_INDEX); } private LongToLongMap edgeIdentifierToEdgeArrayIndex() { return deserializedIfNeeded(() -> this.edgeIdentifierToEdgeArrayIndex, this.fieldEdgeIdentifierToEdgeArrayIndexLock, FIELD_EDGE_IDENTIFIER_TO_EDGE_ARRAY_INDEX); } private LongArray edgeIdentifiers() { return deserializedIfNeeded(() -> this.edgeIdentifiers, this.fieldEdgeIdentifiersLock, FIELD_EDGE_IDENTIFIERS); } private LongToLongMultiMap edgeIndexToRelationIndices() { return deserializedIfNeeded(() -> this.edgeIndexToRelationIndices, this.fieldEdgeIndexToRelationIndicesLock, FIELD_EDGE_INDEX_TO_RELATION_INDICES); } private PolyLineArray edgePolyLines() { return deserializedIfNeeded(() -> this.edgePolyLines, this.fieldEdgePolyLinesLock, FIELD_EDGE_POLY_LINES); } private LongArray edgeStartNodeIndex() { return deserializedIfNeeded(() -> this.edgeStartNodeIndex, this.fieldEdgeStartNodeIndexLock, FIELD_EDGE_START_NODE_INDEX); } private PackedTagStore edgeTags() { return deserializedIfNeeded(() -> this.edgeTags, tags -> tags.setDictionary(dictionary()), this.fieldEdgeTagsLock, FIELD_EDGE_TAGS); } private Set itemRelations(final long[] relationIndices) { final Set result = new LinkedHashSet<>(); if (relationIndices == null) { return result; } for (final long relationIndex : relationIndices) { result.add(new PackedRelation(this, relationIndex)); } return result; } private LongToLongMap lineIdentifierToLineArrayIndex() { return deserializedIfNeeded(() -> this.lineIdentifierToLineArrayIndex, this.fieldLineIdentifierToLineArrayIndexLock, FIELD_LINE_IDENTIFIER_TO_LINE_ARRAY_INDEX); } private LongArray lineIdentifiers() { return deserializedIfNeeded(() -> this.lineIdentifiers, this.fieldLineIdentifiersLock, FIELD_LINE_IDENTIFIERS); } private LongToLongMultiMap lineIndexToRelationIndices() { return deserializedIfNeeded(() -> this.lineIndexToRelationIndices, this.fieldLindIndexToRelationIndicesLock, FIELD_LINE_INDEX_TO_RELATION_INDICES); } private PolyLineArray linePolyLines() { return deserializedIfNeeded(() -> this.linePolyLines, this.fieldLinePolyLinesLock, FIELD_LINE_POLYLINES); } private PackedTagStore lineTags() { return deserializedIfNeeded(() -> this.lineTags, tags -> tags.setDictionary(dictionary()), this.fieldLineTagsLock, FIELD_LINE_TAGS); } // Keep this method around so legacy Atlas files can still be deserialized. @SuppressWarnings("unused") private PackedTagStore newPackedTagStore(final long maximumSize, final int memoryBlockSize, final int subArraySize) { return new PackedTagStore(maximumSize, memoryBlockSize, subArraySize, dictionary()) { private static final long serialVersionUID = 5959934069025112665L; }; } private LongToLongMap nodeIdentifierToNodeArrayIndex() { return deserializedIfNeeded(() -> this.nodeIdentifierToNodeArrayIndex, this.fieldNodeIdentifierToNodeArrayIndexLock, FIELD_NODE_IDENTIFIER_TO_NODE_ARRAY_INDEX); } private LongArray nodeIdentifiers() { return deserializedIfNeeded(() -> this.nodeIdentifiers, this.fieldNodeIdentifiersLock, FIELD_NODE_IDENTIFIERS); } private LongArrayOfArrays nodeInEdgesIndices() { return deserializedIfNeeded(() -> this.nodeInEdgesIndices, this.fieldNodeInEdgesIndicesLock, FIELD_NODE_IN_EDGES_INDICES); } private LongToLongMultiMap nodeIndexToRelationIndices() { return deserializedIfNeeded(() -> this.nodeIndexToRelationIndices, this.fieldNodeIndexToRelationIndicesLock, FIELD_NODE_INDEX_TO_RELATION_INDICES); } private LongArray nodeLocations() { return deserializedIfNeeded(() -> this.nodeLocations, this.fieldNodeLocationsLock, FIELD_NODE_LOCATIONS); } private LongArrayOfArrays nodeOutEdgesIndices() { return deserializedIfNeeded(() -> this.nodeOutEdgesIndices, this.fieldNodeOutEdgesIndicesLock, FIELD_NODE_OUT_EDGES_INDICES); } private PackedTagStore nodeTags() { return deserializedIfNeeded(() -> this.nodeTags, tags -> tags.setDictionary(dictionary()), this.fieldNodeTagsLock, FIELD_NODE_TAGS); } private LongToLongMap pointIdentifierToPointArrayIndex() { return deserializedIfNeeded(() -> this.pointIdentifierToPointArrayIndex, this.fieldPointIdentifierToPointArrayIndexLock, FIELD_POINT_IDENTIFIER_TO_POINT_ARRAY_INDEX); } private LongArray pointIdentifiers() { return deserializedIfNeeded(() -> this.pointIdentifiers, this.fieldPointIdentifiersLock, FIELD_POINT_IDENTIFIERS); } private LongToLongMultiMap pointIndexToRelationIndices() { return deserializedIfNeeded(() -> this.pointIndexToRelationIndices, this.fieldPointIndexToRelationIndicesLock, FIELD_POINT_INDEX_TO_RELATION_INDICES); } private LongArray pointLocations() { return deserializedIfNeeded(() -> this.pointLocations, this.fieldPointLocationsLock, FIELD_POINT_LOCATIONS); } private PackedTagStore pointTags() { return deserializedIfNeeded(() -> this.pointTags, tags -> tags.setDictionary(dictionary()), this.fieldPointTagsLock, FIELD_POINT_TAGS); } private void readObject(final java.io.ObjectInputStream inFile) throws IOException, ClassNotFoundException { inFile.defaultReadObject(); if (this.fieldDictionaryLock == null) { this.fieldDictionaryLock = new Object(); } if (this.fieldEdgeIdentifiersLock == null) { this.fieldEdgeIdentifiersLock = new Object(); } if (this.fieldNodeIdentifiersLock == null) { this.fieldNodeIdentifiersLock = new Object(); } if (this.fieldAreaIdentifiersLock == null) { this.fieldAreaIdentifiersLock = new Object(); } if (this.fieldLineIdentifiersLock == null) { this.fieldLineIdentifiersLock = new Object(); } if (this.fieldPointIdentifiersLock == null) { this.fieldPointIdentifiersLock = new Object(); } if (this.fieldRelationIdentifiersLock == null) { this.fieldRelationIdentifiersLock = new Object(); } if (this.fieldEdgeIdentifierToEdgeArrayIndexLock == null) { this.fieldEdgeIdentifierToEdgeArrayIndexLock = new Object(); } if (this.fieldNodeIdentifierToNodeArrayIndexLock == null) { this.fieldNodeIdentifierToNodeArrayIndexLock = new Object(); } if (this.fieldAreaIdentifierToAreaArrayIndexLock == null) { this.fieldAreaIdentifierToAreaArrayIndexLock = new Object(); } if (this.fieldLineIdentifierToLineArrayIndexLock == null) { this.fieldLineIdentifierToLineArrayIndexLock = new Object(); } if (this.fieldPointIdentifierToPointArrayIndexLock == null) { this.fieldPointIdentifierToPointArrayIndexLock = new Object(); } if (this.fieldRelationIdentifierToRelationArrayIndexLock == null) { this.fieldRelationIdentifierToRelationArrayIndexLock = new Object(); } if (this.fieldNodeLocationsLock == null) { this.fieldNodeLocationsLock = new Object(); } if (this.fieldNodeInEdgesIndicesLock == null) { this.fieldNodeInEdgesIndicesLock = new Object(); } if (this.fieldNodeOutEdgesIndicesLock == null) { this.fieldNodeOutEdgesIndicesLock = new Object(); } if (this.fieldNodeTagsLock == null) { this.fieldNodeTagsLock = new Object(); } if (this.fieldNodeIndexToRelationIndicesLock == null) { this.fieldNodeIndexToRelationIndicesLock = new Object(); } if (this.fieldEdgeStartNodeIndexLock == null) { this.fieldEdgeStartNodeIndexLock = new Object(); } if (this.fieldEdgeEndNodeIndexLock == null) { this.fieldEdgeEndNodeIndexLock = new Object(); } if (this.fieldEdgePolyLinesLock == null) { this.fieldEdgePolyLinesLock = new Object(); } if (this.fieldEdgeTagsLock == null) { this.fieldEdgeTagsLock = new Object(); } if (this.fieldEdgeIndexToRelationIndicesLock == null) { this.fieldEdgeIndexToRelationIndicesLock = new Object(); } if (this.fieldAreaPolygonsLock == null) { this.fieldAreaPolygonsLock = new Object(); } if (this.fieldAreaTagsLock == null) { this.fieldAreaTagsLock = new Object(); } if (this.fieldAreaIndexToRelationIndicesLock == null) { this.fieldAreaIndexToRelationIndicesLock = new Object(); } if (this.fieldLinePolyLinesLock == null) { this.fieldLinePolyLinesLock = new Object(); } if (this.fieldLineTagsLock == null) { this.fieldLineTagsLock = new Object(); } if (this.fieldLindIndexToRelationIndicesLock == null) { this.fieldLindIndexToRelationIndicesLock = new Object(); } if (this.fieldPointLocationsLock == null) { this.fieldPointLocationsLock = new Object(); } if (this.fieldPointTagsLock == null) { this.fieldPointTagsLock = new Object(); } if (this.fieldPointIndexToRelationIndicesLock == null) { this.fieldPointIndexToRelationIndicesLock = new Object(); } if (this.fieldRelationMembersIndicesLock == null) { this.fieldRelationMembersIndicesLock = new Object(); } if (this.fieldRelationMemberTypesLock == null) { this.fieldRelationMemberTypesLock = new Object(); } if (this.fieldRelationMemberRolesLock == null) { this.fieldRelationMemberRolesLock = new Object(); } if (this.fieldRelationTagsLock == null) { this.fieldRelationTagsLock = new Object(); } if (this.fieldRelationIndexToRelationIndicesLock == null) { this.fieldRelationIndexToRelationIndicesLock = new Object(); } if (this.fieldRelationOsmIdentifierToRelationIdentifiersLock == null) { this.fieldRelationOsmIdentifierToRelationIdentifiersLock = new Object(); } if (this.fieldRelationOsmIdentifiersLock == null) { this.fieldRelationOsmIdentifiersLock = new Object(); } if (this.fieldRelationGeometriesLock == null) { this.fieldRelationGeometriesLock = new Object(); } } private ByteArrayOfArrays relationGeometries() { return deserializedIfNeeded(() -> this.relationGeometries, this.fieldRelationGeometriesLock, FIELD_RELATION_GEOMETRIES); } private LongToLongMap relationIdentifierToRelationArrayIndex() { return deserializedIfNeeded(() -> this.relationIdentifierToRelationArrayIndex, this.fieldRelationIdentifierToRelationArrayIndexLock, FIELD_RELATION_IDENTIFIER_TO_RELATION_ARRAY_INDEX); } private LongArray relationIdentifiers() { return deserializedIfNeeded(() -> this.relationIdentifiers, this.fieldRelationIdentifiersLock, FIELD_RELATION_IDENTIFIERS); } private LongToLongMultiMap relationIndexToRelationIndices() { return deserializedIfNeeded(() -> this.relationIndexToRelationIndices, this.fieldRelationIndexToRelationIndicesLock, FIELD_RELATION_INDEX_TO_RELATION_INDICES); } private LongArrayOfArrays relationMemberIndices() { return deserializedIfNeeded(() -> this.relationMemberIndices, this.fieldRelationMembersIndicesLock, FIELD_RELATION_MEMBERS_INDICES); } private IntegerArrayOfArrays relationMemberRoles() { return deserializedIfNeeded(() -> this.relationMemberRoles, this.fieldRelationMemberRolesLock, FIELD_RELATION_MEMBER_ROLES); } private ByteArrayOfArrays relationMemberTypes() { return deserializedIfNeeded(() -> this.relationMemberTypes, this.fieldRelationMemberTypesLock, FIELD_RELATION_MEMBER_TYPES); } private LongToLongMultiMap relationOsmIdentifierToRelationIdentifiers() { return deserializedIfNeeded(() -> this.relationOsmIdentifierToRelationIdentifiers, this.fieldRelationOsmIdentifierToRelationIdentifiersLock, FIELD_RELATION_OSM_IDENTIFIER_TO_RELATION_IDENTIFIERS); } private LongArray relationOsmIdentifiers() { return deserializedIfNeeded(() -> this.relationOsmIdentifiers, this.fieldRelationOsmIdentifiersLock, FIELD_RELATION_OSM_IDENTIFIERS); } private PackedTagStore relationTags() { return deserializedIfNeeded(() -> this.relationTags, tags -> tags.setDictionary(dictionary()), this.fieldRelationTagsLock, FIELD_RELATION_TAGS); } /** * Update references for Node in/out edges * * @param nodeEdgesIndices * Either the nodeInEdges or the nodeOutEdges */ private void updateNodeEdgesReference(final long nodeIndex, final LongArrayOfArrays nodeEdgesIndices, final long edgeIndex) { final long[] nodeEdges = nodeEdgesIndices.get(nodeIndex); final long[] newNodeEdges = new long[nodeEdges.length + 1]; for (int i = 0; i < nodeEdges.length; i++) { newNodeEdges[i] = nodeEdges[i]; } newNodeEdges[newNodeEdges.length - 1] = edgeIndex; nodeEdgesIndices.set(nodeIndex, newNodeEdges); } private void updatePackedTagStore(final PackedTagStore packedTagStore, final long index, final Map tags) { if (tags.isEmpty()) { packedTagStore.add(index, null, null); } else { for (final Map.Entry entry : tags.entrySet()) { packedTagStore.add(index, entry.getKey(), entry.getValue()); } } } private void writeObject(final java.io.ObjectOutputStream out) throws IOException { if (this.serializer != null) { this.serializer.deserializeAllFieldsIfNeeded(); } out.defaultWriteObject(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasBuilder.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasBuilder; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.exception.AtlasIntegrityException; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link AtlasBuilder} for a {@link PackedAtlas}. This is not thread safe! * * @author matthieun */ public final class PackedAtlasBuilder implements AtlasBuilder { private static final Logger logger = LoggerFactory.getLogger(PackedAtlasBuilder.class); private static final int MAXIMUM_RELATION_MEMBER_DEPTH = 500; // In very rare cases, Way Slicing will return slightly non-deterministic cut locations in // different shards. This tolerance allows the PackedAtlasBuilder to identify very closeby nodes // and use them instead. private static final Distance NODE_SEARCH_DISTANCE = Distance.ONE_METER; private static final Distance NODE_TOLERANCE_DISTANCE = Distance.meters(0.1); private PackedAtlas atlas; private AtlasSize sizeEstimates = AtlasSize.DEFAULT; private boolean locked = false; private String name; private AtlasMetaData metaData = new AtlasMetaData(); @Override public void addArea(final long identifier, final Polygon geometry, final Map tags) { initialize(); try { this.atlas.addArea(identifier, geometry, tags); } catch (final Exception e) { logger.error("Error adding Area ({}): {}", identifier, geometry.toWkt(), e); } } @Override public void addEdge(final long identifier, final PolyLine geometry, final Map tags) { initialize(); final Location start = geometry.first(); final Location end = geometry.last(); Long startNodeIdentifier = this.atlas.nodeIdentifierForLocation(start); Long endNodeIdentifier = this.atlas.nodeIdentifierForLocation(end); if (startNodeIdentifier == null) { startNodeIdentifier = this.atlas.nodeIdentifierForEnlargedLocation(start, NODE_SEARCH_DISTANCE, NODE_TOLERANCE_DISTANCE); if (startNodeIdentifier == null) { throw new AtlasIntegrityException( "Atlas does not contain Node for Location {} for edge {}", start, identifier); } logger.warn( "Atlas does not contain Node for Location {} for edge {}. " + "Found very close node {} and using it instead.", start, identifier, startNodeIdentifier); } if (endNodeIdentifier == null) { endNodeIdentifier = this.atlas.nodeIdentifierForEnlargedLocation(end, NODE_SEARCH_DISTANCE, NODE_TOLERANCE_DISTANCE); if (endNodeIdentifier == null) { throw new AtlasIntegrityException( "Atlas does not contain Node for Location {} for edge {}", end, identifier); } logger.warn( "Atlas does not contain Node for Location {} for edge {}. " + "Found very close node {} and using it instead.", end, identifier, endNodeIdentifier); } try { this.atlas.addEdge(identifier, startNodeIdentifier, endNodeIdentifier, geometry, tags); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Edge ({}): {}", identifier, geometry.toWkt(), e); } } @Override public void addLine(final long identifier, final PolyLine geometry, final Map tags) { initialize(); try { this.atlas.addLine(identifier, geometry, tags); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Line ({}): {}", identifier, geometry.toWkt(), e); } } @Override public void addNode(final long identifier, final Location geometry, final Map tags) { initialize(); try { this.atlas.addNode(identifier, geometry, tags); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Node ({}): {}", identifier, geometry.toWkt(), e); } } @Override public void addPoint(final long identifier, final Location geometry, final Map tags) { initialize(); try { this.atlas.addPoint(identifier, geometry, tags); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Point ({}): {}", identifier, geometry.toWkt(), e); } } @Override public void addRelation(final long identifier, final long osmIdentifier, final RelationBean structure, final Map tags) { if (structure.isEmpty()) { throw new CoreException("Cannot add relation {} with an empty member list.", identifier); } initialize(); try { this.atlas.addRelation(identifier, osmIdentifier, structure.getMemberIdentifiers(), structure.getMemberTypes(), structure.getMemberRoles(), tags, null); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Relation ({}): {}", identifier, structure, e); } } public void addRelation(final long identifier, final long osmIdentifier, final RelationBean structure, final Map tags, final MultiPolygon geometry) { if (structure.isEmpty()) { throw new CoreException("Cannot add relation {} with an empty member list.", identifier); } initialize(); try { this.atlas.addRelation(identifier, osmIdentifier, structure.getMemberIdentifiers(), structure.getMemberTypes(), structure.getMemberRoles(), tags, geometry); } catch (final AtlasIntegrityException e) { throw e; } catch (final Exception e) { logger.error("Error adding Relation ({}): {}", identifier, structure.toString(), e); } } @Override public Atlas get() { initialize(); this.locked = true; if (this.atlas.isEmpty()) { logger.warn("An Atlas is Located, and therefore cannot be empty."); return null; } if (Iterables.size(this.atlas) == this.atlas.numberOfRelations()) { logger.warn( "An Atlas is Located, and therefore cannot be made of only relations (which cannot be located as there are no other features)."); return null; } verifyNegativeEdgesHaveMainEdge(); this.atlas.relations().forEach(relation -> { try { // Make sure that the relations are not looping to each other, or just bounds-less. validateRelation(relation, relation.getIdentifier(), 0); } catch (final Exception e) { throw new CoreException("Relation {} is corrupted. Invalidating Atlas!", relation.getIdentifier(), e); } }); // Update the meta data so the Atlas sizes are correct. final AtlasSize updatedAtlasSize = new AtlasSize(this.atlas.numberOfEdges(), this.atlas.numberOfNodes(), this.atlas.numberOfAreas(), this.atlas.numberOfLines(), this.atlas.numberOfPoints(), this.atlas.numberOfRelations()); this.atlas.setMetaData(this.metaData.copyWithNewSize(updatedAtlasSize)); return this.atlas; } public PackedAtlas peek() { initialize(); return this.atlas; } @Override public void setMetaData(final AtlasMetaData metaData) { this.metaData = metaData; } @Override public void setSizeEstimates(final AtlasSize estimates) { this.sizeEstimates = estimates; } public PackedAtlasBuilder withEnhancedRelationGeometry() { initialize(true); return this; } public PackedAtlasBuilder withMetaData(final AtlasMetaData metaData) { setMetaData(metaData); return this; } public PackedAtlasBuilder withName(final String name) { this.name = name; return this; } public PackedAtlasBuilder withSizeEstimates(final AtlasSize estimates) { setSizeEstimates(estimates); return this; } private void initialize() { initialize(false); } private void initialize(final boolean withEnhancedRelationGeometry) { if (this.locked) { throw new CoreException("Cannot keep adding items to a locked graph."); } if (this.atlas == null) { this.atlas = new PackedAtlas(this.sizeEstimates, withEnhancedRelationGeometry); this.atlas.setName(this.name); } } /** * Recursive call to make sure that the relations are really bounded and do not loop on each * other. * * @param relation */ private void validateRelation(final Relation relation, final long parentIdentifier, final int depth) { if (depth > MAXIMUM_RELATION_MEMBER_DEPTH) { throw new CoreException( "Relation {} referencing each other more than {} levels deep, without hitting any bounded feature.", parentIdentifier, MAXIMUM_RELATION_MEMBER_DEPTH); } for (final RelationMember member : relation.members()) { if (member.getEntity() instanceof AtlasItem) { return; } else { validateRelation((Relation) member.getEntity(), parentIdentifier, depth + 1); } } } private void verifyNegativeEdgesHaveMainEdge() { this.atlas.edges().forEach(edge -> { final long edgeIdentifier = edge.getIdentifier(); if (edgeIdentifier < 0 && this.atlas.edge(-edgeIdentifier) == null) { throw new AtlasIntegrityException( "Cannot build an Atlas with a negative edge without its positive counterpart: {}", edgeIdentifier); } }); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasCloner.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Optional; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; /** * Atlas Cloner. Mostly useful to get a {@link MultiAtlas} and clone it into one single * {@link PackedAtlas} * * @author matthieun */ public class PackedAtlasCloner { private String shardName = null; private Optional> additionalMetaDataTags = Optional.empty(); public PackedAtlasCloner() { } public PackedAtlasCloner(final String shardName) { this.shardName = shardName; } /** * Clone an {@link Atlas} * * @param atlas * The source {@link Atlas} * @return A cloned {@link PackedAtlas} */ public PackedAtlas cloneFrom(final Atlas atlas) { final PackedAtlasBuilder builder = new PackedAtlasBuilder(); builder.setSizeEstimates(atlas.metaData().getSize()); AtlasMetaData metaData = atlas.metaData(); if (this.shardName != null) { metaData.copyWithNewShardName(this.shardName); } if (this.additionalMetaDataTags.isPresent()) { final Map atlasTags = metaData.getTags(); atlasTags.putAll(this.additionalMetaDataTags.get()); metaData = new AtlasMetaData(metaData.getSize(), metaData.isOriginal(), metaData.getCodeVersion().orElse(null), metaData.getDataVersion().orElse(null), metaData.getCountry().orElse(null), metaData.getShardName().orElse(null), atlasTags); } builder.setMetaData(metaData); builder.withEnhancedRelationGeometry(); atlas.nodes().forEach( node -> builder.addNode(node.getIdentifier(), node.getLocation(), node.getTags())); atlas.edges().forEach( edge -> builder.addEdge(edge.getIdentifier(), edge.asPolyLine(), edge.getTags())); atlas.areas().forEach( area -> builder.addArea(area.getIdentifier(), area.asPolygon(), area.getTags())); atlas.lines().forEach( line -> builder.addLine(line.getIdentifier(), line.asPolyLine(), line.getTags())); atlas.points().forEach(point -> builder.addPoint(point.getIdentifier(), point.getLocation(), point.getTags())); // It's crucial to add relations in lowest order to highest order, to avoid adding a // relation which may contain an un-added sub-relation. atlas.relationsLowerOrderFirst().forEach(relation -> addRelation(builder, relation)); return (PackedAtlas) builder.get(); } /** * Adds the passed in extra tags to the {@link AtlasMetaData} when the atlas is cloned. *

* CAUTION: This will overwrite current tags if there is already a tag with the same key in the * tag map. * * @param additionalMetaDataTags * Extra {@link AtlasMetaData} tags to add to the cloned atlas * @return The updated {@link PackedAtlasCloner} */ public PackedAtlasCloner withAdditionalMetaDataTags( final Map additionalMetaDataTags) { this.additionalMetaDataTags = Optional.ofNullable(additionalMetaDataTags); return this; } private void addRelation(final PackedAtlasBuilder builder, final Relation relation) { final RelationBean bean = new RelationBean(); relation.members().forEach(member -> bean.addItem(member.getEntity().getIdentifier(), member.getRole(), member.getEntity().getType())); final Optional geom = relation.asMultiPolygon(); if (geom.isPresent()) { builder.addRelation(relation.getIdentifier(), relation.osmRelationIdentifier(), bean, relation.getTags(), geom.get()); } else { builder.addRelation(relation.getIdentifier(), relation.osmRelationIdentifier(), bean, relation.getTags()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasLogMessages.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; /** * @author matthieun */ final class PackedAtlasLogMessages { static final String ALREADY_EXISTS_EXCEPTION_MESSAGE = "{} with identifier {} already exists."; private PackedAtlasLogMessages() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlasSerializer.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas.AtlasSerializationFormat; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.streaming.CounterOutputStream; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.resource.zip.ZipFileWritableResource; import org.openstreetmap.atlas.streaming.resource.zip.ZipResource; import org.openstreetmap.atlas.streaming.resource.zip.ZipResource.ZipIterator; import org.openstreetmap.atlas.streaming.resource.zip.ZipWritableResource; import org.openstreetmap.atlas.utilities.arrays.ByteArrayOfArrays; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StreamIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Class that serializes and deserializes {@link PackedAtlas}s to a {@link ZipResource} * * @author matthieun * @author lcram */ public final class PackedAtlasSerializer { /** * Exception that is thrown in case a field is in a {@link ZipResource} but the current * implementation of the {@link PackedAtlas} does not recognize it. * * @author matthieun */ private static class MissingFieldException extends CoreException { private static final long serialVersionUID = 6780849464228478451L; MissingFieldException(final String message) { super(message); } MissingFieldException(final String message, final Object... items) { super(message, items); } MissingFieldException(final String message, final Throwable cause) { super(message, cause); } } public static final String META_DATA_ERROR_MESSAGE = "MetaData not here!"; private static final Logger logger = LoggerFactory.getLogger(PackedAtlasSerializer.class); // The fields not serialized. private static final StringList EXCLUDED_FIELDS = new StringList(PackedAtlas.FIELD_BOUNDS, PackedAtlas.FIELD_SERIAL_VERSION_UID, PackedAtlas.FIELD_LOGGER, "$SWITCH_TABLE$", PackedAtlas.FIELD_SERIALIZER, PackedAtlas.FIELD_SAVE_SERIALIZATION_FORMAT, PackedAtlas.FIELD_LOAD_SERIALIZATION_FORMAT, PackedAtlas.FIELD_PREFIX, PackedAtlas.FIELD_CONTAINS_ENHANCED_RELATION_GEOMETRY, PackedAtlas.FIELD_BUILT_RELATION_GEOMETRIES, /* https://stackoverflow.com/a/39037512/1558687 */"$jacocoData"); private final PackedAtlas atlas; private final ZipResource source; /** * Use reflection to create a {@link PackedAtlas} from a serialized resource. * * @param resource * The resource * @return The deserialized {@link PackedAtlas} */ protected static PackedAtlas load(final Resource resource) { // Create an empty Atlas. final PackedAtlas atlas = new PackedAtlas(); // Build the serializer with it final PackedAtlasSerializer serializer = new PackedAtlasSerializer(atlas, resource); // Assign the serializer to the Atlas! Then the Atlas will load all the fields depending on // demand. serializer.assign(); // This is for backwards compatibility and will slow Atlas loading determineAtlasLoadFormat(atlas); return atlas; } /* * Try loading the meta data to make sure the data format is appropriate. Keep trying formats * until we find the right one */ private static void determineAtlasLoadFormat(final PackedAtlas atlas) { final AtlasSerializationFormat[] possibleFormats = AtlasSerializationFormat.values(); for (final AtlasSerializationFormat candidateFormat : possibleFormats) { logger.trace("Trying load format {} for atlas {}", candidateFormat, atlas.getName()); atlas.setLoadSerializationFormat(candidateFormat); try { atlas.metaData(); } catch (final CoreException exception) { logger.debug("Load format {} invalid for atlas {}", candidateFormat, atlas.getName(), exception); continue; } // If we make it here, then we found the appropriate format logger.trace("Using load format {} for atlas {}", candidateFormat, atlas.getName()); /* * Now, if we are PROTOBUF, let's check for the enhanced relation geometry that some * atlases may contain. */ if (atlas.getLoadSerializationFormat() == AtlasSerializationFormat.PROTOBUF) { try { atlas.setContainsEnhancedRelationGeometry(true); final ByteArrayOfArrays array = atlas.enhancedRelationGeometries(); if (array == null) { atlas.setContainsEnhancedRelationGeometry(false); } } catch (final CoreException exception) { if ("Unable to read Atlas field relationGeometries" .equals(exception.getMessage())) { atlas.setContainsEnhancedRelationGeometry(false); } else { throw exception; } } } return; } throw new CoreException("Could not determine a valid load format for atlas {}", atlas.getName()); } /** * Construct a new {@link PackedAtlasSerializer}. * * @param atlas * The {@link Atlas} to be serialized / deserialized * @param resource * The resource where to serialize / deserialize from. */ protected PackedAtlasSerializer(final PackedAtlas atlas, final Resource resource) { this.atlas = atlas; if (resource instanceof File && !resource.isGzipped()) { // Make sure to use ZipFileWritableResource to take advantage of the random access. this.source = new ZipFileWritableResource((File) resource); } else if (resource instanceof WritableResource) { this.source = new ZipWritableResource((WritableResource) resource); } else { this.source = new ZipResource(resource); } } /** * Go after all the fields that might not have been deserialized and deserialize them */ protected void deserializeAllFieldsIfNeeded() { fields().map(Field::getName).forEach(this::deserializeIfNeeded); } /** * This method is used by the {@link PackedAtlas} to access its own fields! * * @param name * The name of the field */ protected void deserializeIfNeeded(final String name) { final Object member; try { final Field field = readField(name); member = getField(field); if (member == null) { if (this.source == null) { throw new CoreException( "The PackedAtlasSerializer has not been properly assigned."); } // If the field is not populated, this will trigger a load (partial or not, // depending on the zip resource) load(name); } } catch (final Exception e) { throw new CoreException("Unable to read Atlas field {}", name, e); } } /** * Save an Atlas file to a {@link ZipWritableResource}. This method uses reflection to identify * all the fields in the {@link PackedAtlas}, and stores each field into a separate zip entry, * named after the field itself. */ protected void save() { if (this.source instanceof ZipWritableResource) { // Load the Atlas completely if it has not been loaded yet this.atlas.getSerializer() .ifPresent(PackedAtlasSerializer::deserializeAllFieldsIfNeeded); final ZipWritableResource destination = (ZipWritableResource) this.source; // Isolate the metaData field final Field metaData = readField(PackedAtlas.FIELD_META_DATA); final Iterable firstResource = Iterables.from(fieldTranslator(metaData)); final Iterable fieldResources = fields().filter(field -> { final String fieldName = field.getName(); /* * If this atlas does not contain enhanced relation geometries, skip serialization */ if (!this.atlas.containsEnhancedRelationGeometry() && PackedAtlas.FIELD_RELATION_GEOMETRIES.equals(fieldName)) { return false; } return !PackedAtlas.FIELD_META_DATA.equals(fieldName) && !EXCLUDED_FIELDS.startsWithContains(fieldName) && !fieldName.contains("Lock"); }).map(this::fieldTranslator).collect(); // Put the metaData field first, always. final Iterable result = new MultiIterable<>(firstResource, fieldResources); destination.writeAndClose(result); } else { throw new CoreException("The ZipResource {} is not writable.", this.source); } } /** * Assign itself as the Atlas' official serializer */ private void assign() { setField(readField(PackedAtlas.FIELD_SERIALIZER), this); } /** * @return True if the underlying {@link Resource} allows for random access to the serialized * fields. */ private boolean canLoadWithRandomAccess() { return this.source instanceof ZipFileWritableResource; } private OutputStream compress(final OutputStream out) throws IOException { return out; } private InputStream decompress(final InputStream input) throws IOException { return input; } private void deserializeAllFields() { Iterables.stream(this.source.entries()).forEach(resource -> { final String name = resource.getName(); try { final Field field = readField(name); final Object value = deserializeResource(resource, name); setField(field, value); } catch (final MissingFieldException e) { // Skipping field, comes from a legacy serialized file. We however have to read it // fully to move to the next one. Here we skip the selection logic of // deserializeResource and just force Java deserialization deserializeJavaResource(resource); } }); } private Object deserializeJavaResource(final Resource resource) { try (ObjectInputStream input = new ObjectInputStream(decompress(resource.read()))) { final Time start = Time.now(); final Object result = input.readObject(); Streams.close(input); logger.trace("Loaded Field {} from {} in {}", resource.getName(), this.source, start.elapsedSince()); return result; } catch (final Exception e) { throw new CoreException("Could not load Field {} from {}", resource.getName(), this.source, e); } } private Object deserializeProtoResource(final Resource resource, final String fieldName) { final Field field = readField(fieldName); final Class fieldClass = field.getType(); Constructor fieldClassConstructor = null; // We need to obtain a dummy instance of the field we want to deserialize. We then use this // dummy instance as a handle to get the correct {@link ProtoAdapter}. try { fieldClassConstructor = fieldClass.getDeclaredConstructor(); } catch (final Exception exception) { throw new CoreException("Class {} does not implement a nullary constructor", fieldClass.getName(), exception); } fieldClassConstructor.setAccessible(true); Object handle = null; try { handle = fieldClassConstructor.newInstance(); } catch (final Exception exception) { throw new CoreException("Failed to create instance of {}", fieldClass.getName(), exception); } ProtoSerializable protoHandle = null; try { protoHandle = (ProtoSerializable) handle; } catch (final ClassCastException exception) { throw new CoreException("{} is not ProtoSerializable", fieldClass.getName(), exception); } return protoHandle.getProtoAdapter().deserialize(resource.readBytesAndClose()); } private Object deserializeResource(final Resource resource, final String fieldName) { final AtlasSerializationFormat loadFormat = this.atlas.getLoadSerializationFormat(); Object result = null; switch (loadFormat) { case JAVA: result = deserializeJavaResource(resource); break; case PROTOBUF: result = deserializeProtoResource(resource, fieldName); break; default: throw new CoreException("Unsupported serialization format {}", loadFormat.toString()); } if (result == null) { throw new CoreException("Unable to deserialize field {} from resource {} in {}.", fieldName, resource.getName(), this.atlas.getName()); } return result; } /** * Deserialize a specific field and assign it to the Atlas. * * @param name * The name of the field. */ private void deserializeSingleField(final String name) { final Object result; if (canLoadWithRandomAccess()) { if (PackedAtlas.FIELD_RELATION_GEOMETRIES.equals(name) && !this.atlas.containsEnhancedRelationGeometry()) { return; } final Resource resource = ((ZipFileWritableResource) this.source).entryForName(name); result = deserializeResource(resource, name); } else if (PackedAtlas.FIELD_META_DATA.equals(name)) { // The metaData field is always the first. final Iterable resources = this.source.entries(); try (ZipIterator iterator = (ZipIterator) resources.iterator()) { final Resource resource = iterator.next(); if (resource == null) { throw new CoreException(META_DATA_ERROR_MESSAGE); } result = deserializeResource(resource, name); } } else { throw new CoreException( "Cannot deserialize a specific field without a ZipFileWritableResource"); } setField(readField(name), result); } /** * The function that translates a reflection {@link Field} into a {@link Resource} * * @param field * The field * @return The resource */ private Resource fieldTranslator(final Field field) { final AtlasSerializationFormat saveFormat = this.atlas.getSaveSerializationFormat(); switch (saveFormat) { case JAVA: final Object objectCandidate = getField(field); return makeJavaResource(objectCandidate, field.getName()); case PROTOBUF: final ProtoSerializable protoCandidate = (ProtoSerializable) getField(field); return makeProtoResource(protoCandidate, field.getName()); default: throw new CoreException("Unsupported serialization format {}", saveFormat.toString()); } } private StreamIterable fields() { return Iterables.stream(Iterables.from(PackedAtlas.class.getDeclaredFields())) .filter(field -> !EXCLUDED_FIELDS.startsWithContains(field.getName())).map(field -> { field.setAccessible(true); return field; }); } private Object getField(final Field field) { try { return field.get(this.atlas); } catch (final Exception e) { throw new CoreException("Unable to access field {} for {}", field.getName(), this.atlas.getName(), e); } } /** * De-serialize a specific field and set it to the Atlas. * * @param name * The name of the field. */ private void load(final String name) { if (canLoadWithRandomAccess() || PackedAtlas.FIELD_META_DATA.equals(name)) { deserializeSingleField(name); } else { deserializeAllFields(); } } /** * Transform a field of this Atlas into a readable {@link Resource}. The underlying * implementation stores everything in a {@link ByteArrayResource} * * @param field * The field to translate * @param name * The name of the resource * @return The resource */ private Resource makeJavaResource(final Object field, final String name) { // First pass read, to count the size final CounterOutputStream counterOutputStream = new CounterOutputStream(); try (ObjectOutputStream outCounter = new ObjectOutputStream( compress(new BufferedOutputStream(counterOutputStream)))) { outCounter.writeObject(field); } catch (final Exception e) { throw new CoreException("Could not count the size of {}.", field, e); } final long count = counterOutputStream.getCount(); // Second pass, write to the memory resource. final ByteArrayResource resource = new ByteArrayResource(count).withName(name); logger.trace("Saving field {}", resource.getName()); if (field == null) { logger.warn("Field {} is null in atlas {} of size {}", name, this.atlas.getName(), this.atlas.size()); return resource; } try (ObjectOutputStream out = new ObjectOutputStream( compress(new BufferedOutputStream(resource.write())))) { out.writeObject(field); } catch (final Exception e) { throw new CoreException("Could not convert {} to a readable resource.", field, e); } return resource; } private Resource makeProtoResource(final ProtoSerializable field, final String name) { // We automatically get the correct adapter for whatever type 'field' happens to be final ProtoAdapter adapter = field.getProtoAdapter(); // The adapter handles all the actual serialization using the protobuf classes. Easy! final byte[] byteContents = adapter.serialize(field); final ByteArrayResource resource = new ByteArrayResource(byteContents.length) .withName(name); try (BufferedOutputStream out = new BufferedOutputStream(resource.write())) { out.write(byteContents); } catch (final Exception e) { throw new CoreException("Could not convert {} to a readable resource.", field, e); } return resource; } private Field readField(final String name) throws MissingFieldException { try { final Field result = PackedAtlas.class.getDeclaredField(name); result.setAccessible(true); return result; } catch (final NoSuchFieldException e) { logger.warn("Unable to access field {}", name); throw new MissingFieldException("Unable to access field {}", name, e); } } /** * Assign a field to the Atlas * * @param field * The field to assign * @param object * The object to assign to the field. */ private void setField(final Field field, final Object object) { try { field.set(this.atlas, object); } catch (final Exception e) { throw new CoreException("Cannot set field {} for Atlas {}", field.getName(), this.atlas.getName(), e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedEdge.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Edge} from a {@link PackedAtlas} * * @author matthieun */ public class PackedEdge extends Edge { private static final long serialVersionUID = -7425733302988626570L; private final long index; protected PackedEdge(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public PolyLine asPolyLine() { return packedAtlas().edgePolyLine(this.index); } @Override public Node end() { return packedAtlas().edgeEndNode(this.index); } @Override public long getIdentifier() { return packedAtlas().edgeIdentifier(this.index); } @Override public Map getTags() { return packedAtlas().edgeTags(this.index); } @Override public Set relations() { return packedAtlas().edgeRelations(this.index); } @Override public Node start() { return packedAtlas().edgeStartNode(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedLine.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Edge} from a {@link PackedAtlas} * * @author matthieun */ public class PackedLine extends Line { private static final long serialVersionUID = 3087755941210424968L; private final long index; protected PackedLine(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public PolyLine asPolyLine() { return packedAtlas().linePolyLine(this.index); } @Override public long getIdentifier() { return packedAtlas().lineIdentifier(this.index); } @Override public Map getTags() { return packedAtlas().lineTags(this.index); } @Override public Set relations() { return packedAtlas().lineRelations(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedNode.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Set; import java.util.SortedSet; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Node} built from a {@link PackedAtlas} * * @author matthieun */ public class PackedNode extends Node { private static final long serialVersionUID = -4505441893548672843L; private final long index; protected PackedNode(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public long getIdentifier() { return packedAtlas().nodeIdentifier(this.index); } @Override public Location getLocation() { return packedAtlas().nodeLocation(this.index); } @Override public Map getTags() { return packedAtlas().nodeTags(this.index); } @Override public SortedSet inEdges() { return packedAtlas().nodeInEdges(this.index); } @Override public SortedSet outEdges() { return packedAtlas().nodeOutEdges(this.index); } @Override public Set relations() { return packedAtlas().nodeRelations(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedPoint.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; /** * {@link Edge} from a {@link PackedAtlas} * * @author matthieun */ public class PackedPoint extends Point { private static final long serialVersionUID = -7143958478767647582L; private final long index; protected PackedPoint(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public long getIdentifier() { return packedAtlas().pointIdentifier(this.index); } @Override public Location getLocation() { return packedAtlas().pointLocation(this.index); } @Override public Map getTags() { return packedAtlas().pointTags(this.index); } @Override public Set relations() { return packedAtlas().pointRelations(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) this.getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedRelation.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import org.locationtech.jts.geom.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMemberList; /** * @author matthieun */ public class PackedRelation extends Relation { private static final long serialVersionUID = -1912941368972318403L; private final long index; protected PackedRelation(final PackedAtlas atlas, final long index) { super(atlas); this.index = index; } @Override public RelationMemberList allKnownOsmMembers() { return packedAtlas().relationAllKnownOsmMembers(this.index); } @Override public List allRelationsWithSameOsmIdentifier() { return packedAtlas().relationAllRelationsWithSameOsmIdentifier(this.index); } @Override public Optional asMultiPolygon() { // return the previously stored result if (this.getBadGeom() || this.getGeom() != null) { return Optional.ofNullable(this.getGeom()); } MultiPolygon relationGeometry = null; if (packedAtlas().containsEnhancedRelationGeometry()) { relationGeometry = packedAtlas().relationGeometry(this.index); this.setGeom(relationGeometry); } if (relationGeometry == null) { return super.asMultiPolygon(); } return Optional.ofNullable(relationGeometry); } @Override public long getIdentifier() { return packedAtlas().relationIdentifier(this.index); } @Override public Map getTags() { return packedAtlas().relationTags(this.index); } @Override public RelationMemberList members() { return packedAtlas().relationMembers(this.index); } @Override public Long osmRelationIdentifier() { return packedAtlas().relationOsmIdentifier(this.index); } @Override public Set relations() { return packedAtlas().relationRelations(this.index); } private PackedAtlas packedAtlas() { return (PackedAtlas) getAtlas(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedTagStore.java ================================================ package org.openstreetmap.atlas.geography.atlas.packed; import java.io.Serializable; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoPackedTagStoreAdapter; import org.openstreetmap.atlas.utilities.arrays.Arrays; import org.openstreetmap.atlas.utilities.arrays.IntegerArrayOfArrays; import org.openstreetmap.atlas.utilities.compression.IntegerDictionary; /** * Store OSM Key-Value pairs, relying on the sub-class to provide Dictionaries. This allows for * sharing dictionaries if necessary. The key/value storage is in arrays to minimize space, which * assumes each item will have a reasonably small number of key-value pairs. * * @author matthieun * @author lcram */ public class PackedTagStore implements Serializable, ProtoSerializable { // Keep track of the field names for reflection code in the ProtoAdapter public static final String FIELD_KEYS = "keys"; public static final String FIELD_VALUES = "values"; public static final String FIELD_INDEX = "index"; private static final long serialVersionUID = -5240324410665237846L; private final IntegerArrayOfArrays keys; private final IntegerArrayOfArrays values; private transient IntegerDictionary dictionary; private long index = 0L; public PackedTagStore() { this.keys = null; this.values = null; this.dictionary = null; } public PackedTagStore(final long maximumSize, final int memoryBlockSize, final int subArraySize, final IntegerDictionary dictionary) { this.keys = new IntegerArrayOfArrays(maximumSize, memoryBlockSize, subArraySize); this.values = new IntegerArrayOfArrays(maximumSize, memoryBlockSize, subArraySize); this.dictionary = dictionary; } /** * Add a key/value pair at the specified index * * @param index * The index * @param key * The key * @param value * The value */ public void add(final long index, final String key, final String value) { if (index > size()) { throw new CoreException("Cannot add. Invalid index {} is bigger than the size {}", index, size()); } final int keyIndex = keysDictionary().add(key); final int valueIndex = valuesDictionary().add(value); final int[] keyArray; final int[] valueArray; if (index == this.index) { // We are adding a new row if (key == null || value == null) { keyArray = new int[0]; valueArray = new int[0]; } else { keyArray = new int[1]; valueArray = new int[1]; keyArray[0] = keyIndex; valueArray[0] = valueIndex; } this.keys.add(keyArray); this.values.add(valueArray); this.index++; } else { // We are adding a key/value pair to an existing item if (key == null || value == null) { // Do not add anything return; } keyArray = Arrays.addNewItem(this.keys.get(index), keyIndex); valueArray = Arrays.addNewItem(this.values.get(index), valueIndex); this.keys.set(index, keyArray); this.values.set(index, valueArray); } } /** * @param index * The index to check for * @param key * The key to test the presence of * @return True if the key is present at the specified index */ public boolean containsKey(final long index, final String key) { if (key == null) { throw new CoreException("Cannot test if a null key is contained"); } final int[] keyArray = this.keys.get(index); for (final int keyIndex : keyArray) { if (key.equals(keysDictionary().word(keyIndex))) { return true; } } return false; } @Override public boolean equals(final Object other) { if (other instanceof PackedTagStore) { if (this == other) { return true; } final PackedTagStore that = (PackedTagStore) other; if (!this.keys.equals(that.keys)) { return false; } if (!this.values.equals(that.values)) { return false; } if (!Objects.equals(this.keysDictionary(), that.keysDictionary())) { return false; } if (!Objects.equals(this.valuesDictionary(), that.valuesDictionary())) { return false; } return true; } return false; } /** * @param index * The index to check for * @param key * The key to get the value from * @return The value for the specified key at the specified index. Returns null if the key is * not present. */ public String get(final long index, final String key) { if (key == null) { throw new CoreException("Cannot get a null key's value"); } final int[] keyArray = this.keys.get(index); for (int i = 0; i < keyArray.length; i++) { final int keyIndex = keyArray[i]; if (key.equals(keysDictionary().word(keyIndex))) { final int valueIndex = this.values.get(index)[i]; return valuesDictionary().word(valueIndex); } } return null; } @Override public ProtoAdapter getProtoAdapter() { return new ProtoPackedTagStoreAdapter(); } @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; int hash = hashSeed * initialPrime + this.keys.hashCode(); hash = hashSeed * hash + this.values.hashCode(); final int keysDictionaryHash = this.keysDictionary() == null ? 0 : this.keysDictionary().hashCode(); final int valuesDictionaryHash = this.valuesDictionary() == null ? 0 : this.valuesDictionary().hashCode(); hash = hashSeed * hash + keysDictionaryHash; hash = hashSeed * hash + valuesDictionaryHash; return hash; } /** * @param index * The index to look for * @return All the keys at a specified index */ public Set keySet(final long index) { final Set result = new HashSet<>(); final int[] keyArray = this.keys.get(index); for (final int keyIndex : keyArray) { result.add(keysDictionary().word(keyIndex)); } return result; } /** * @param index * The index to look for * @return All the key/value pairs at a specified index */ public Map keyValuePairs(final long index) { if (null == this.keys || this.keys.isEmpty()) { // No tags return new HashMap<>(); } final int[] keyArray = this.keys.get(index); final int[] valueArray = this.values.get(index); final Map result = new HashMap<>(keyArray.length); for (int i = 0; i < keyArray.length; i++) { final int keyIndex = keyArray[i]; final int valueIndex = valueArray[i]; result.put(keysDictionary().word(keyIndex), valuesDictionary().word(valueIndex)); } return result; } /** * @return The dictionary for keys */ public IntegerDictionary keysDictionary() { return this.dictionary; } public void setDictionary(final IntegerDictionary dictionary) { this.dictionary = dictionary; } /** * @return The size of this tag store */ public long size() { return this.index; } public void trim() { this.keys.trim(); this.values.trim(); } /** * @return The dictionary for values */ public IntegerDictionary valuesDictionary() { return this.dictionary; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/packed/README.md ================================================ # `PackedAtlas` This is the before `Atlas`. It is immutable, and stores all its items in large arrays in memory. ## Array Storage All information is kept in large arrays in memory. Each array contains data for a specific type, with each index in the array always representing the same feature. For example, there is an array that contains the `Edge` start nodes indices, another one that contains the `Edge` end node indices. In each of those two arrays, the item at index 3 is always in reference to the same `Edge` with OSM identifier 123. Here are all the most important arrays in the PackedAtlas: * `dictionary` * `edgeIdentifiers` * `nodeIdentifiers` * `areaIdentifiers` * `lineIdentifiers` * `pointIdentifiers` * `relationIdentifiers` * `edgeIdentifierToEdgeArrayIndex` * `nodeIdentifierToNodeArrayIndex` * `areaIdentifierToAreaArrayIndex` * `lineIdentifierToLineArrayIndex` * `pointIdentifierToPointArrayIndex` * `relationIdentifierToRelationArrayIndex` * `nodeLocations` * `nodeInEdgesIndices` * `nodeOutEdgesIndices` * `nodeTags` * `nodeIndexToRelationIndices` * `edgeStartNodeIndex` * `edgeEndNodeIndex` * `edgePolyLines` * `edgeTags` * `edgeIndexToRelationIndices` * `areaPolygons` * `areaTags` * `areaIndexToRelationIndices` * `linePolyLines` * `lineTags` * `lineIndexToRelationIndices` * `pointLocations` * `pointTags` * `pointIndexToRelationIndices` * `relationMemberIndices` * `relationMemberTypes` * `relationMemberRoles` * `relationTags` * `relationIndexToRelationIndices` * `relationOsmIdentifierToRelationIdentifiers` * `relationOsmIdentifiers` ## Flyweight Atlas features All Atlas features are following the flyweight design pattern. What that means is every [`PackedEdge`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedEdge.java), [`PackedNode`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedNode.java), [`PackedArea`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedArea.java), [`PackedLine`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedLine.java), [`PackedPoint`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedPoint.java) or [`PackedRelation`](/src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedRelation.java) contains only two things: a reference to the Atlas object it belongs to, and the index it is positioned at in all the arrays in that Atlas. This makes the feature objects really lightweight and fast to create. When a user asks for the incoming edges to a `PackedNode`, then the `PackedNode` relays the query to its own `PackedAtlas` along with its index, and the `PackedAtlas` returns the result which is relayed to the user by the `PackedNode` object. ## Serialization / De-serialization A `PackedAtlas` can be serialized to a `WritableResource` using the `PackedAtlasSerializer`. During that process, all the arrays are pushed to a non-compressed zip stream in which each array is a zip entry with the same name. Each array is serialized in its zip entry using either standard Java serialization or protobuf. In case the `Resource` is a file, then the user has random access to each zip entry. That means that all the arrays are lazily de-serialized only when needed. This is extremely useful when opening an Atlas file just to check `Node` connectivity for example. Only the arrays relative with `Node`s and `Edge`s will be loaded. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/AtlasLoadingOption.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf; import java.io.Serializable; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.tags.filters.ConfiguredTaggableFilter; import org.openstreetmap.atlas.utilities.configuration.StandardConfiguration; /** * With this {@link AtlasLoadingOption} you can specify which feature you want to load to Atlas * * @author tony */ public final class AtlasLoadingOption implements Serializable { private static final long serialVersionUID = 1811691207451027561L; public static final String ATLAS_EDGE_FILTER_NAME = "atlas-edge"; public static final String ATLAS_AREA_FILTER_NAME = "atlas-area"; public static final String ATLAS_RELATION_SLICING_FILTER_NAME = "atlas-relation-slicing"; public static final String ATLAS_RELATION_SLICING_CONSOLIDATE_FILTER_NAME = "atlas-relation-slicing-consolidate"; public static final String ATLAS_WAY_SECTION_FILTER_NAME = "atlas-way-section"; private static final BridgeConfiguredFilter DEFAULT_EDGE_FILTER = new BridgeConfiguredFilter("", ATLAS_EDGE_FILTER_NAME, new StandardConfiguration(new InputStreamResource(() -> AtlasLoadingOption.class .getResourceAsStream(ATLAS_EDGE_FILTER_NAME + FileSuffix.JSON.toString())))); private static final BridgeConfiguredFilter DEFAULT_AREA_FILTER = new BridgeConfiguredFilter("", ATLAS_AREA_FILTER_NAME, new StandardConfiguration(new InputStreamResource(() -> AtlasLoadingOption.class .getResourceAsStream(ATLAS_AREA_FILTER_NAME + FileSuffix.JSON.toString())))); private static final BridgeConfiguredFilter DEFAULT_RELATION_SLICING_FILTER = new BridgeConfiguredFilter( "", ATLAS_RELATION_SLICING_FILTER_NAME, new StandardConfiguration( new InputStreamResource(() -> AtlasLoadingOption.class.getResourceAsStream( ATLAS_RELATION_SLICING_FILTER_NAME + FileSuffix.JSON.toString())))); private static final BridgeConfiguredFilter DEFAULT_RELATION_SLICING_CONSOLIDATE_FILTER = new BridgeConfiguredFilter( "", ATLAS_RELATION_SLICING_CONSOLIDATE_FILTER_NAME, new StandardConfiguration( new InputStreamResource(() -> AtlasLoadingOption.class.getResourceAsStream( ATLAS_RELATION_SLICING_FILTER_NAME + FileSuffix.JSON.toString())))); private static final BridgeConfiguredFilter DEFAULT_WAY_SECTION_FILTER = new BridgeConfiguredFilter( "", ATLAS_WAY_SECTION_FILTER_NAME, new StandardConfiguration( new InputStreamResource(() -> AtlasLoadingOption.class.getResourceAsStream( ATLAS_WAY_SECTION_FILTER_NAME + FileSuffix.JSON.toString())))); private static final ConfiguredTaggableFilter DEFAULT_OSM_PBF_WAY_FILTER = new ConfiguredTaggableFilter( new StandardConfiguration(new InputStreamResource( () -> AtlasLoadingOption.class.getResourceAsStream("osm-pbf-way.json")))); private static final ConfiguredTaggableFilter DEFAULT_OSM_PBF_NODE_FILTER = new ConfiguredTaggableFilter( new StandardConfiguration(new InputStreamResource( () -> AtlasLoadingOption.class.getResourceAsStream("osm-pbf-node.json")))); private static final ConfiguredTaggableFilter DEFAULT_OSM_PBF_RELATION_FILTER = new ConfiguredTaggableFilter( new StandardConfiguration(new InputStreamResource( () -> AtlasLoadingOption.class.getResourceAsStream("osm-pbf-relation.json")))); private boolean loadAtlasPoint; private boolean loadAtlasLine; private boolean loadAtlasArea; private boolean loadAtlasNode; private boolean loadAtlasEdge; private BridgeConfiguredFilter edgeFilter = DEFAULT_EDGE_FILTER; private BridgeConfiguredFilter areaFilter = DEFAULT_AREA_FILTER; private BridgeConfiguredFilter waySectionFilter = DEFAULT_WAY_SECTION_FILTER; private ConfiguredTaggableFilter osmPbfWayFilter = DEFAULT_OSM_PBF_WAY_FILTER; private ConfiguredTaggableFilter osmPbfNodeFilter = DEFAULT_OSM_PBF_NODE_FILTER; private ConfiguredTaggableFilter osmPbfRelationFilter = DEFAULT_OSM_PBF_RELATION_FILTER; private BridgeConfiguredFilter relationSlicingFilter = DEFAULT_RELATION_SLICING_FILTER; private BridgeConfiguredFilter relationSlicingConsolidateFilter = DEFAULT_RELATION_SLICING_CONSOLIDATE_FILTER; private boolean loadAtlasRelation; private boolean loadOsmBound; private boolean countrySlicing; /** Used to indicate that all objects should be kept */ private boolean keepAll; private boolean waySectioning; private boolean loadWaysSpanningCountryBoundaries; private String countryCode; private CountryBoundaryMap countryBoundaryMap; public static AtlasLoadingOption createOptionWithAllEnabled( final CountryBoundaryMap countryBoundaryMap) { final AtlasLoadingOption option = new AtlasLoadingOption(); option.setCountrySlicing(true); option.setWaySectioning(true); option.setCountryBoundaryMap(countryBoundaryMap); return option; } public static AtlasLoadingOption createOptionWithNoSlicing() { return new AtlasLoadingOption(); } public static AtlasLoadingOption createOptionWithOnlyNodesAndWayNoSlicing() { final AtlasLoadingOption option = new AtlasLoadingOption(); option.setLoadAtlasPoint(false); option.setLoadAtlasLine(false); option.setLoadAtlasArea(false); option.setLoadAtlasRelation(false); return option; } public static AtlasLoadingOption createOptionWithOnlyNodesAndWaysAndSlicing( final CountryBoundaryMap countryBoundaryMap) { final AtlasLoadingOption option = new AtlasLoadingOption(); option.setLoadAtlasPoint(false); option.setLoadAtlasLine(false); option.setLoadAtlasArea(false); option.setLoadAtlasRelation(false); option.setCountrySlicing(true); option.setWaySectioning(true); option.setCountryBoundaryMap(countryBoundaryMap); return option; } public static AtlasLoadingOption createOptionWithOnlySectioning() { final AtlasLoadingOption option = new AtlasLoadingOption(); option.setCountrySlicing(false); option.setWaySectioning(true); return option; } public static AtlasLoadingOption withNoFilter() { final StringResource pbfFilter = new StringResource("{\"filters\":[]}"); final ConfiguredTaggableFilter filter = new ConfiguredTaggableFilter( new StandardConfiguration(pbfFilter)); final AtlasLoadingOption atlasLoadingOption = new AtlasLoadingOption(); atlasLoadingOption.setOsmPbfWayFilter(filter); atlasLoadingOption.setOsmPbfNodeFilter(filter); atlasLoadingOption.setOsmPbfRelationFilter(filter); atlasLoadingOption.setWaySectioning(true); return atlasLoadingOption; } private AtlasLoadingOption() { this.loadAtlasPoint = true; this.loadAtlasNode = true; this.loadAtlasLine = true; this.loadAtlasEdge = true; this.loadAtlasArea = true; this.loadAtlasRelation = true; this.loadOsmBound = true; this.countrySlicing = false; this.keepAll = false; this.waySectioning = false; this.loadWaysSpanningCountryBoundaries = true; this.countryBoundaryMap = null; } public BridgeConfiguredFilter getAreaFilter() { return this.areaFilter; } public CountryBoundaryMap getCountryBoundaryMap() { return this.countryBoundaryMap; } public String getCountryCode() { return this.countryCode; } public BridgeConfiguredFilter getEdgeFilter() { return this.edgeFilter; } public ConfiguredTaggableFilter getOsmPbfNodeFilter() { return this.osmPbfNodeFilter; } public ConfiguredTaggableFilter getOsmPbfRelationFilter() { return this.osmPbfRelationFilter; } public ConfiguredTaggableFilter getOsmPbfWayFilter() { return this.osmPbfWayFilter; } public BridgeConfiguredFilter getRelationSlicingConsolidateFilter() { return this.relationSlicingConsolidateFilter; } public BridgeConfiguredFilter getRelationSlicingFilter() { return this.relationSlicingFilter; } public BridgeConfiguredFilter getWaySectionFilter() { return this.waySectionFilter; } public boolean isCountrySlicing() { return this.countrySlicing; } /** * Check to see if the atlas should not be filtered or deduplicated. This option takes * precedence over all filtering options. * * @return {@code true} if we should not drop any items */ public boolean isKeepAll() { return this.keepAll; } public boolean isLoadAtlasArea() { return this.loadAtlasArea; } public boolean isLoadAtlasEdge() { return this.loadAtlasEdge; } public boolean isLoadAtlasLine() { return this.loadAtlasLine; } public boolean isLoadAtlasNode() { return this.loadAtlasNode; } public boolean isLoadAtlasPoint() { return this.loadAtlasPoint; } public boolean isLoadAtlasRelation() { return this.loadAtlasRelation; } public boolean isLoadOsmBound() { return this.loadOsmBound; } public boolean isLoadOsmNode() { return isLoadAtlasNode() || isLoadAtlasPoint(); } public boolean isLoadOsmRelation() { return isLoadAtlasRelation(); } public boolean isLoadOsmWay() { return isLoadAtlasEdge() || isLoadAtlasLine(); } public boolean isLoadWaysSpanningCountryBoundaries() { return this.loadWaysSpanningCountryBoundaries; } public boolean isWaySectioning() { return this.waySectioning; } public void setAreaFilter(final BridgeConfiguredFilter areaFilter) { this.areaFilter = areaFilter; } public void setCountryBoundaryMap(final CountryBoundaryMap countryBoundaryMap) { this.countryBoundaryMap = countryBoundaryMap; } public AtlasLoadingOption setCountryCode(final String countryCode) { this.countryCode = countryCode; return this; } public AtlasLoadingOption setCountrySlicing(final boolean isCountrySlicing) { this.countrySlicing = isCountrySlicing; return this; } public void setEdgeFilter(final BridgeConfiguredFilter edgeFilter) { this.edgeFilter = edgeFilter; } /** * Set whether or not all objects should be kept, regardless of filters. * * @param isKeepAll * {@code true} to keep all objects * @return {@code this}, for easy chaining */ public AtlasLoadingOption setKeepAll(final boolean isKeepAll) { this.keepAll = isKeepAll; return this; } public AtlasLoadingOption setLoadAtlasArea(final boolean isLoadAtlasArea) { this.loadAtlasArea = isLoadAtlasArea; return this; } public AtlasLoadingOption setLoadAtlasEdge(final boolean isLoadAtlasEdge) { this.loadAtlasEdge = isLoadAtlasEdge; return this; } public AtlasLoadingOption setLoadAtlasLine(final boolean isLoadAtlasLine) { this.loadAtlasLine = isLoadAtlasLine; return this; } public AtlasLoadingOption setLoadAtlasNode(final boolean isLoadAtlasNode) { this.loadAtlasNode = isLoadAtlasNode; return this; } public AtlasLoadingOption setLoadAtlasPoint(final boolean isLoadAtlasPoint) { this.loadAtlasPoint = isLoadAtlasPoint; return this; } public AtlasLoadingOption setLoadAtlasRelation(final boolean isLoadAtlasRelation) { this.loadAtlasRelation = isLoadAtlasRelation; return this; } public AtlasLoadingOption setLoadOsmBound(final boolean isLoadOsmBound) { this.loadOsmBound = isLoadOsmBound; return this; } public AtlasLoadingOption setLoadWaysSpanningCountryBoundaries( final boolean loadWaysSpanningCountryBoundaries) { this.loadWaysSpanningCountryBoundaries = loadWaysSpanningCountryBoundaries; return this; } public void setOsmPbfNodeFilter(final ConfiguredTaggableFilter osmPbfNodeFilter) { this.osmPbfNodeFilter = osmPbfNodeFilter; } public void setOsmPbfRelationFilter(final ConfiguredTaggableFilter osmPbfRelationFilter) { this.osmPbfRelationFilter = osmPbfRelationFilter; } public void setOsmPbfWayFilter(final ConfiguredTaggableFilter osmPbfWayFilter) { this.osmPbfWayFilter = osmPbfWayFilter; } public void setRelationSlicingConsolidateFilter( final BridgeConfiguredFilter relationSlicingConsolidateFilter) { this.relationSlicingConsolidateFilter = relationSlicingConsolidateFilter; } public void setRelationSlicingFilter(final BridgeConfiguredFilter relationSlicingFilter) { this.relationSlicingFilter = relationSlicingFilter; } public void setWaySectionFilter(final BridgeConfiguredFilter waySectionFilter) { this.waySectionFilter = waySectionFilter; } public AtlasLoadingOption setWaySectioning(final boolean isWaySectioning) { this.waySectioning = isWaySectioning; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/BridgeConfiguredFilter.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf; import java.io.Serializable; import java.util.List; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.ConfiguredTaggableFilter; import org.openstreetmap.atlas.utilities.configuration.Configuration; import org.openstreetmap.atlas.utilities.configuration.ConfigurationReader; import org.openstreetmap.atlas.utilities.configuration.ConfiguredFilter; import com.google.common.collect.Lists; /** * This class is a bridge between {@link ConfiguredTaggableFilter} and {@link ConfiguredFilter} for * use in {@link AtlasLoadingOption}, as a backwards-compatible option until * {@link ConfiguredFilter} is the only option used. * * @author matthieun */ public class BridgeConfiguredFilter implements Predicate, Serializable { private static final long serialVersionUID = -1496420126649881929L; private static final String EMPTY_MARKER = "N/A"; private ConfiguredTaggableFilter configuredTaggableFilter; private ConfiguredFilter configuredFilter; public BridgeConfiguredFilter(final String root, final String name, final Configuration configuration) { final ConfigurationReader reader = new ConfigurationReader(""); final List configuredTaggableFilterFilters = reader.configurationValues( configuration, ConfiguredTaggableFilter.FILTERS_CONFIGURATION_NAME, Lists.newArrayList(EMPTY_MARKER)); if (configuredTaggableFilterFilters.size() == 1 && EMPTY_MARKER.equals(configuredTaggableFilterFilters.get(0))) { // It is a new ConfiguredFilter this.configuredFilter = ConfiguredFilter.from(root, name, configuration); } else { // It is a legacy ConfiguredTaggableFilter this.configuredTaggableFilter = new ConfiguredTaggableFilter(configuration); } } @Override public boolean test(final AtlasEntity atlasEntity) { return this.configuredFilter != null ? this.configuredFilter.test(atlasEntity) : this.configuredTaggableFilter.test(atlasEntity); } public boolean test(final Taggable taggable) { return this.configuredFilter != null ? this.configuredFilter.test(taggable) : this.configuredTaggableFilter.test(taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/CloseableOsmosisReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import crosby.binary.osmosis.OsmosisReader; /** * {@link Closeable} version of an {@link OsmosisReader} that prevents {@link InputStream} leaks. * * @author matthieun */ public class CloseableOsmosisReader extends OsmosisReader implements Closeable { private final InputStream inputStream; public CloseableOsmosisReader(final InputStream input) { super(input); this.inputStream = input; } @Override public void close() throws IOException { this.inputStream.close(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/converters/TagMapToTagCollectionConverter.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.converters; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.utilities.conversion.Converter; import org.openstreetmap.osmosis.core.domain.v0_6.Tag; /** * @author matthieun */ public class TagMapToTagCollectionConverter implements Converter, Collection> { @Override public Collection convert(final Map object) { final List result = new ArrayList<>(); object.forEach((key, value) -> result.add(new Tag(key, value))); return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/AbstractIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; import org.openstreetmap.atlas.exception.CoreException; /** * Identifier management like create new way, node and relation identifier * * @author tony */ public abstract class AbstractIdentifierFactory { public static final long IDENTIFIER_SCALE_DEFAULT = 1000; private final long[] referenceIdentifiers; private int index; private long delta; private final long identifierScale; public AbstractIdentifierFactory(final long referenceIdentifier) { this(new long[] { referenceIdentifier }); } public AbstractIdentifierFactory(final long referenceIdentifier, final long identifierScale) { this(new long[] { referenceIdentifier }, identifierScale); } public AbstractIdentifierFactory(final long[] referenceIdentifierArray) { this.referenceIdentifiers = referenceIdentifierArray; this.delta = 0; this.index = 0; this.identifierScale = IDENTIFIER_SCALE_DEFAULT; } public AbstractIdentifierFactory(final long[] referenceIdentifierArray, final long identifierScale) { this.referenceIdentifiers = referenceIdentifierArray; this.delta = 0; this.index = 0; this.identifierScale = identifierScale; } public long getDelta() { return this.delta; } public long getIdentifierScale() { return this.identifierScale; } public long getReferenceIdentifier() { return this.referenceIdentifiers[this.index]; } public boolean hasMore() { return this.delta < this.identifierScale - 1 || this.index < this.referenceIdentifiers.length - 1; } public abstract long nextIdentifier(); protected void incrementDelta() { this.delta++; if (this.delta >= this.identifierScale) { if (this.index < this.referenceIdentifiers.length - 1) { this.index++; this.delta = 1; } else { throw new CoreException("Entity {} has been split into more than 999 pieces", this.referenceIdentifiers); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/CountrySlicingIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; /** * Identifier factory for country slicing * * @author tony */ public class CountrySlicingIdentifierFactory extends AbstractIdentifierFactory { public CountrySlicingIdentifierFactory(final long referenceIdentifier) { super(referenceIdentifier); } public CountrySlicingIdentifierFactory(final long[] referenceIdentifiers) { super(referenceIdentifiers); } @Override public long nextIdentifier() { incrementDelta(); return (getReferenceIdentifier() / IDENTIFIER_SCALE_DEFAULT + getDelta()) * IDENTIFIER_SCALE_DEFAULT; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/PaddingIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; /** * IdentifierFactory for padding only * * @author tony */ public final class PaddingIdentifierFactory { private static final long IDENTIFIER_PADDING = 1_000_000; public static long pad(final long identifier) { return identifier * IDENTIFIER_PADDING; } private PaddingIdentifierFactory() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/PointIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; /** * Identifier factory for points (mainly used for synthetic boundary nodes) * * @author samg */ public class PointIdentifierFactory extends AbstractIdentifierFactory { private static final long IDENFITIER_SCALE = 1000000; public PointIdentifierFactory(final long referenceIdentifier) { super(referenceIdentifier, IDENFITIER_SCALE); } @Override public long nextIdentifier() { incrementDelta(); return getReferenceIdentifier() + getDelta(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/ReverseIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; /** * Common methods to extract information from an Atlas identifier. Atlas identifier is defined as * {OSM identifier}{3 digits for country slicing id}{3 digits for way-section id}. * * @author matthieun */ public class ReverseIdentifierFactory { /** * Returns the country code for the given identifier. Example: 222222001003 returns 1. * * @param countryCodeAndWaySectionedIdentifier * The full atlas identifier * @return the country code for given identifier */ public long getCountryCode(final long countryCodeAndWaySectionedIdentifier) { return countryCodeAndWaySectionedIdentifier / AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT % AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT; } /** * Returns the OSM identifier and country portion for the given identifier. This truncates the * identifier to get rid of the way-sectioning piece. Example: 222222001003 returns 222222001. * * @param countryCodeAndWaySectionedIdentifier * The full atlas identifier * @return the identifier without the way-section portion for given identifier */ public long getCountryOsmIdentifier(final long countryCodeAndWaySectionedIdentifier) { return Math.abs(countryCodeAndWaySectionedIdentifier / AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT); } /** * Returns the OSM identifier padded by the 6 digits allocated for country-slicing and * way-sectioning. Example: 222222001003 returns 222222000000. * * @param countryCodeAndWaySectionedIdentifier * The full atlas identifier * @return the OSM identifier padded by the 6 digits allocated for country-slicing and * way-sectioning */ public long getFirstAtlasIdentifier(final long countryCodeAndWaySectionedIdentifier) { return Math.abs(getOsmIdentifier(countryCodeAndWaySectionedIdentifier) * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT); } /** * Returns the OSM identifier for the given identifier. This truncates the identifier to get rid * of the country slicing and way-sectioning pieces. Example: 222222001003 returns 222222. * * @param countryCodeAndWaySectionedIdentifier * The full atlas identifier * @return the OSM identifier for the given identifier */ public long getOsmIdentifier(final long countryCodeAndWaySectionedIdentifier) { return Math.abs(countryCodeAndWaySectionedIdentifier / (AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT)); } /** * Given the identifier that has the OSM identifier and country slicing portion, this will * return the first complete Atlas identifier, including the way-section index. Example: * 222222001 returns 222222001000. * * @param countryOsmIdentifier * The identifier with OSM identifier and country-slicing pieces * @return the first complete Atlas identifier, including the way-section index */ public long getStartIdentifier(final long countryOsmIdentifier) { return countryOsmIdentifier * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT; } /** * Similar to {@link #getStartIdentifier(long)}, this takes the OSM identifier and the country * code as two separate inputs and returns the first complete Atlas identifier, including the * way-section index. Example: 222222 and 001 returns 222222001000. * * @param osmIdentifier * The OSM identifier * @param countryCode * The country-code identifier * @return the first complete Atlas identifier, including the way-section index */ public long getStartIdentifier(final long osmIdentifier, final long countryCode) { return (osmIdentifier * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT + countryCode) * AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT; } /** * Returns the way-section index for the given identifier. Example: 222222001003 returns 3. * * @param countryCodeAndWaySectionedIdentifier * The full atlas identifier * @return the way-section index for given identifier */ public long getWaySectionIndex(final long countryCodeAndWaySectionedIdentifier) { return countryCodeAndWaySectionedIdentifier % AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/pbf/slicing/identifier/WaySectionIdentifierFactory.java ================================================ package org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier; /** * Identifier factory for way section * * @author tony */ public class WaySectionIdentifierFactory extends AbstractIdentifierFactory { public WaySectionIdentifierFactory(final long referenceIdentifier) { super(referenceIdentifier); } @Override public long nextIdentifier() { incrementDelta(); return getReferenceIdentifier() + getDelta(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation/OsmPbfCounter.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.creation; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.TagMap; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.RouteTag; import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer; import org.openstreetmap.osmosis.core.domain.v0_6.Bound; import org.openstreetmap.osmosis.core.domain.v0_6.Entity; import org.openstreetmap.osmosis.core.domain.v0_6.EntityType; import org.openstreetmap.osmosis.core.domain.v0_6.Node; import org.openstreetmap.osmosis.core.domain.v0_6.Relation; import org.openstreetmap.osmosis.core.domain.v0_6.RelationMember; import org.openstreetmap.osmosis.core.domain.v0_6.Way; import org.openstreetmap.osmosis.core.domain.v0_6.WayNode; import org.openstreetmap.osmosis.core.task.v0_6.Sink; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link OsmPbfCounter} is responsible for identifying and counting the number of * {@link Point}s, {@link Line}s and {@link org.openstreetmap.atlas.geography.atlas.items.Relation}s * that will be brought in from the given OSM PBF file. This information will be used to populate * the {@link AtlasSize} field to efficiently construct a Raw {@link Atlas}. The logic for * determining whether a feature should be brought in is as follows: * *

    *
  • Look at each OSM Node, if it's inside the given bounding box, bring it in. Keep track of all * nodes that were not brought in. *
  • Look at each OSM Way, if it has a Node that was brought in, bring in the Way and all other * Nodes that are part of this Way (since they may not have been originally brought in). *
  • Look at each OSM Relation at a shallow level - if it has a member (Node, Way or Relation) * that was brought in, go ahead and bring this Relation in. At this stage, we don't look at nested * relations. If we can't find a shallow member, stage this relation. *
  • Look at all staged relations and try to bring any in. If we loop through all staged relations * without bringing anything in, we can conclude that all remaining relations are either fully * outside the given boundary or are looping over each other. *
* * @author mgostintsev */ public class OsmPbfCounter implements Sink { private static final Logger logger = LoggerFactory.getLogger(OsmPbfCounter.class); private static final int MAXIMUM_NETWORK_EXTENSION = 2; private final AtlasLoadingOption loadingOption; private final GeometricSurface boundingBox; // Identifiers to bring in to the raw atlas private final Set nodeIdentifiersToInclude = new HashSet<>(); private final Set nodeIdentifiersBroughtInByWaysOrRelations = new HashSet<>(); private final Set wayIdentifiersToInclude = new HashSet<>(); private final Set relationIdentifiersToInclude = new HashSet<>(); // Keep track of included nodes so that they can be used in calculating if a way intersects the // given shard private final Map nodeIdentifierToLocation = new HashMap<>(); // Keep track of excluded ways to see if we need to add them later private final Map waysToExclude = new HashMap<>(); // Keep track of non-shallow relations to see if we need to add them later private final List stagedRelations = new ArrayList<>(); /** * Default constructor. * * @param loadingOption * The {@link AtlasLoadingOption} to use * @param boundingBox * The bounding box to consider when including features in the raw atlas */ public OsmPbfCounter(final AtlasLoadingOption loadingOption, final GeometricSurface boundingBox) { this.loadingOption = loadingOption; this.boundingBox = boundingBox; } @Override public void close() { // Process all staged Relations processStagedRelations(); // Grab any bridges, ferries or other ways that may be outside the immediate boundary bringInConnectedOutsideWays(); // Combine all included nodes into a single collection this.nodeIdentifiersToInclude.addAll(this.nodeIdentifiersBroughtInByWaysOrRelations); logger.info("Released OSM PBF Counter"); } @Override public void complete() { // No-Op } /** * @return the set of all OSM Node identifiers to include in the raw Atlas. */ public Set getIncludedNodeIdentifiers() { return this.nodeIdentifiersToInclude; } /** * @return the set of all OSM Way identifiers to include in the raw Atlas. */ public Set getIncludedWayIdentifiers() { return this.wayIdentifiersToInclude; } @Override public void initialize(final Map metaData) { logger.info("Initialized OSM PBF Counter successfully"); } /** * @return the number of {@link Line} objects found */ public long lineCount() { return this.wayIdentifiersToInclude.size(); } /** * @return the number of {@link Point} objects found */ public long pointCount() { return this.nodeIdentifiersToInclude.size(); } @Override public void process(final EntityContainer entityContainer) { final Entity rawEntity = entityContainer.getEntity(); if (OsmPbfReader.shouldProcessEntity(this.loadingOption, rawEntity)) { // store all node locations for calculating way geometry if (rawEntity instanceof Node) { final Node node = (Node) rawEntity; final Location nodeLocation = new Location(Latitude.degrees(node.getLatitude()), Longitude.degrees(node.getLongitude())); this.nodeIdentifierToLocation.put(rawEntity.getId(), nodeLocation); } if (shouldLoadOsmNode(rawEntity)) { // This node is within the boundary or we are using the generated atlas file for QA // purposes, bring it in this.nodeIdentifiersToInclude.add(rawEntity.getId()); } else if (shouldLoadOsmWay(rawEntity)) { final Way way = (Way) rawEntity; if (wayIntersectsBoundary(way)) { // This way contains at least one shape-point within the given bounding box. // Bring it and all of its nodes in to the atlas. addWayNodes(this.nodeIdentifiersBroughtInByWaysOrRelations, way); this.wayIdentifiersToInclude.add(way.getId()); } else { // This way doesn't have any shape-points within the given boundary. Mark it as // a way to exclude so it can be revisited during relation processing this.waysToExclude.put(way.getId(), way); } } else if (shouldLoadOsmRelation(rawEntity)) { final Relation relation = (Relation) rawEntity; if (relationContainsMemberWithinBoundary(relation)) { // Shallow check showed that this relation has a member that is inside our // boundary, mark all members and relation as inside markRelationAndMembersInsideBoundary(relation); } else { // Stage the relation - it might be added later this.stagedRelations.add(relation); } } else if (rawEntity instanceof Bound) { logger.trace("Encountered PBF Bound {}, skipping over it.", rawEntity.getId()); } } } /** * @return the number of {@link org.openstreetmap.atlas.geography.atlas.items.Relation} objects * found. */ public long relationCount() { return this.relationIdentifiersToInclude.size(); } private void addWayNodes(final Set set, final Way way) { way.getWayNodes().forEach(wayNode -> set.add(wayNode.getNodeId())); } /** * Sometimes there are OSM ways (bridges, ferry routes, etc.) that are connected to the road * network, but are outside the working country boundary. This method will expand up to * {@link #MAXIMUM_NETWORK_EXTENSION} connections past our existing boundary nodes and pull in * any ways that meet the above criteria. The while loop terminates if we've exceeded the * maximum number of extensions or if we haven't added anything during the previous iteration. */ private void bringInConnectedOutsideWays() { if (this.loadingOption.isLoadWaysSpanningCountryBoundaries()) { int extensionCounter = 0; final Set alreadyAddedWays = new HashSet<>(); final AtomicBoolean addedNewEdge = new AtomicBoolean(true); while (extensionCounter < MAXIMUM_NETWORK_EXTENSION && addedNewEdge.get()) { logger.trace("Adding connected ways outside boundary pass {}", extensionCounter); addedNewEdge.set(false); this.waysToExclude.values().stream().filter(this::isHighwayOrFerry) .filter(way -> !alreadyAddedWays.contains(way.getId())).forEach(way -> { final List wayNodes = way.getWayNodes(); for (final WayNode wayNode : wayNodes) { final long identifier = wayNode.getNodeId(); if (this.nodeIdentifiersBroughtInByWaysOrRelations .contains(identifier)) { // Add way and its members logger.trace("Adding connected way with identifier {}", way.getId()); this.wayIdentifiersToInclude.add(way.getId()); addWayNodes(this.nodeIdentifiersBroughtInByWaysOrRelations, way); addedNewEdge.set(true); alreadyAddedWays.add(way.getId()); break; } } }); extensionCounter++; } } } private boolean isHighwayOrFerry(final Way way) { final TagMap taggableWay = new TagMap(way.getTags()); return HighwayTag.isCoreWay(taggableWay) || RouteTag.isFerry(taggableWay); } private void markRelationAndMembersInsideBoundary(final Relation relation) { // Add all the members for (final RelationMember member : relation.getMembers()) { final EntityType memberType = member.getMemberType(); final Long memberIdentifier = member.getMemberId(); if (memberType == EntityType.Node) { this.nodeIdentifiersToInclude.add(memberIdentifier); } else if (memberType == EntityType.Way) { this.wayIdentifiersToInclude.add(memberIdentifier); // If this line was originally excluded, bring it in now final Way toAdd = this.waysToExclude.get(memberIdentifier); if (toAdd != null) { addWayNodes(this.nodeIdentifiersBroughtInByWaysOrRelations, toAdd); this.waysToExclude.remove(memberIdentifier, toAdd); } } else if (memberType == EntityType.Relation) { this.relationIdentifiersToInclude.add(member.getMemberId()); } } // Add the relation itself this.relationIdentifiersToInclude.add(relation.getId()); } private boolean nodeWithinTargetBoundary(final Node node) { return this.boundingBox.fullyGeometricallyEncloses(new Location( Latitude.degrees(node.getLatitude()), Longitude.degrees(node.getLongitude()))); } private void processStagedRelations() { List stagedRelations = this.stagedRelations; int currentStagedRelationSize = this.stagedRelations.size(); int previousStagedRelationSize = 0; // Loop through all staged relations and see if adding any of the staged relations trigger a // new relation to be added. Once we have a full cycle where no new relation has been marked // as inside the boundary, we can safely conclude that all remaining relations have members // outside of the boundary while (!this.stagedRelations.isEmpty() && currentStagedRelationSize != previousStagedRelationSize) { final List updatedStagedRelations = new ArrayList<>(); for (final Relation relation : stagedRelations) { if (relationContainsMemberWithinBoundary(relation)) { markRelationAndMembersInsideBoundary(relation); } else { updatedStagedRelations.add(relation); } } stagedRelations = updatedStagedRelations; previousStagedRelationSize = currentStagedRelationSize; currentStagedRelationSize = stagedRelations.size(); } } private boolean relationContainsMemberWithinBoundary(final Relation relation) { // This relation has already been marked as inside if (this.relationIdentifiersToInclude.contains(relation.getId())) { return true; } // Do a shallow check to see if any members hit for (final RelationMember member : relation.getMembers()) { final EntityType memberType = member.getMemberType(); if (memberType == EntityType.Node && this.nodeIdentifiersToInclude.contains(member.getMemberId())) { return true; } else if (memberType == EntityType.Way && this.wayIdentifiersToInclude.contains(member.getMemberId())) { return true; } else if (memberType == EntityType.Relation && this.relationIdentifiersToInclude.contains(member.getMemberId())) { return true; } } // We've looped through every member (excluding nested relations) and found no match return false; } private boolean shouldLoadOsmNode(final Entity entity) { // For QA purposes, it is necessary to keep nodes that are outside the target boundary. // For example, atlas-checks needs to know all the node ids in order to reverse a way and // then create an // osmChange file for MapRoulette. return this.loadingOption.isLoadOsmNode() && entity instanceof Node && (nodeWithinTargetBoundary((Node) entity) || this.loadingOption.isKeepAll()); } private boolean shouldLoadOsmRelation(final Entity entity) { return this.loadingOption.isLoadOsmRelation() && entity instanceof Relation; } private boolean shouldLoadOsmWay(final Entity entity) { return this.loadingOption.isLoadOsmWay() && entity instanceof Way; } private boolean wayIntersectsBoundary(final Way way) { // CASE 1: Line crosses (or is enclosed by) the shard bounds and has at least one shapepoint // within the shard bounds ArrayList wayNodesLocations = new ArrayList<>(); for (final WayNode node : way.getWayNodes()) { // nodes are processed first so allNodes will contain all node locations wayNodesLocations.add(this.nodeIdentifierToLocation.get(node.getNodeId())); if (this.nodeIdentifiersToInclude.contains(node.getNodeId())) { this.wayIdentifiersToInclude.add(way.getId()); return true; } } // CASE 2: Line crossed the shard but has no shapepoints within it, so we must check for // intersections wayNodesLocations = wayNodesLocations.stream().filter(node -> node != null) .collect(Collectors.toCollection(ArrayList::new)); if (wayNodesLocations.isEmpty()) { return false; } final PolyLine wayGeometry = new PolyLine(wayNodesLocations); if (wayGeometry.isPoint() || wayGeometry.isEmpty() || wayGeometry.bounds() == null) { return false; } // Checking the bounds of the polyline instead of the actual geometry may include some // extraneous lines, but is much more performant. The extra lines will be filtered out after // the slicing process if (this.boundingBox.bounds().overlaps(wayGeometry.bounds())) { this.wayIdentifiersToInclude.add(way.getId()); return true; } // If we reach here, the way doesn't have a node anywhere inside (or on) the given boundary return false; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation/OsmPbfReader.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.creation; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.PaddingIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.TagMap; import org.openstreetmap.atlas.tags.AtlasTag; import org.openstreetmap.atlas.tags.LastEditChangesetTag; import org.openstreetmap.atlas.tags.LastEditTimeTag; import org.openstreetmap.atlas.tags.LastEditUserIdentifierTag; import org.openstreetmap.atlas.tags.LastEditUserNameTag; import org.openstreetmap.atlas.tags.LastEditVersionTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer; import org.openstreetmap.osmosis.core.domain.v0_6.Bound; import org.openstreetmap.osmosis.core.domain.v0_6.Entity; import org.openstreetmap.osmosis.core.domain.v0_6.EntityType; import org.openstreetmap.osmosis.core.domain.v0_6.Node; import org.openstreetmap.osmosis.core.domain.v0_6.Relation; import org.openstreetmap.osmosis.core.domain.v0_6.RelationMember; import org.openstreetmap.osmosis.core.domain.v0_6.Tag; import org.openstreetmap.osmosis.core.domain.v0_6.Way; import org.openstreetmap.osmosis.core.domain.v0_6.WayNode; import org.openstreetmap.osmosis.core.task.v0_6.Sink; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link OsmPbfReader} is responsible for reading OSM PBF entities and converting them to Atlas * Entities. It will map a PBF {@link Node} to an Atlas {@link Point}, a PBF {@link Way} to an Atlas * {@link Line} and a PBF {@link Relation} to an Atlas * {@link org.openstreetmap.atlas.geography.atlas.items.Relation}. Any exclusions based on tags are * allowed via the passed in {@link AtlasLoadingOption} configuration. There is no support for * Atlas @ {@link org.openstreetmap.atlas.geography.atlas.items.Node}s, {@link Edge}s or * {@link Area}s. This is also assuming that the Osmosis --completeWays flag is used during PBF * creation. * * @author mgostintsev */ public class OsmPbfReader implements Sink { private static final Logger logger = LoggerFactory.getLogger(OsmPbfReader.class); private static final String MISSING_MEMBER_MESSAGE = "Relation {} contains {} {} as a member, but it's either filtered out or this PBF shard does not contain it."; private static final int MINIMUM_CLOSED_WAY_LENGTH = 4; private final PackedAtlasBuilder builder; private final AtlasLoadingOption loadingOption; private final Set nodeIdentifiersToInclude = new HashSet<>(); private final Set wayIdentifiersToInclude = new HashSet<>(); private final Set pointIdentifiersFromFilteredLines = new HashSet<>(); private final List stagedRelations = new ArrayList<>(); private final RawAtlasStatistic statistics = new RawAtlasStatistic(logger); /** * Determines if the given {@link Entity} should be brought into the {@link Atlas}. Ideally, all * features will be brought in. However, to make {@link Atlas} generation flexible and fit all * use cases, this is configurable. * * @param loadingOption * The {@link AtlasLoadingOption} to use for configuration lookup. * @param entity * The candidate {@link Entity} * @return {@code true} if this {@link Entity} should be brought into the {@link Atlas}. */ public static boolean shouldProcessEntity(final AtlasLoadingOption loadingOption, final Entity entity) { // The keepAll option is primarily used for QA purposes. Everything must stay. if (loadingOption.isKeepAll()) { return true; } else if (entity instanceof Node) { return loadingOption.getOsmPbfNodeFilter().test(Taggable.with(entity.getTags())); } else if (entity instanceof Way) { return loadingOption.getOsmPbfWayFilter().test(Taggable.with(entity.getTags())); } else if (entity instanceof Relation) { return loadingOption.getOsmPbfRelationFilter().test(Taggable.with(entity.getTags())); } else { // No Bound filtering return true; } } /** * Default constructor * * @param loadingOption * The {@link AtlasLoadingOption} to use * @param builder * The {@link PackedAtlasBuilder} to construct the raw Atlas */ public OsmPbfReader(final AtlasLoadingOption loadingOption, final PackedAtlasBuilder builder) { this.builder = builder; this.loadingOption = loadingOption; } @Override public void close() { // We've processed all Nodes, Ways and shallow Relations to this point. Now, we need to // handle Relations that contain Relation members properly. this.processStagedRelations(); this.statistics.summary(); logger.info("Released OSM PBF Reader"); } @Override public void complete() { // No-op } @Override public void initialize(final Map metaData) { logger.info("Initialized OSM PBF Reader successfully"); } @Override public void process(final EntityContainer entityContainer) { final Entity rawEntity = entityContainer.getEntity(); if (shouldProcessEntity(this.loadingOption, rawEntity)) { if (rawEntity instanceof Node && this.nodeIdentifiersToInclude.contains(rawEntity.getId())) { processNode(rawEntity); } else if (rawEntity instanceof Way && this.wayIdentifiersToInclude.contains(rawEntity.getId())) { processWay(rawEntity); } else if (this.loadingOption.isLoadOsmRelation() && rawEntity instanceof Relation) { processRelation(rawEntity); } else if (rawEntity instanceof Bound) { logger.trace("Encountered PBF Bound {}, skipping over it.", rawEntity.getId()); } } else { recordNodeIdentifiersFromFilteredEntity(rawEntity); logFilteredStatistics(rawEntity); logger.trace("Filtering out OSM {} {} from Raw Atlas", rawEntity.getType(), rawEntity.getId()); } } /** * Sets all the Node identifiers marked for inclusion. * * @param nodesToInclude * The set of Node identifiers to include in the atlas */ public void setIncludedNodes(final Set nodesToInclude) { this.nodeIdentifiersToInclude.addAll(nodesToInclude); } /** * Sets all the Way identifiers marked for inclusion. * * @param waysToInclude * The set of Way identifiers to include in the atlas */ public void setIncludedWays(final Set waysToInclude) { this.wayIdentifiersToInclude.addAll(waysToInclude); } /** * @return all the {@link Point} identifiers that make up {@link Line}s that were filtered. We * process all PBF {@link Node}s first and add them as Atlas {@link Point}s. After this, * we may filter out some PBF {@link Way}s. As a result, our Atlas file may contain * points which aren't used by any lines. We want to record these and see if we can * filter them out downstream. */ protected Set getPointIdentifiersFromFilteredLines() { return this.pointIdentifiersFromFilteredLines; } /** * Constructs an Atlas {@link org.openstreetmap.atlas.geography.atlas.items.Relation}. In the * process, a relation can be dropped if it doesn't contain any members. The members could be * empty, if they were filtered out by the PBF ingest criteria or if this PBF file doesn't * contain them. * * @param relation * The {@link Relation} to add */ private void addRelation(final Relation relation) { final RelationBean bean = constructRelationBean(relation); if (!bean.isEmpty()) { this.builder.addRelation(padIdentifier(relation.getId()), relation.getId(), bean, populateEntityTags(relation).getTags()); this.statistics.recordCreatedRelation(); } else { this.statistics.recordDroppedRelation(); logger.debug("Cannot add empty Relation {} to the Atlas. We're either filtering" + " out the members that make up the Relation or none of the " + "members are present in this PBF shard.", relation.getId()); } } /** * Creates a {@link RelationBean} for the given {@link Relation}. Note: The returned bean can be * empty. * * @param relation * The {@link Relation} for which to create the {@link RelationBean} * @return the created {@link RelationBean} */ private RelationBean constructRelationBean(final Relation relation) { final RelationBean bean = new RelationBean(); for (final RelationMember member : relation.getMembers()) { final long memberIdentifier = padIdentifier(member.getMemberId()); final EntityType memberType = member.getMemberType(); final String role = member.getMemberRole(); if (memberType == EntityType.Node) { if (this.builder.peek().point(memberIdentifier) != null) { bean.addItem(memberIdentifier, role, ItemType.POINT); } else { logger.trace(MISSING_MEMBER_MESSAGE, relation.getId(), EntityType.Node, memberIdentifier); } } else if (memberType == EntityType.Way) { if (this.builder.peek().line(memberIdentifier) != null) { bean.addItem(memberIdentifier, role, ItemType.LINE); } else if (this.builder.peek().area(memberIdentifier) != null) { bean.addItem(memberIdentifier, role, ItemType.AREA); } else { logger.trace(MISSING_MEMBER_MESSAGE, relation.getId(), EntityType.Way, memberIdentifier); } } else if (memberType == EntityType.Relation) { if (this.builder.peek().relation(memberIdentifier) != null) { bean.addItem(memberIdentifier, role, ItemType.RELATION); } else { logger.trace(MISSING_MEMBER_MESSAGE, relation.getId(), EntityType.Relation, memberIdentifier); } } else { // A Bound should never be a Relation member; if this is the case, log and continue logger.error("Invalid Bound member {} found for Relation {}", member.getMemberId(), relation.getId()); } } return bean; } private Polygon constructWayPolygon(final Way way) { final List wayNodes = way.getWayNodes(); final List wayLocations = new ArrayList<>(); wayNodes.forEach( node -> wayLocations.add(getNodeLocation(padIdentifier(node.getNodeId())))); wayLocations.remove(wayLocations.size() - 1); return new Polygon(wayLocations); } /** * Constructs a {@link PolyLine} given an OSM PBF {@link Way}. The {@link Way} object doesn't * contain the coordinates of its geometry, only the references to the {@link Node}s * identifiers. We need to look up the {@link Location} of each {@link Node} and re-construct * the {@link PolyLine} manually. * * @param way * The {@link Way} for which to construct the {@link PolyLine} * @return the constructed {@link PolyLine} */ private PolyLine constructWayPolyline(final Way way) { final List wayNodes = way.getWayNodes(); final List wayLocations = new ArrayList<>(); wayNodes.forEach( node -> wayLocations.add(getNodeLocation(padIdentifier(node.getNodeId())))); return new PolyLine(wayLocations); } /** * Checks if the given {@link Relation} contains an un-indexed member {@link Relation}. * * @param relation * The {@link Relation} to check * @return {@code true} if the given {@link Relation} contains a member {@link Relation} that * hasn't yet been indexed. */ private boolean containsUnindexedSubRelation(final Relation relation) { return relation.getMembers().stream() .anyMatch(member -> member.getMemberType() == EntityType.Relation && this.builder .peek().relation(padIdentifier(member.getMemberId())) == null); } /** * Looks up the {@link Location} of a {@link Node}, given its identifier. This identifier is * used to look up the corresponding Atlas {@link Point}. * * @param identifier * The {@link Node} identifier, in whose {@link Location} we're interested in. * @return the extracted {@link Location} */ private Location getNodeLocation(final long identifier) { final Point point = this.builder.peek().point(identifier); if (point != null) { return point.getLocation(); } else { throw new CoreException( "Unable to find Point {} in Atlas. " + "It was either filtered out or the PBF file is malformed.", identifier); } } /** * A {@link Way} is invalid if it's of size 0 or 1; or if it's of size 2 and has the same start * and end node. * * @param way * The {@link Way} to validate * @return {@code true} if the given {@link Way} is invalid */ private boolean isInvalidWay(final Way way) { final List wayNodes = way.getWayNodes(); return wayNodes.size() < 2 || wayNodes.size() == 2 && wayNodes.get(0).getNodeId() == wayNodes.get(1).getNodeId() || wayNodes.size() < MINIMUM_CLOSED_WAY_LENGTH && getNodeLocation( padIdentifier(wayNodes.get(0).getNodeId())) .equals(getNodeLocation( padIdentifier(wayNodes.get(wayNodes.size() - 1).getNodeId()))); } /** * Log any {@link Entity}s that got filtered by ingest configuration. * * @param entity * The filtered {@link Entity} */ private void logFilteredStatistics(final Entity entity) { if (entity instanceof Node) { this.statistics.recordFilteredNode(); } else if (entity instanceof Way) { this.statistics.recordFilteredWay(); } else if (entity instanceof Relation) { this.statistics.recordFilteredRelation(); } else { // No-Op. We don't log bounds. } } /** * @return {@code true} if we need to pad identifiers when creating atlas entities. */ private boolean needsPadding() { return this.loadingOption.isCountrySlicing() || this.loadingOption.isWaySectioning(); } /** * Pads the given OSM identifier, by appending 6 digits to it. The first 3 appended digits are * the country code identifier and the last 3 digits are the way-section identifier. * * @param identifier * The original OSM identifier * @return a padded identifier */ private long padIdentifier(final long identifier) { if (needsPadding()) { return PaddingIdentifierFactory.pad(identifier); } else { return identifier; } } /** * First, creates an {@link Entity} {@link Tag} for specific OSM attributes we're interested in * propagating to the {@link AtlasEntity}. Secondly, converts the given {@link Entity}'s * collection of {@link Tag}s to a {@link Map} of key/value pairs used to build an * {@link AtlasEntity}'s tag set. * * @param entity * The {@link Entity} being processed * @return a {@link Map} of key/value tags */ private TagMap populateEntityTags(final Entity entity) { // Update the entity's tags to contain specific OSM attributes we care about, so that these // get translated to Atlas Entity tags. storeOsmEntityAttributesAsTags(entity); return new TagMap(entity.getTags()); } /** * Uses the {@link Node} OSM identifier, geometry and tags to create an Atlas {@link Point}. * * @param entity * The {@link Entity} that will become an Atlas {@link Point} */ private void processNode(final Entity entity) { final Node node = (Node) entity; this.builder.addPoint(padIdentifier(node.getId()), new Location(Latitude.degrees(node.getLatitude()), Longitude.degrees(node.getLongitude())), populateEntityTags(node).getTags()); this.statistics.recordCreatedPoint(); } /** * Tries to create an Atlas {@link org.openstreetmap.atlas.geography.atlas.items.Relation}. If * the {@link Entity} contains a member that's also a relation and that member hasn't been * processed yet, then we add the given {@link Relation} to a Collection of staged relations to * process later (see {@link #close()} method). Otherwise, we add it. * * @param entity * The {@link Entity} that will become an Atlas * {@link org.openstreetmap.atlas.geography.atlas.items.Relation} */ private void processRelation(final Entity entity) { final Relation relation = (Relation) entity; if (containsUnindexedSubRelation(relation)) { // Stage this Relation, it has a member relation that we haven't processed yet this.stagedRelations.add(relation); } else { addRelation(relation); } } /** * Processes all non-shallow {@link Relation}s - any {@link Relation} that has another * {@link Relation} as a member. This is called near the end of the processing, once we've * successfully added all {@link Node}s, {@link Way}s and shallow {@link Relation}s. If the * number of staged relations doesn't change from one iteration to the next, then we know that * any future iteration will not un-stage any of the relations. Therefore, we can try to add the * relations in the current state. */ private void processStagedRelations() { int previousStagedRelationSize = Integer.MAX_VALUE; List stagedRelations = this.stagedRelations; int currentStagedRelationSize = stagedRelations.size(); while (!stagedRelations.isEmpty() && currentStagedRelationSize < previousStagedRelationSize) { final List updatedStagedRelations = new ArrayList<>(); for (final Relation relation : stagedRelations) { if (containsUnindexedSubRelation(relation)) { updatedStagedRelations.add(relation); } else { addRelation(relation); } } stagedRelations = updatedStagedRelations; previousStagedRelationSize = currentStagedRelationSize; currentStagedRelationSize = stagedRelations.size(); } // If we weren't able to process any of the staged relations and there are still some left, // go ahead and try to add these in their current state. We can potentially add a relation // that has an incomplete member list, however we are relying on the fact that a neighboring // shard will contain this relation and the missing members. When the two are combined, the // relation will become whole again. if (currentStagedRelationSize == previousStagedRelationSize && !stagedRelations.isEmpty()) { boolean changesMade = false; while (!stagedRelations.isEmpty()) { final List staging = new ArrayList<>(); staging.addAll(stagedRelations); for (final Relation relation : stagedRelations) { if (relation.getMembers().stream() .anyMatch(member -> member.getMemberType().equals(EntityType.Relation) && staging.stream().anyMatch( staged -> staged.getId() == member.getMemberId()))) { logger.error("Relation {} had staging member {}", relation.getId(), relation .getMembers().stream() .filter(member -> member.getMemberType().equals(EntityType.Relation) && staging.stream().anyMatch( staged -> staged.getId() == member.getMemberId())) .collect(Collectors.toSet())); } else { addRelation(relation); staging.remove(relation); changesMade = true; } } stagedRelations = staging; if (!changesMade) { logger.error( "No changes found for staged relation loop. Staged relations were {}", stagedRelations); logger.error("Just in case, first relation was {}", stagedRelations.iterator().next()); break; } changesMade = false; } if (!stagedRelations.isEmpty()) { final List stagedRelationsButFinal = stagedRelations; final SortedSet sortedByNumberOfParents = new TreeSet<>((r1, r2) -> { final int r1Parents = r1.getMembers().stream() .filter(member1 -> member1.getMemberType().equals(EntityType.Relation) && stagedRelationsButFinal.stream().anyMatch( staged1 -> staged1.getId() == member1.getMemberId())) .collect(Collectors.toSet()).size(); final int r2Parents = r2.getMembers().stream() .filter(member2 -> member2.getMemberType().equals(EntityType.Relation) && stagedRelationsButFinal.stream().anyMatch( staged2 -> staged2.getId() == member2.getMemberId())) .collect(Collectors.toSet()).size(); if (r1Parents < r2Parents) { return 1; } else if (r1Parents > r2Parents) { return -1; } else { return Long.compare(r1.getId(), r2.getId()); } }); stagedRelations.forEach(sortedByNumberOfParents::add); sortedByNumberOfParents.forEach(this::addRelation); } } } /** * Uses the {@link Way} identifier, re-constructed geometry and tags to create an Atlas * {@link Line}. * * @param entity * The {@link Entity} that will become an Atlas {@link Line} */ private void processWay(final Entity entity) { final Way way = (Way) entity; if (isInvalidWay(way)) { this.statistics.recordDroppedWay(); } else { final PolyLine wayPolyLine = constructWayPolyline(way); final TagMap wayTags = populateEntityTags(way); if (wayPolyLine.first().equals(wayPolyLine.last())) { boolean kept = false; if (this.loadingOption.getAreaFilter().test(wayTags)) { this.builder.addArea(padIdentifier(way.getId()), constructWayPolygon(way), wayTags.getTags()); this.statistics.recordCreatedLine(); kept = true; } if (this.loadingOption.getEdgeFilter().test(wayTags)) { this.builder.addLine(padIdentifier(way.getId()), wayPolyLine, wayTags.getTags()); this.statistics.recordCreatedLine(); kept = true; } if (!kept) { this.statistics.recordDroppedWay(); } } else { this.builder.addLine(padIdentifier(way.getId()), wayPolyLine, wayTags.getTags()); this.statistics.recordCreatedLine(); } } } /** * Because of how an Atlas is constructed, we add all PBF {@link Node}s as {@link Point}s. * However, if we filter out a PBF {@link Way}, we want to keep track of all {@link Node}s that * make up that {@link Way}, in case we need to remove them as well. We are going to keep track * of all filtered PBF {@link Node}/Atlas {@link Point} identifiers, and use subAtlas * functionality to filter them out after the Atlas is built. For now, ignore filtering any * Nodes that come from filtered Relations. This will be handled in the way-sectioning code. * * @param entity * The {@link Entity} whose Node (Atlas Point) identifiers we want to filter out */ private void recordNodeIdentifiersFromFilteredEntity(final Entity entity) { if (entity instanceof Way) { final List wayNodes = ((Way) entity).getWayNodes(); wayNodes.forEach(node -> { this.pointIdentifiersFromFilteredLines.add(padIdentifier(node.getNodeId())); this.statistics.recordFilteredNode(); }); } } /** * Grabs desired OSM attributes (such as last edited time) from given {@link Entity} and creates * a corresponding {@link Tag} value for each. * * @param entity * The {@link Entity}, whose attributes we want to save. */ private void storeOsmEntityAttributesAsTags(final Entity entity) { final Collection tags = entity.getTags(); for (final String tag : AtlasTag.TAGS_FROM_OSM) { if (tag.equals(LastEditTimeTag.KEY)) { tags.add(new Tag(tag, String.valueOf(entity.getTimestamp().getTime()))); } else if (tag.equals(LastEditUserIdentifierTag.KEY)) { tags.add(new Tag(tag, String.valueOf(entity.getUser().getId()))); } else if (tag.equals(LastEditUserNameTag.KEY)) { tags.add(new Tag(tag, entity.getUser().getName())); } else if (tag.equals(LastEditVersionTag.KEY)) { tags.add(new Tag(tag, String.valueOf(entity.getVersion()))); } else if (tag.equals(LastEditChangesetTag.KEY)) { tags.add(new Tag(tag, String.valueOf(entity.getChangesetId()))); } else { logger.error( "Trying to add mandatory tag {}, but no behavior defined for getting the value.", tag); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation/README.md ================================================ # `Raw Atlas Creation` ## Overview This package is responsible for the initial OSM PBF file to Atlas file transformation. The result is an intermediate Atlas file, that is not country-sliced or way-sectioned. The advantage to this is that we get to leverage the Atlas API layer and most importantly, the spatial index to make slicing and sectioning as straight-forward as possible. ## Raw Atlas Terminology ### Stages The raw Atlas flow is comprised of three separate stages. They're explained in detail below: 1. Raw Atlas - this is the first Atlas type that is created. It's a direct representation of the PBF data in Atlas format. This Atlas contains only Points, Lines, Areas, and Relations. It does **not** contain country code assignment or way-sectioning. 2. Sliced Atlas - the sliced Atlas is made up of Points, Lines, Areas, and Relations with each one containing one or more country code for which country it belongs to. For details on this process, see the Slicing package README file. 3. Sectioned Atlas - this is the final sliced and sectioned artifact, which is made up of all Atlas entities - Points, Nodes, Edges, Lines, Areas and Relations and is fully ready for processing. ### General Principles There are a few general principles that are observed throughout the ingest process. 1. The desired goal is to achieve parity with OSM and get as accurate a representation as possible - including ingesting bad data. The reasoning for this is to be able to leverage the Atlas and write [atlas-checks](https://github.com/osmlab/atlas-checks) to detect bad data, which will lead to OSM data fixes. 2. Do all complex processing once. If we have access to all the data during the ingest piece and if we're building MultiAtlases to country-slice and section the road network correctly, bundle all similar processing together to save any downstream users the hassle of having to re-build MutliAtlases or make inferences based on impartial Atlas views. 3. Rely on synthetic tags. Instead of silently fixing bad data or making data-altering decisions - rely on synthetic tags where possible to identify specific cases that may require additional attention or custom atlas-checks. ## Raw Atlas Implementation Details The PBF ingestion process is configuration driven. This means that the user has full control of what features end up in the final Atlas. The default configuration, found in the [pbf resource package](https://github.com/osmlab/atlas/tree/dev/src/main/resources/org/openstreetmap/atlas/geography/atlas/pbf), attempts full parity with OSM - meaning it ingests almost all features. There are a couple of implementation details to call out for the raw Atlas creation. The input protobuf file is created using the [Osmosis library](https://github.com/openstreetmap/osmosis) and it's structured with a distinct order - the file contains the Nodes first, then the Ways and lastly the Relations. Each Way references the Node identifiers that are used to construct itself. This is something problematic, because we have no Node location or tag properties of the individual Nodes when processing each Way. To solve this, we must either create a Node map or make two passes over the PBF file - once to read the ways and a second time to read the Nodes for the Ways we're interested in. It's a lot faster to read the file twice, rather than resize the underlying `PackedAtlas` arrays during build time. In the actual implementation, the `OsmPbfCounter` class is responsible for identifying what to bring in, keeping track of relevant Nodes and counts. The `OsmPbfReader` will then go through the file a second time and build the raw Atlas using the information from the counter. ## Synthetic Tags The raw Atlas creation process add a new synthetic tag as part of the final Atlas - `SyntheticDuplicateOsmNodeTag`. This tag signifies that the input OSM data contains two or more stacked Nodes at the same Location. This is almost always a data error, that we are handling graciously and deterministically. The Node with the lowest identifier is kept, while all others are excluded from the final Atlas. There are two caveats to call out here. The first caveat is that if there are Nodes with different `LayerTag` values at the same `Location`, we will keep the lowest occurring Node for each layer in order to preserve proper connectivity. The second caveat is that we can potentially remove a Node that has rich tagging and keep the Node that has no tagging. This is a potential problem, but the only way to ensure deterministic processing. Ideally, the presence of this synthetic tag will prompt the creation of an atlas-check that will result in data fixes for such cases. ## Raw Atlas Ingestion Logic The specific logic for what gets ingested into the raw Atlas is as follows: 1. For each OSM Node, if it's inside the `Shard` boundary that's being processed, bring it in. As a side note, we keep track of all Nodes that were not brought in, in case we have to pull them in later. 2. For each OSM Way, if the Way has a Node that was brought in, bring in the entire Way and all other Nodes that are part of this Way (since they may not have been originally brought in as they could be outside the target `Shard` boundary). 3. For each OSM Relation, process Relations in a queue structure - where we only look at Relations that have no member Relations or have had their member Relations already processed. For each Relation, if it contains a member that was brought into the Atlas (Node, Way or Relation) then bring in this Relation in to the Atlas. ## Code Sample There are a few ways to create raw Atlas files. Each one is outlined here. The first and most simple one is to build a raw Atlas from some PBF resource, irrespective of `Shard` boundary or any kind of `AtlasLoadingOption`: ```java final String pbfPath = "/path/to/pbf/resource"; final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(pbfPath)); final Atlas rawAtlas = rawAtlasGenerator.build(); ``` The second way is to build a raw Atlas given a PBF resource and a specific boundary of interest: ```java final String pbfPath = "/path/to/pbf/resource"; // The boundary of interest final MultiPolygon boundary; final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(pbfPath), boundary); final Atlas rawAtlas = rawAtlasGenerator.build(); ``` Lastly, we can specify all three fields of interest - the PBF resource, the specific `AtlasLoadingOption` and boundary of interest: ```java final String pbfPath = "/path/to/pbf/resource"; // The boundary of interest final MultiPolygon boundary; // Country boundary shapes backed by a spatial index final CountryBoundaryMap countryBoundaryMap; final AtlasLoadingOption loadingOption = AtlasLoadingOption.createOptionWithAllEnabled(countryBoundaryMap); final RawAtlasGenerator generator = new RawAtlasGenerator(new File(pbfPath), loadingOption, boundary); final Atlas rawAtlas = rawAtlasGenerator.build(); ``` ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation/RawAtlasGenerator.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.creation; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.CloseableOsmosisReader; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.tags.AtlasTag; import org.openstreetmap.atlas.tags.LayerTag; import org.openstreetmap.atlas.tags.SyntheticDuplicateOsmNodeTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.time.Time; import org.openstreetmap.osmosis.core.task.v0_6.Sink; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link RawAtlasGenerator} loads an OSM protobuf file and constructs a raw {@link Atlas} from * it. A raw {@link Atlas} will only contains Atlas {@link Point}s, {@link Line}s and * {@link Relation}s. The protobuf file is structured in a specific way - there is a distinct order: * the file first contains Nodes, then Ways and lastly Relations. Each Way only references the * identifier, it doesn't contain any location or tag properties of the Nodes used to construct * itself. In order to process them - we can either create a Node map or read the file twice. We * also want to identify all features that will be part of the Atlas before building it so we can * create an accurate {@link AtlasSize}. It is a lot faster to read the file twice than resize the * underlying {@link PackedAtlas} arrays during build time. In our implementation, the * {@link OsmPbfCounter} is responsible for identifying and counting what to pull in by going * through the PBF file once. The {@link OsmPbfReader} will go through it a second time and build * the raw Atlas using the information obtained from the counter. * * @author mgostintsev */ public class RawAtlasGenerator { private static final Logger logger = LoggerFactory.getLogger(RawAtlasGenerator.class); // Used to identify and count all entities pulled into the Atlas private final OsmPbfCounter pbfCounter; // Used to build the raw Atlas given the information from the OsmPbfCounter private final OsmPbfReader pbfReader; // The target bounding box. Anything outside of this will be discarded. private final MultiPolygon boundingBox; // Builder to build raw Atlas private final PackedAtlasBuilder builder; // Any configurations needed private final AtlasLoadingOption atlasLoadingOption; // Osmosis supplier private final Supplier osmosisReaderSupplier; // Raw atlas metadata private AtlasMetaData metaData = new AtlasMetaData(); /** * Constructor that supplies the maximum bounds possible as the bounding box. * * @param resource * The OSM PBF {@link Resource} to use */ public RawAtlasGenerator(final Resource resource) { this(resource, AtlasLoadingOption.createOptionWithOnlySectioning(), MultiPolygon.MAXIMUM); } /** * Default constructor. * * @param resource * The OSM PBF {@link Resource} to use * @param loadingOption * The {@link AtlasLoadingOption} to use * @param boundingBox * The bounding box to consider when including features in the raw atlas */ public RawAtlasGenerator(final Resource resource, final AtlasLoadingOption loadingOption, final MultiPolygon boundingBox) { this(() -> new CloseableOsmosisReader(resource.read()), loadingOption, boundingBox); } /** * Constructor that uses the default configuration with a given bounding box. * * @param resource * The OSM PBF {@link Resource} to use * @param boundingBox * The bounding box to consider when including features in the raw atlas */ public RawAtlasGenerator(final Resource resource, final MultiPolygon boundingBox) { this(resource, AtlasLoadingOption.createOptionWithNoSlicing(), boundingBox); } public RawAtlasGenerator(final Supplier osmosisReaderSupplier, final AtlasLoadingOption atlasLoadingOption, final MultiPolygon boundingBox) { this.osmosisReaderSupplier = osmosisReaderSupplier; this.atlasLoadingOption = atlasLoadingOption; this.boundingBox = boundingBox; this.builder = new PackedAtlasBuilder(); this.pbfReader = new OsmPbfReader(atlasLoadingOption, this.builder); this.pbfCounter = new OsmPbfCounter(atlasLoadingOption, this.boundingBox); } /** * Loops through the PBF file once to gather the entity counts. Updates the * {@link AtlasMetaData} and {@link AtlasSize}, then proceeds to loop through the PBF file a * second time to build the raw {@link Atlas}. * * @return the raw {@link Atlas}, can be {@code null}. */ public Atlas build() { prepareBuild(); // Second pass -- loop through the PBF file again. This time, read the entities and // construct a raw Atlas. return buildRawAtlas(); } /** * Works the same way as build() above, but doesn't trim duplicate and extraneous points from * the atlas. This is used as a faster way to build the atlas when verifying the validity of PBF * files. * * @return the raw {@link Atlas}, can be {@code null} or filled with duplicate points */ public Atlas buildNoTrim() { prepareBuild(); // Second pass -- loop through the PBF file again. This time, read the entities and // construct a raw Atlas. return buildRawAtlasNoTrim(); } /** * Save raw {@link Atlas} as geoJson. * * @param resource * The {@link WritableResource} to save to. */ public void saveAsGeojson(final WritableResource resource) { logger.info("Saving Raw Atlas as geojson"); build().saveAsGeoJson(resource); } /** * Save raw {@link Atlas} as text. * * @param resource * The {@link WritableResource} to save to. */ public void saveAsText(final WritableResource resource) { logger.info("Saving Raw Atlas as text"); build().saveAsText(resource); } /** * Save raw {@link Atlas}. * * @param resource * The {@link WritableResource} to save to. */ public void saveAtlas(final WritableResource resource) { logger.info("Saving Raw Atlas file"); build().save(resource); } /** * Use given {@link AtlasMetaData} object * * @param metaData * {@link AtlasMetaData} to use * @return the updated {@link RawAtlasGenerator} */ public RawAtlasGenerator withMetaData(final AtlasMetaData metaData) { this.metaData = metaData; return this; } /** * Loops through the given OSM PBF file and builds the raw {@link Atlas}. * * @return the raw {@link Atlas}, possibly {@code null} if no {@link Atlas} was built. */ private Atlas buildRawAtlas() { final String shardName = this.metaData.getShardName().orElse("unknown"); final Atlas atlas = buildRawAtlasNoTrim(); if (atlas == null) { logger.info("Generated empty raw Atlas for PBF Shard {}", shardName); return atlas; } else { final Time trimTime = Time.now(); final Atlas trimmedAtlas; if (this.atlasLoadingOption.isKeepAll()) { trimmedAtlas = atlas; } else { trimmedAtlas = removeDuplicateAndExtraneousPointsFromAtlas(atlas); } logger.info("Trimmed Raw Atlas for {} in {}", shardName, trimTime.elapsedSince()); if (trimmedAtlas == null) { logger.info("Empty raw Atlas after filtering for PBF Shard {}", shardName); } return trimmedAtlas; } } private Atlas buildRawAtlasNoTrim() { final String shardName = this.metaData.getShardName().orElse("unknown"); final Time parseTime = Time.now(); try (CloseableOsmosisReader reader = connectOsmPbfToPbfConsumer(this.pbfReader)) { reader.run(); } catch (final Exception e) { throw new CoreException("Atlas creation error for PBF shard {}", shardName, e); } logger.info("Read PBF for {} in {}", shardName, parseTime.elapsedSince()); final Time buildTime = Time.now(); final Atlas atlas = this.builder.get(); logger.info("Built Raw Atlas for {} in {}", shardName, buildTime.elapsedSince()); return atlas; } /** * Connects the given {@link Sink} implementation to the PBF File. */ private CloseableOsmosisReader connectOsmPbfToPbfConsumer(final Sink consumer) { final CloseableOsmosisReader reader = this.osmosisReaderSupplier.get(); reader.setSink(consumer); return reader; } /** * Loops through the given OSM PBF file and count the all the {@link Point}s, {@link Line}s and * {@link Relation}s. These will be used to initialize the {@link AtlasSize} to efficiently * build the raw {@link Atlas}. */ private void countOsmPbfEntities() { final Time countTime = Time.now(); try (CloseableOsmosisReader counter = connectOsmPbfToPbfConsumer(this.pbfCounter)) { counter.run(); } catch (final Exception e) { throw new CoreException("Error counting PBF entities", e); } logger.info("Counted PBF Entities in {}", countTime.elapsedSince()); } private long getLayerTagValueForPoint(final Atlas atlas, final long identifier) { return LayerTag.getTaggedOrImpliedValue(atlas.point(identifier), 0L); } /** * Check if the {@link Point} with the given identifier is a {@link Relation} member in the * given {@link Atlas}. * * @param atlas * The {@link Atlas} to check * @param pointIdentifier * The {@link Point} identifier to use * @return {@code true} if the given {@link Point} identifier is a {@link Relation} member in * the given {@link Atlas} */ private boolean isRelationMember(final Atlas atlas, final long pointIdentifier) { return !atlas.point(pointIdentifier).relations().isEmpty(); } /** * Check if the {@link Point} with the given identifier is a shape point for some {@link Line} * in the given {@link Atlas}. * * @param atlas * The {@link Atlas} to check * @param pointIdentifier * The {@link Point} identifier to use * @return {@code true} if the given {@link Point} identifier is a shape point for some * {@link Line} in the given {@link Atlas} */ private boolean isShapePoint(final Atlas atlas, final long pointIdentifier) { return Iterables .size(atlas.linesContaining(atlas.point(pointIdentifier).getLocation())) > 0; } /** * A simple point is one that only has the mandatory entity tags. See * {@link AtlasTag#TAGS_FROM_OSM} for the 5 tags. Examples of non-simple points include stop * lights, barriers, etc. * * @param atlas * The {@link Atlas} to check * @param pointIdentifier * The {@link Point} identifier to use * @return {@code true} if the given identifier represents a simple {@link Point} */ private boolean isSimplePoint(final Atlas atlas, final long pointIdentifier) { return atlas.point(pointIdentifier).getTags().size() == AtlasTag.TAGS_FROM_OSM.size(); } private boolean locationPartOfMultipleWaysWithDifferentLayerTags(final Atlas atlas, final Location location) { final long distinctLayerTagValues = StreamSupport .stream(atlas.linesContaining(location).spliterator(), false) .map(line -> LayerTag.getTaggedOrImpliedValue(atlas.line(line.getIdentifier()), 0L)) .distinct().count(); return distinctLayerTagValues > 1; } /** * Populates the {@link AtlasMetaData} used to build the raw {@link Atlas}. Specifically, * records any {@link Node}, {@link Way} and {@link Relation} filtering that may have been used. * This may also populate other options that were passed in the {@link AtlasLoadingOption} * parameter at construction. */ private void populateAtlasMetadata() { // AtlasMetaData returns a new HashMap with every getTags() call. // We therefore want to operate on a "copy" of the original tags, // and create a new metadata object. final Map originalTags = this.metaData.getTags(); originalTags.put(AtlasMetaData.OSM_PBF_NODE_CONFIGURATION, this.atlasLoadingOption.getOsmPbfNodeFilter().toString()); originalTags.put(AtlasMetaData.OSM_PBF_WAY_CONFIGURATION, this.atlasLoadingOption.getOsmPbfWayFilter().toString()); originalTags.put(AtlasMetaData.OSM_PBF_RELATION_CONFIGURATION, this.atlasLoadingOption.getOsmPbfRelationFilter().toString()); originalTags.put(AtlasMetaData.KEEP_ALL_CONFIGURATION, Boolean.toString(this.atlasLoadingOption.isKeepAll())); // While AtlasMetaData returns a new map by default, this accounts for the possibility that // it may change in the future. if (!originalTags.equals(this.metaData.getTags())) { this.metaData = this.metaData.copyWithNewTags(originalTags); } this.builder.setMetaData(this.metaData); } /** * Get the set of {@link Point}s that make up all the filtered PBF {@link Way}s and see if we * can remove them from the generated raw Atlas. Criteria for removal are: *
    *
  • The {@link Point} has to be simple. This avoids removing non-shape point features. *
  • The {@link Point} cannot be a {@link Relation} member. *
  • The {@link Point} cannot be a shape point for an existing {@link Line}. *
* * @param atlas * The {@link Atlas} being filtered from * @return the {@link Set} of {@link Point} identifiers that are safe to filter out */ private Set preFilterPointsToRemove(final Atlas atlas) { return this.pbfReader.getPointIdentifiersFromFilteredLines().stream() .filter(identifier -> atlas.point(identifier) != null) .filter(identifier -> isSimplePoint(atlas, identifier)) .filter(identifier -> !isRelationMember(atlas, identifier)) .filter(identifier -> !isShapePoint(atlas, identifier)).collect(Collectors.toSet()); } private void prepareBuild() { countOsmPbfEntities(); // Update the metadata to reflect any configuration that was used and use count results to // set the AtlasSize estimate. populateAtlasMetadata(); setAtlasSizeEstimate(); this.builder.withEnhancedRelationGeometry(); // Update the reader to be aware of any included nodes/ways to avoid repeated calculations this.pbfReader.setIncludedNodes(this.pbfCounter.getIncludedNodeIdentifiers()); this.pbfReader.setIncludedWays(this.pbfCounter.getIncludedWayIdentifiers()); } private Atlas rebuildAtlas(final Atlas atlas, final Set pointsToRemove, final Set pointsNeedingSyntheticTag) { final PackedAtlasBuilder rebuilder = new PackedAtlasBuilder(); // Set the metadata and size. Use existing Atlas as estimate. rebuilder.setMetaData(this.metaData); final AtlasSize size = new AtlasSize(0, 0, atlas.numberOfAreas(), atlas.numberOfLines(), atlas.numberOfPoints(), atlas.numberOfRelations()); rebuilder.setSizeEstimates(size); // Add Points atlas.points().forEach(point -> { final long identifier = point.getIdentifier(); // Only add if this point isn't being removed if (!pointsToRemove.contains(identifier)) { final Map tags = point.getTags(); if (pointsNeedingSyntheticTag.contains(identifier)) { // Add the synthetic tag tags.put(SyntheticDuplicateOsmNodeTag.KEY, SyntheticDuplicateOsmNodeTag.YES.toString()); } // Add the Point rebuilder.addPoint(identifier, point.getLocation(), tags); } }); // Add Lines atlas.lines().forEach( line -> rebuilder.addLine(line.getIdentifier(), line.asPolyLine(), line.getTags())); // Add Lines atlas.areas().forEach( area -> rebuilder.addArea(area.getIdentifier(), area.asPolygon(), area.getTags())); // Add Relations // Keep a set of all relations that have members that have been removed, so if that member // is the only member, we do not add the parent relation either. final Set relationsToCheckForRemoval = new HashSet<>(); atlas.relationsLowerOrderFirst().forEach(relation -> { final RelationBean bean = new RelationBean(); relation.members().forEach(member -> { final AtlasEntity entity = member.getEntity(); final long memberIdentifier = entity.getIdentifier(); if (entity.getType() == ItemType.POINT && pointsToRemove.contains(memberIdentifier)) { // Make sure not to add any removed points logger.debug( "Excluding point {} from relation {} since point was removed from Atlas", memberIdentifier, relation.getIdentifier()); } else if (entity.getType() == ItemType.RELATION && relationsToCheckForRemoval.contains(memberIdentifier)) { // Make sure not to add any removed relations logger.debug( "Excluding relation member {} from parent relation {} since that relation member became empty", memberIdentifier, relation.getIdentifier()); } else { bean.addItem(memberIdentifier, member.getRole(), entity.getType()); } }); if (!bean.isEmpty()) { rebuilder.addRelation(relation.getIdentifier(), relation.getOsmIdentifier(), bean, relation.getTags()); } else { final long relationIdentifier = relation.getIdentifier(); logger.debug("Relation {} bean is empty, dropping from Atlas", relationIdentifier); relationsToCheckForRemoval.add(relationIdentifier); } }); // Build and return the new Atlas return rebuilder.get(); } /** * We may need to remove {@link Point}s from the built raw Atlas. There are two scenarios for * removal: *

*

    *
  • 1. A {@link Point} was a shape point for an OSM {@link Way} that was removed. This point * doesn't have any tags, isn't a part of a {@link Relation} and doesn't intersect with any * other features in the Atlas. *
  • 2. There are multiple {@link Point}s at a {@link Location}. In this case, we sort all the * points, keep the one with the smallest identifier, add a {@link SyntheticDuplicateOsmNodeTag} * and remove the rest of the duplicate points. Two notes: 1. We keep Nodes if they have * different layer tagging. This way, we aren't creating a false connection between an overpass * and a road beneath it, which happened to have a way node at the identical location. 2. We are * potentially tossing out OSM Nodes with non-empty tags. However, this is the most * deterministic and simple way to handle this. The presence of the synthetic tag will make it * easy to write an Atlas Check to resolve the data error. *
* * @param atlas * The {@link Atlas} to remove the Points from * @return a new {@link Atlas} without the extra points or the given Atlas if no removal is * needed */ private Atlas removeDuplicateAndExtraneousPointsFromAtlas(final Atlas atlas) { final Set pointsToRemove = new HashSet<>(); final Set duplicatePointsToKeep = new HashSet<>(); for (final Point point : atlas.points()) { // Don't try to de-duplicate points we've already handled if (!pointsToRemove.contains(point.getIdentifier()) && !duplicatePointsToKeep.contains(point.getIdentifier())) { final Set duplicatePoints = Iterables .stream(atlas.pointsAt(point.getLocation())).map(Point::getIdentifier) .collectToSet(); if (!duplicatePoints.isEmpty() && duplicatePoints.size() > 1) { // Factor in ways that pass through these points. If these points are part of // ways that have a different layer tag value, then // keep all of them to avoid merging ways that shouldn't be merged. if (locationPartOfMultipleWaysWithDifferentLayerTags(atlas, point.getLocation())) { duplicatePointsToKeep.addAll(duplicatePoints); continue; } // Sort the points final Set sortedDuplicates = Iterables.asSortedSet(duplicatePoints); final Set uniqueLayerValues = new HashSet<>(); // Keep the point with the smallest identifier (deterministic) for each layer final Iterator duplicateIterator = sortedDuplicates.iterator(); final long duplicatePointToKeep = duplicateIterator.next(); final long layerValue = getLayerTagValueForPoint(atlas, duplicatePointToKeep); duplicatePointsToKeep.add(duplicatePointToKeep); duplicateIterator.remove(); uniqueLayerValues.add(layerValue); while (duplicateIterator.hasNext()) { final long candidateToKeep = duplicateIterator.next(); final long candidateLayerValue = getLayerTagValueForPoint(atlas, candidateToKeep); if (!uniqueLayerValues.contains(candidateLayerValue)) { // Keep the point if it has a unique layer value duplicatePointsToKeep.add(candidateToKeep); duplicateIterator.remove(); } } // Remove all remaining (non-kept) points pointsToRemove.addAll(sortedDuplicates); } } } // Remove any non-used shape points from filtered lines if (!this.pbfReader.getPointIdentifiersFromFilteredLines().isEmpty()) { pointsToRemove.addAll(preFilterPointsToRemove(atlas)); } // Remove points or return the original atlas if (pointsToRemove.isEmpty()) { return atlas; } else { // Rebuild the Atlas to add the synthetic tags and get rid of the removed points return rebuildAtlas(atlas, pointsToRemove, duplicatePointsToKeep); } } /** * Sets the {@link AtlasSize} to efficiently build the raw {@link Atlas}, using the values * obtained from the {@link OsmPbfCounter}. */ private void setAtlasSizeEstimate() { final AtlasSize size = new AtlasSize(0, 0, this.pbfCounter.lineCount(), this.pbfCounter.lineCount(), this.pbfCounter.pointCount(), this.pbfCounter.relationCount()); this.builder.setSizeEstimates(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/creation/RawAtlasStatistic.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.creation; import java.util.function.Consumer; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.statistic.storeless.CounterWithStatistic; import org.slf4j.Logger; /** * Keeps track of created and filtered (by configuration) {@link Point}s, {@link Line}s and * {@link Relation}s during raw {@link Atlas} generation - see {@link OsmPbfReader}. Also, tracks * any {@link Relation}s that were dropped. * * @author mgostintsev */ public class RawAtlasStatistic { private static final long LOG_FREQUENCY = 10_000; private final Logger logger; // Added entities private final CounterWithStatistic points; private final CounterWithStatistic lines; private final CounterWithStatistic relations; // Filtered entities private final CounterWithStatistic filteredNodes; private final CounterWithStatistic filteredWays; private final CounterWithStatistic filteredRelations; // Dropped entities private final CounterWithStatistic droppedWays; private final CounterWithStatistic droppedRelations; public RawAtlasStatistic(final Logger logger) { this.logger = logger; final Consumer log = logger::info; this.points = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Added Point"); this.points.logUsingLevel(log); this.lines = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Added Line"); this.lines.logUsingLevel(log); this.relations = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Added Relation"); this.relations.logUsingLevel(log); this.filteredNodes = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Filtered Node"); this.filteredNodes.logUsingLevel(log); this.filteredWays = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Filtered Way"); this.filteredWays.logUsingLevel(log); this.filteredRelations = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Filtered Relation"); this.filteredRelations.logUsingLevel(log); this.droppedWays = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Dropped Way"); this.droppedWays.logUsingLevel(log); this.droppedRelations = new CounterWithStatistic(this.logger, LOG_FREQUENCY, "Dropped Relation"); this.droppedRelations.logUsingLevel(log); } public void recordCreatedLine() { this.lines.increment(); } public void recordCreatedPoint() { this.points.increment(); } public void recordCreatedRelation() { this.relations.increment(); } public void recordDroppedRelation() { this.droppedRelations.increment(); } public void recordDroppedWay() { this.droppedWays.increment(); } public void recordFilteredNode() { this.filteredNodes.increment(); } public void recordFilteredRelation() { this.filteredRelations.increment(); } public void recordFilteredWay() { this.filteredWays.increment(); } public void summary() { this.logger.trace("PBF to Raw Atlas Summary"); this.points.summaryWithoutTimer(); this.lines.summaryWithoutTimer(); this.relations.summaryWithoutTimer(); this.filteredNodes.summaryWithoutTimer(); this.filteredWays.summaryWithoutTimer(); this.filteredRelations.summaryWithoutTimer(); this.droppedWays.summaryWithoutTimer(); this.droppedRelations.summaryWithoutTimer(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/sectioning/AtlasSectionProcessor.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.sectioning; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.change.ChangeAtlas; import org.openstreetmap.atlas.geography.atlas.change.ChangeBuilder; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompleteNode; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.dynamic.DynamicAtlas; import org.openstreetmap.atlas.geography.atlas.dynamic.policy.DynamicAtlasPolicy; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.WaySectionIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.tags.LayerTag; import org.openstreetmap.atlas.tags.SyntheticInvalidWaySectionTag; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.time.Time; import org.openstreetmap.osmosis.core.domain.v0_6.Node; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Way-section processor that converts Lines to Edges. This will work in two ways: 1. Section a * given atlas. 2. Given a shard, sharding and an atlas fetcher policy to - leverage * {@link DynamicAtlas} to build an Atlas that contains all edges from the initial shard to their * completion as well as any edges that may intersect them. For the second case above, we are * guaranteed to have consistent identifiers across shards after way-sectioning, since we are * relying on the line shape point order creation and identifying all edge intersections that span * shard boundaries. * * @author mgostintsev * @author samg */ public class AtlasSectionProcessor { private static final Logger logger = LoggerFactory.getLogger(AtlasSectionProcessor.class); private static final int MINIMUM_NODES_TO_QUALIFY_AS_A_EDGE = 2; // Logging constants private static final String STARTED_SECTIONING = "Started way-sectioning for Atlas {}"; private static final String FINISHED_SECTIONING = "Finished way-sectioning for Atlas {} in {}"; private static final String STARTED_EDGE_CREATION = "Started creating Edges for Atlas {}"; private static final String FINISHED_EDGE_CREATION = "Finished creating Edges for Atlas {} in {}}"; private static final String STARTED_NODE_CREATION = "Started creating Nodes for Atlas {}"; private static final String FINISHED_NODE_CREATION = "Finished creating Nodes for Atlas {} in {}}"; private static final String STARTED_EXCESS_POINT_REMOVAL = "Started removing excess Points for Atlas {}"; private static final String FINISHED_EXCESS_POINT_REMOVAL = "Finished removing excess Points for Atlas {} in {}}"; private static final String STARTED_POINT_ADDITION = "Started adding additional Points for Atlas {}"; private static final String FINISHED_POINT_ADDITION = "Finished adding additional Points for Atlas {} in {}"; // Expand the initial shard boundary to capture any edges that are crossing the shard boundary private static final Distance SHARD_EXPANSION_DISTANCE = Distance.meters(20); private final Atlas inputAtlas; private final AtlasLoadingOption loadingOption; // Bring in all lines that will become edges private final List loadedShards = new ArrayList<>(); private final Predicate dynamicAtlasExpansionFilter; private final Set changes = Collections .newSetFromMap(new ConcurrentHashMap()); private final Map nodeMap = new ConcurrentHashMap<>(); /** * Determines if we should section at the given {@link Location}. Relies on the underlying * {@link AtlasLoadingOption} configuration to make the decision. If true, this implies the * point at this {@link Location} should be a {@link Node}. Sectioning is either based on the * tag values of the underlying points at the location, or the existence of an intersecting Edge * at that location * * @param location * The {@link Location} to check * @param line * The {@link Line} this {@link Location} belongs to * @param loading * The AtlasLoadingOption to use for Edge test * @param inputAtlas * The source input atlas * @return {@code true} if we should section at the given {@link Location} */ public static boolean shouldSectionAtLocation(final Location location, final Line line, final AtlasLoadingOption loading, final Atlas inputAtlas) { final long targetLayerValue = LayerTag.getTaggedOrImpliedValue(line, 0L); return Iterables.stream(inputAtlas.pointsAt(location)) .anyMatch(point -> loading.getWaySectionFilter().test(point)) || Iterables // Find all intersecting edges .stream(inputAtlas.linesContaining(location, target -> target.getIdentifier() != line.getIdentifier() && target.asPolyLine().contains(location))) .filter(loading.getEdgeFilter()::test) // Check whether that edge has a different layer value as the line we're // looking at and that our point is its start or end node .anyMatch(candidate -> { final long layerValue = LayerTag.getTaggedOrImpliedValue(candidate, 0L); if (targetLayerValue == layerValue) { return true; } final boolean edgesOnDifferentLayers = targetLayerValue != layerValue; final PolyLine candidatePolyline = candidate.asPolyLine(); final boolean intersectionIsAtEndPoint = candidatePolyline.first() .equals(location) || candidatePolyline.last().equals(location); return edgesOnDifferentLayers && intersectionIsAtEndPoint; }); } /** * Default constructor. Will section given raw {@link Atlas} file. * * @param inputAtlas * The {@link Atlas} to section * @param loadingOption * The {@link AtlasLoadingOption} to use */ public AtlasSectionProcessor(final Atlas inputAtlas, final AtlasLoadingOption loadingOption) { this.inputAtlas = inputAtlas; this.loadingOption = loadingOption; this.dynamicAtlasExpansionFilter = entity -> entity instanceof Line && this.loadingOption.getEdgeFilter().test(entity); } /** * Takes in a starting {@link Shard} and uses the given sharding and atlas fetcher function to * build a {@link DynamicAtlas}, which is then sectioned. This guarantees consistent identifiers * across the constructed atlas. The sharding and atlas fetcher function must be provided and * the sharding must the same as the one used to generate the input shard. The overall logic for * atlas construction and sectioning is: *
    *
  • Grab the atlas for the starting shard in its entirety, expand out if there are any edges * bleeding into neighboring shards. *
  • Once the full atlas is built, way-section it. *
  • After sectioning is completed and atlas is rebuild, cut a sub-atlas representing the * bounds of the original shard being processed. This is a soft cut, so any edges that start in * the shard and end in neighboring shards, will be captured. *
* * @param initialShard * The initial {@link Shard} to start at * @param loadingOption * The {@link AtlasLoadingOption} to use * @param sharding * The {@link Sharding} to use to know which neighboring atlas files to get * @param atlasFetcher * The fetching policy to use to obtain adjacent atlas files */ public AtlasSectionProcessor(final Shard initialShard, final AtlasLoadingOption loadingOption, final Sharding sharding, final Function> atlasFetcher) { this.loadingOption = loadingOption; this.dynamicAtlasExpansionFilter = entity -> entity instanceof Line && this.loadingOption.getEdgeFilter().test(entity); if (sharding == null || atlasFetcher == null) { throw new IllegalArgumentException( "Must supply a valid sharding and fetcher function for sectioning!"); } this.inputAtlas = buildExpandedAtlas(initialShard, sharding, atlasFetcher); } /** * Slices the given {@link Atlas}. * * @return the way-sectioned {@link Atlas} */ public Atlas run() { final Time overallTime = Time.now(); Time time = Time.now(); logger.info(STARTED_SECTIONING, this.getShardOrAtlasName()); logger.info(STARTED_EDGE_CREATION, this.getShardOrAtlasName()); this.inputAtlas.lines(this.loadingOption.getEdgeFilter()::test).forEach(this::section); logger.info(FINISHED_EDGE_CREATION, this.getShardOrAtlasName(), time.elapsedSince().asMilliseconds()); time = Time.now(); logger.info(STARTED_NODE_CREATION, this.getShardOrAtlasName()); this.nodeMap.values() .forEach(node -> this.changes.add(FeatureChange.add(node, this.inputAtlas))); logger.info(FINISHED_NODE_CREATION, this.getShardOrAtlasName(), time.elapsedSince().asMilliseconds()); time = Time.now(); // If this atlas is supposed to keep everything, add the points that are not also saved as a // node. if (this.loadingOption.isKeepAll()) { logger.info(STARTED_POINT_ADDITION, this.getShardOrAtlasName()); this.inputAtlas.points().forEach(point -> { final CompleteNode possibleDupe = this.nodeMap.get(point.getLocation()); if (possibleDupe == null || possibleDupe.getOsmIdentifier() != point.getOsmIdentifier()) { this.changes.add(FeatureChange.add(CompletePoint.from(point))); } }); logger.info(FINISHED_POINT_ADDITION, this.getShardOrAtlasName(), time.elapsedSince().asMilliseconds()); } else { logger.info(STARTED_EXCESS_POINT_REMOVAL, this.getShardOrAtlasName()); this.inputAtlas.points().forEach(point -> { // we care about a point if and only if it has pre-existing OSM tags OR it belongs // to a future edge OR we are doing QA if (!this.loadingOption.isKeepAll() && point.getOsmTags().isEmpty() && point.relations().isEmpty()) { this.changes.add(FeatureChange.remove(CompletePoint.shallowFrom(point))); } }); logger.info(FINISHED_EXCESS_POINT_REMOVAL, this.getShardOrAtlasName(), time.elapsedSince().asMilliseconds()); } logger.info(FINISHED_SECTIONING, this.getShardOrAtlasName(), overallTime.elapsedSince().asMilliseconds()); // return either the unchanged original Atlas, or a cut-down version of the sectioned Atlas if (this.changes.isEmpty()) { if (this.loadedShards.isEmpty() || this.loadedShards.size() == 1) { return this.inputAtlas.cloneToPackedAtlas(); } return cutSubAtlasForOriginalShard(this.inputAtlas).cloneToPackedAtlas(); } final String country = this.loadingOption.getCountryCode(); final String shardOrAtlasName = this.getShardOrAtlasName(); final ChangeAtlas sectionedAtlas = new ChangeAtlas(this.inputAtlas, new ChangeBuilder().addAll(this.changes).get()) { private static final long serialVersionUID = -1379576156041355921L; @Override public synchronized AtlasMetaData metaData() { // Override meta-data here so the country code is properly included. final AtlasMetaData metaData = super.metaData(); final var originalTags = metaData.getTags(); // Remove country shards to keep old behavior where they were dropped, but keep // other tags. originalTags.remove("countryShards"); return new AtlasMetaData(metaData.getSize(), false, metaData.getCodeVersion().orElse(null), metaData.getDataVersion().orElse(null), country, shardOrAtlasName, originalTags); } }; if (this.loadedShards.isEmpty()) { return sectionedAtlas.cloneToPackedAtlas(); } return cutSubAtlasForOriginalShard(sectionedAtlas).cloneToPackedAtlas(); } /** * Grabs the atlas for the initial shard, in its entirety. Then proceeds to expand out to * surrounding shards if there are any edges bleeding over the shard bounds plus * {@link #SHARD_EXPANSION_DISTANCE}. Finally, will return the constructed Atlas. * * @param initialShard * The initial {@link Shard} being processed * @param sharding * The {@link Sharding} used to identify which shards to fetch * @param fullySlicedAtlasFetcher * The fetcher policy to retrieve an Atlas file for each shard * @return the expanded {@link Atlas} */ private Atlas buildExpandedAtlas(final Shard initialShard, final Sharding sharding, final Function> fullySlicedAtlasFetcher) { // Keep track of all loaded shards. This will be used to cut the sub-atlas for the shard // we're processing after all sectioning is completed. Initial shard will always be first! this.loadedShards.add(initialShard); final DynamicAtlasPolicy policy = new DynamicAtlasPolicy(fullySlicedAtlasFetcher, sharding, initialShard.bounds().expand(SHARD_EXPANSION_DISTANCE), Rectangle.MAXIMUM) .withDeferredLoading(true).withExtendIndefinitely(false) .withAtlasEntitiesToConsiderForExpansion(this.dynamicAtlasExpansionFilter::test); final DynamicAtlas atlas = new DynamicAtlas(policy); atlas.preemptiveLoad(); return atlas; } /** * Takes a polyline for a new Edge and adds the feature to the ChangeSet * * @param line * The {@link Line} being converted to an {@link Edge} * @param edgePolyLine * The polyline defining the geometry of the {@link Edge} * @param edgeIdentifier * The identifier for the {@link Edge} * @param hasReverseEdge * Boolean for if a reverse {@link Edge} should be made as well * @param tags * The tags for the new {@link Edge} */ private void createEdge(final Line line, final PolyLine edgePolyLine, final long edgeIdentifier, final boolean hasReverseEdge, final Map tags) { // if a node already exists for the start/end locations, use theml otherwise make new ones CompleteNode startNode = this.nodeMap.get(edgePolyLine.first()); if (startNode == null) { final SortedSet inEdges = new TreeSet<>(); if (hasReverseEdge) { inEdges.add(-edgeIdentifier); } final SortedSet outEdges = new TreeSet<>(); outEdges.add(edgeIdentifier); startNode = createNode(line, edgePolyLine.first(), inEdges, outEdges); } else { if (hasReverseEdge) { startNode.withAddedInEdgeIdentifier(-edgeIdentifier); } startNode.withAddedOutEdgeIdentifier(edgeIdentifier); } CompleteNode endNode = this.nodeMap.get(edgePolyLine.last()); if (endNode == null) { final SortedSet inEdges = new TreeSet<>(); inEdges.add(edgeIdentifier); final SortedSet outEdges = new TreeSet<>(); if (hasReverseEdge) { outEdges.add(-edgeIdentifier); } endNode = createNode(line, edgePolyLine.last(), inEdges, outEdges); } else { if (hasReverseEdge) { endNode.withAddedOutEdgeIdentifier(-edgeIdentifier); } endNode.withAddedInEdgeIdentifier(edgeIdentifier); } final Set relations = new HashSet<>(); line.relations().forEach(relation -> relations.add(relation.getIdentifier())); final CompleteEdge newEdge = new CompleteEdge(edgeIdentifier, edgePolyLine, tags, startNode.getIdentifier(), endNode.getIdentifier(), relations); final Set nonGeometricRelations = new HashSet<>(); for (final Long relationId : relations) { if (!this.inputAtlas.relation(relationId).isGeometric()) { nonGeometricRelations.add(relationId); } } final CompleteEdge newReverseEdge = new CompleteEdge(-edgeIdentifier, edgePolyLine.reversed(), tags, endNode.getIdentifier(), startNode.getIdentifier(), nonGeometricRelations); updateRelations(line, newEdge, newReverseEdge, hasReverseEdge); this.changes.add(FeatureChange.add(newEdge, this.inputAtlas)); if (hasReverseEdge) { this.changes.add(FeatureChange.add(newReverseEdge, this.inputAtlas)); } } /** * Helper method to make a new Node for an Edge * * @param line * The {@link Line} being converted to an {@link Edge} * @param nodeLocation * The {@link Location} for the {@link Node} being made * @param inEdges * The identifiers for the {@link Edge}s going into the {@link Node} being made * @param outEdges * The identifiers for the {@link Edge}s going out of the {@link Node} being made * @return */ private CompleteNode createNode(final Line line, final Location nodeLocation, final SortedSet inEdges, final SortedSet outEdges) { if (!this.inputAtlas.pointsAt(nodeLocation).iterator().hasNext()) { throw new CoreException( "Couldn't find node at {} while sectioning Line {} for Atlas {}", nodeLocation.toString(), line.toString(), getShardOrAtlasName()); } final Point pointForNode = this.inputAtlas.pointsAt(nodeLocation).iterator().next(); // Drop nodes that don't have tags when we don't need them for other purposes (e.g., QA) if (!this.loadingOption.isKeepAll() && pointForNode.getOsmTags().isEmpty()) { this.changes.add(FeatureChange.remove(CompletePoint.shallowFrom(pointForNode))); } final Set relationIds = new HashSet<>(); pointForNode.relations().forEach(relation -> relationIds.add(relation.getIdentifier())); final CompleteNode node = new CompleteNode(pointForNode.getIdentifier(), pointForNode.getLocation(), pointForNode.getTags(), inEdges, outEdges, relationIds); this.nodeMap.put(node.getLocation(), node); pointForNode.relations().forEach(relation -> relation .membersMatching(member -> member.getEntity().getType().equals(ItemType.POINT) && member.getEntity().getIdentifier() == pointForNode.getIdentifier()) .forEach(member -> this.changes.add(FeatureChange.add(CompleteRelation .shallowFrom(relation).withAddedMember(node, member.getRole()))))); return node; } /** * Takes a given Line and its Nodes, and turns into Edges starting and ending at each Node. * * @param line * The {@link Line} being converted to an {@link Edge} * @param nodes * The identifiers for the {@link Point}s that will be converted into the * {@link Node} for the {@link Edge} * @param isReversed * Boolean for if the geometry of the {@link Edge} should reversed * @param hasReverseEdge * Boolean for if a reverse {@link Edge} should be made as well * @param remainder * Any remaining linear geometry at the end of the {@link Line} being converted to an * {@link Edge}-- in some circumstances, this geometry will be converted to its own * {@link Edge}, but in many cases it will be combined into the last {@link Edge} to * reduce the number of {@link Edge}s made */ private void createSections(final Line line, final List nodes, final boolean isReversed, final boolean hasReverseEdge, final PolyLine remainder) { // Prepare the nodes identifiers, identifier factory and one way information final WaySectionIdentifierFactory identifierFactory = new WaySectionIdentifierFactory( line.getIdentifier()); // if the edge geometry is going to be singular, make that directly and bypass the loops if (remainder != null && remainder.size() == line.asPolyLine().size()) { createEdge(line, remainder, identifierFactory.nextIdentifier(), hasReverseEdge, line.getTags()); return; } else if (!line.isClosed() && nodes.size() == 2 && nodes.get(0) == 0 && line.asPolyLine().size() - 1 == nodes.get(1)) { createEdge(line, isReversed ? line.asPolyLine().reversed() : line.asPolyLine(), line.getIdentifier(), hasReverseEdge, line.getTags()); return; } final Iterator nodesIterator = nodes.iterator(); int startIndex = nodesIterator.next(); // iterate over all node locations and make edges for each polyline section while (identifierFactory.hasMore() && nodesIterator.hasNext()) { int endIndex = nodesIterator.next(); final long edgeIdentifier = identifierFactory.nextIdentifier(); final Map tags = line.getTags(); // if there are no more identifiers left, fast forward to the end of the line if (!identifierFactory.hasMore()) { // Update the tags to indicate this edge wasn't way-sectioned tags.put(SyntheticInvalidWaySectionTag.KEY, SyntheticInvalidWaySectionTag.YES.toString()); endIndex = line.asPolyLine().size() - 1; } final PolyLine rawPolyLine = new PolyLine(line.asPolyLine().truncate(startIndex, line.asPolyLine().size() - endIndex - 1)); PolyLine edgePolyLine = isReversed ? rawPolyLine.reversed() : rawPolyLine; // if this is the last node, deal with the remainder. if the remainder if (!nodesIterator.hasNext() && remainder != null) { final Location potentialStitchLocation = isReversed ? edgePolyLine.first() : edgePolyLine.last(); // if we need a section at the last location, we'll make the remainder its own edge. // otherwise, we'll combine it with the last edge to reduce excess edges if (shouldSectionAtLocation(potentialStitchLocation, line, this.loadingOption, this.inputAtlas)) { final long remainderIdentifier = identifierFactory.nextIdentifier(); createEdge(line, remainder, remainderIdentifier, hasReverseEdge, tags); } else { edgePolyLine = isReversed ? remainder.append(edgePolyLine) : edgePolyLine.append(remainder); } } createEdge(line, edgePolyLine, edgeIdentifier, hasReverseEdge, tags); startIndex = endIndex; } } /** * Up to this point, we've constructed the {@link DynamicAtlas} and way-sectioned it. Since * we're only responsible for returning an Atlas for the provided shard, we now need to cut a * sub-atlas for the initial shard boundary and return it. We can leverage the loaded shards * parameter, which will always contain the starting shard as the first shard and all other * loaded shards after. If no other shards were loaded, simply return the given Atlas. * * @param atlas * The {@link Atlas} file we need to trim * @return the {@link Atlas} for the bounds of the input shard */ private Atlas cutSubAtlasForOriginalShard(final Atlas atlas) { try { // The first shard is always the initial one. Use its bounds to build the atlas. final Rectangle originalShardBounds = this.loadedShards.get(0).bounds(); return atlas.subAtlas(originalShardBounds, AtlasCutType.SOFT_CUT) .orElseThrow(() -> new CoreException( "Cannot have an empty atlas after way sectioning {}", this.loadedShards.get(0).getName())); } catch (final Exception e) { throw new CoreException("Error creating sub-atlas for original shard bounds", e); } } /** * Iterate over a Line's locations to determine which ones qualify as a location to section at * * @param line * The {@link Line} being converted to an {@link Edge} * @param linePolyLine * The polyline for the {@link Line} being converted to an {@link Edge} * @return */ private List findNodesForEdge(final Line line, final PolyLine linePolyLine) { final List nodesForEdge = new ArrayList<>(); final Set selfIntersections = linePolyLine.selfIntersections(); Location previousLocation = null; for (int i = 0; i < linePolyLine.size(); i++) { final Location location = linePolyLine.get(i); if (i == 0 || i == linePolyLine.size() - 1) { nodesForEdge.add(i); } else if (location.equals(previousLocation)) { // NOOP } else if (selfIntersections.contains(location) || shouldSectionAtLocation(location, line, this.loadingOption, this.inputAtlas)) { nodesForEdge.add(i); } previousLocation = location; } return nodesForEdge; } /** * Just a helper method to use a consistent naming scheme * * @return A String for either the Shard name or the Atlas name */ private String getShardOrAtlasName() { // Default to getting the Shard name, if available, otherwise fall back to atlas name if (!this.loadedShards.isEmpty()) { return this.loadedShards.get(0).getName(); } else { return this.inputAtlas.getName(); } } /** * Takes a Line, finds its Nodes, then makes Edges for each section. * * @param line * The {@link Line} being converted to an {@link Edge} */ private void section(final Line line) { final PolyLine polyLine = line.asPolyLine(); // Determines if we need to reverse the polyline and if a reverse edge is needed final PbfOneWay oneWay = PbfOneWay.forTag(line); final boolean hasReverseEdge = oneWay == PbfOneWay.NO; final boolean isReversed = oneWay == PbfOneWay.REVERSED; final List nodesForEdge = findNodesForEdge(line, polyLine); if (nodesForEdge.size() < MINIMUM_NODES_TO_QUALIFY_AS_A_EDGE) { logger.error("Edge {} hass less than {} nodes, cannot be sectioned!", line.getIdentifier(), MINIMUM_NODES_TO_QUALIFY_AS_A_EDGE); this.changes .add(FeatureChange.add( CompleteLine.shallowFrom(line).withTags(line.getTags()).withAddedTag( SyntheticInvalidWaySectionTag.KEY, SyntheticInvalidWaySectionTag.YES.toString()), this.inputAtlas)); return; } // Initialize start location PolyLine remainder = null; // we'll preprocess rings a bit to make them consistently sectioned if (line.isClosed()) { if (nodesForEdge.size() == 2) { // the node is just the beginning/end of the Edge, and connects to another Edge, // like a cul-de-sac if (nodesForEdge.get(0) == 0 && nodesForEdge.get(1) == line.numberOfShapePoints() - 1 && shouldSectionAtLocation(polyLine.get(0), line, this.loadingOption, this.inputAtlas)) { // we just want a single Edge for the whole loop, connecting back to itself // noop } else { // corner case-- a ring with no other connected Edges that starts and ends at // the same node. make an artificial node halfway through to make two Edges nodesForEdge.remove(1); nodesForEdge.add(polyLine.size() / 2); nodesForEdge.add(polyLine.size() - 1); } } else if (polyLine.isSimple()) { final int nextIndex = nodesForEdge.get(nodesForEdge.size() - 2); remainder = new PolyLine(polyLine.truncate(nextIndex, 0)); nodesForEdge.remove(nodesForEdge.size() - 1); if (!shouldSectionAtLocation(polyLine.get(0), line, this.loadingOption, this.inputAtlas)) { nodesForEdge.remove(0); final int startIndex = nodesForEdge.get(0); remainder = remainder.append( new PolyLine(polyLine.truncate(0, polyLine.size() - 1 - startIndex))); } if (isReversed) { remainder = remainder.reversed(); } } } createSections(line, nodesForEdge, isReversed, hasReverseEdge, remainder); this.changes.add(FeatureChange.remove(CompleteLine.shallowFrom(line), this.inputAtlas)); } /** * Iterate over Relations to make sure they're synchronized with the changes for new Edges * * @param line * The {@link Line} being converted to an {@link Edge} * @param newEdge * The new {@link Edge} * @param newReverseEdge * The new reverse {@link Edge} * @param hasReverseEdge * Boolean representing the existence of a reverse {@link Edge} */ private void updateRelations(final Line line, final CompleteEdge newEdge, final CompleteEdge newReverseEdge, final boolean hasReverseEdge) { line.relations().forEach(relation -> relation .membersMatching(member -> member.getEntity().getType().equals(ItemType.LINE) && member.getEntity().getIdentifier() == line.getIdentifier()) .forEach(member -> { this.changes.add(FeatureChange.add(CompleteRelation.shallowFrom(relation) .withAddedMember(newEdge, member.getRole()))); if (hasReverseEdge && !relation.isGeometric()) { this.changes.add(FeatureChange.add(CompleteRelation.shallowFrom(relation) .withAddedMember(newReverseEdge, member.getRole()))); } })); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/sectioning/PbfOneWay.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.sectioning; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.JunctionTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.tags.oneway.OneWayTag; /** * One way attribute of an OSM Way * * @author tony * @author matthieun */ public enum PbfOneWay { YES, NO, REVERSED; /** * Determines the whether the given {@link Taggable} is a one-way, non-one-way, reversed or * closed Edge. * * @param taggable * The {@link Taggable} to look at * @return the {@link PbfOneWay} for the given {@link Taggable} */ public static PbfOneWay forTag(final Taggable taggable) { if (OneWayTag.isExplicitlyTwoWay(taggable)) { return NO; } else if (OneWayTag.isTwoWay(taggable)) { if (JunctionTag.isRoundabout(taggable) || Validators.isOfType(taggable, HighwayTag.class, HighwayTag.MOTORWAY)) { // Override the two-way here, as a roundabout takes precedence as a one way road in // OSM, when no one way tag is specified. Similarly, a motorway tag implies a // one way road. The same does NOT hold true for motorway_link. return YES; } return NO; } else if (OneWayTag.isOneWayForward(taggable)) { return YES; } else if (OneWayTag.isOneWayReversed(taggable)) { return REVERSED; } else { return NO; } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/sectioning/README.md ================================================ # `Atlas Sectioning` ## Overview Atlas Sectioning is the process of converting certain `Lines` and `Points` into navigable `Edges` and `Nodes`. As input, the process takes in an Atlas with no `Edges` or `Nodes` and produces one that has these entities. ## Detailed Steps The sectioning process is comprised of the following steps: 1. Assemble a `DynamicAtlas` that will be used for sectioning. The reason for a `DynamicAtlas` is to be able to capture all OSM Ways that go outside the `Shard` bounds being processed and also capture any intersections of the each of these ways. This is the driving force behind the guarantee of consistent `Edge` identifiers. If we can see a way in its entirety as well as all of its intersecting ways, we can section it deterministically and consistently, irrespective of which `Shard` we are looking from and apply this consistent view to the current `Shard` being processed. The logic for building the `DynamicAtlas` is to grab the full Atlas for the initial `Shard`, then proceed to expand out to surrounding Shards if there are any Edges bleeding over the initial `Shard` boundary. One caveat to call out here is that the `Sharding` being passed in, **must** be the same `Sharding` that was used to generate the input raw Atlas. If it's not, there may be inconsistencies in the understanding of the surrounding shards that are available. 2. Iterate over all the input `Line`s and identify the ones that become `Edges`, based on the tags for the `Line`. 1. If a `Line` is set to become an `Edge`, we identify all location-based `Nodes` for that `Edge`. A `Node` will be created in one of the 4 cases: * A self-intersection - there is a repeated non-consecutive intersection in the `Edge` * Configuration-based sectioning dependent on tagging (ex: section at a barrier) * Intersection with another `Edge` with the same `LayerTag` value * Intersection with another `Edge` with a different `LayerTag` value, that either starts or ends at the point of intersection 2. If the `Line` is closed, apply some consistency logic to ensure we get a good sectioning result * If there are only 2 `Node` candidates, then check to see if it's a basic loop with a single `Node` at the beginning/end of the `Edge` * If so, make it into one `Edge` * Otherwise, it's disconnected from the rest of the `Edge` network, so split it artificially at the halfway point and make two `Edge`s for the loop * If it has more than 2 `Nodes`, put the last `Edge` section into a "remainder". If the beginning/end of the `Line` isn't connected to anything else, then we'll add the remainder to it so we don't create excess `Edge`s. Otherwise, we'll make an `Edge` from the remainder 3. Using the calculated `Node` candidates and remainder geometry, make `Node` and `Edges` for the `Line` 4. Remove the old `Line` from the `Atlas` 3. Iterate over the staged CompleteNode entities and add them as FeatureChanges-- waiting until after all the `Edges` have been processed ensures this process just simplifies things from a consistency standpoint 4. Iterate over all remaining `Points` in the `Atlas` and remove them if they don't have explicit tags or belong to any `Relations` 5. If there were changes, make a new `ChangeAtlas` and return it, otherwise return the original `Atlas` ## Synthetic Tags The sectioning process add a new synthetic tag as part of the final Atlas - `SyntheticInvalidWaySectionTag`. This tag signifies that we couldn't fully section the given Atlas Line. Specifically, we created 998 Edges and put the rest of the un-sectioned remnant into `Edge` 999. This is usually an indicator of bad data. We have seen this in the case of really long duplicated OSM Ways. If the two Ways are stacked and have over 1000 shape points, then each shape point would become an Atlas `Node` and result in sectioning. This is technically also possible for really long valid OSM Ways that have more than 999 intersections for the duration of the Way. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/sectioning/TagMap.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.sectioning; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.ManMadeTag; import org.openstreetmap.atlas.tags.RailwayTag; import org.openstreetmap.atlas.tags.RouteTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.osmosis.core.domain.v0_6.Tag; /** * A wrapper for tags for an OSM entity. * * @author tony * @author matthieun */ public class TagMap implements Taggable { private final Map tags; public TagMap(final Collection tagCollection) { this.tags = new HashMap<>(); tagCollection.forEach(tag -> this.tags.put(tag.getKey(), tag.getValue())); } public TagMap(final Map tags) { this.tags = tags; } public PbfOneWay getOneWay() { return PbfOneWay.forTag(this); } @Override public Optional getTag(final String key) { return Optional.ofNullable(this.tags.get(key)); } @Override public Map getTags() { return this.tags; } public boolean hasHighwayTag() { return HighwayTag.highwayTag(this).isPresent(); } public boolean isEmpty() { return this.tags.size() == 0; } public boolean matchFerry() { return RouteTag.isFerry(this); } public boolean matchPier() { return ManMadeTag.isPier(this); } public boolean matchRailway() { return RailwayTag.isRailway(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/slicing/CountryCodeProperties.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.slicing; import org.locationtech.jts.geom.Geometry; import org.openstreetmap.atlas.tags.ISOCountryTag; /** * Simple container that tracks country code and nearest neighbor values returned from a JTS * {@link Geometry}. * * @author mgostintsev */ public class CountryCodeProperties { private final String iso3CountryCode; public CountryCodeProperties(final String iso3CountryCode) { this.iso3CountryCode = iso3CountryCode; } /** * @return a string to represent country code in iso_3 format. If multiple countries, they'll be * separated by comma. e.g. USA,CAN */ public String getIso3CountryCode() { return this.iso3CountryCode; } /** * @return {@code true} if the country code field contains more than 1 country */ public boolean inMultipleCountries() { return this.iso3CountryCode.contains(ISOCountryTag.COUNTRY_DELIMITER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/slicing/README.md ================================================ # `Raw Atlas Slicing` ## Overview Slicing is the operation of taking an `Atlas` and tagging all of its entities with country codes, as well as ensuring that *only* entities from the loaded country code persist in the output `Atlas`. While this operation may sound straightforward, data complexities frequently result in corner cases and complicated calculations in order to make sure the entities in the output `Atlas` are sensible. The term "slicing" describes these more difficult cases because the geometry of an entity frequently spans multiple countries, and thus must be "sliced" into portions that either a) are contained entirely in only one country boundary polygon or b) exactly along the shared overlap of multiple country boundary exterior rings and thus have geometry shared by a set of countries. ### Initialization The slicer is constructed with a minimum input of `Atlas` and `AtlasLoadingOption` * The `Atlas` should be raw, i.e. not previously sliced and consisting of only `Line`, `Point`, and `Relation` entities * The `AtlasLoadingOption` will use the following paramters: * `CountryBoundaryMap` to determine country boundary definitions * `countryCodes` to determine which entities to keep in the final Atlas-- any entity whose country code is *not* in the `countryCodes` set will be removed from the final tlas! * `relationSlicingFilter` to determine which `Relations` to expand the `DynamicAtlas` on and attempt to slice-- at a minimum, these relations must be `type->multipolygon` or `type->boundary`, but additional tagging critieria are acceptable here * `edgeFilter` to determine which `Line` entities should be considered future `Edge` entities * This is important because closed `Line` entities will be sliced as two-dimenstional polygons but closed `Edge` entities (such as a traffic circle) will be sliced as linear features If the constructor that takes in a `Sharding` object and `Atlas` fetcher function is used, then additionally the initial `Atlas` will be converted to a `DynamicAtlas` expanded on the `relationSlicingFilter` in the `AtlasLoadingOption`. Once the constructor is called, three maps are initialized for `CompleteEntity` representations of all three `AtlasEntity` types. These objects will be used to track changes made during the slicing operation. ### Slicing Steps The high level operations of slicing are executed in the `slice()` method, and can be summarized as follows: 1. Slice all `Line` entities in the `Atlas` following a basic logic fork: * If the `Line` entity is closed (i.e. a loop) and neither a future `Edge` or a multipolygon `Relation` member, then slice it as an `Area` (2d geometry) * Otherwise, slice it as a linear entity 2. Slice all multipolygon type `Relation` entities 3. Slice all `Point` entities 4. Filter any remaining `Relation` entities-- because we don't expand on all `Relations`, it's impossible to deterministically "slice" these `Relations`, so instead we just filter out any members that are outside the country code set being sliced and update the country tag for the `Relation` to be the sum of its remaining members. 5. Add all `CompleteEntities` in the staged entity maps as `FeatureChange.ADD`-- these are either existing features being updated or new entities being added, so `FeatureChange.ADD` is always appropriate 6. Build a new `ChangeAtlas` out of these changes, then cut out any entities that lay outside the shard bounds (frequently happens for data loaded in during the `DynamicAtlas`expansion for `Relations`) ### Geometry Slicing Slicing of all entities follows the same general approach. 1. The data for the entity is constructed into a relevant JTS geometry. * For example, for a closed `Line` that meets the criteria for an `Area`, a JTS `Polygon` is created 2. This internal envelope for this geometry is checked against the spatial index of the boundary map to calculate which country boundary polygons it intersects * Should this geometry exclusively belong to one country, its geometry is left unaltered and the country code tag is updated to contain this country 3. Next, geometry is checked for validity-- it must be [valid geometry](http://www.ogc.org/docs/is/) and not empty * This is critical because if we filter out occasional invalid geomtery coming out of the slicing operation; inputting invalid geometry is a guarantee of junk output * If it's invalid, then we remove it from the Atlas 4. The JTS geometry is then divided into the portions intersected by each country boundary polygon, creating a map of country code to geometric pieces. * ALL of these pieces must meet the requirements for validity and significance, because incredibly small lines or polygons are likely junk data or irrelevant * Should the operation return a GeometryCollection, all geometries in the collection are separated and added to the results Set independently after being checked for validity and significance 5. The results are post-processed based on the type of AtlasEntity * These operations are largely similar but there are a few different expectations based on AtlasEntity type-- for example, a sliced linear `Line` will attempt to join all resulting `LineString` pieces for each country using `LineMerger` because occasionally a few of these `LineStrings` can be merged * Additionally, the map returned here will use `SortedMap` to guarantee deterministic consistency in `Line` slice creation regardless of what country code settings are used, etc. 6. On the small chance that the slicing operation returned an empty set or all significant geometry was in exclusively one country, the geometry will be unaltered and the country code tag will be updated to have either the country-missing value or the single country only, respectively 7. Additionally, should the number of slices be greater than the country identifier space (000-999), then the operation will fail out and the entity will be tagged with all country codes its geometry spanned 8. Finally, the slice geomtries will be converted sequentially based on their SortedMap ordering into new `AtlasEntities` with the same tags as their parent entity, but with the geomtery of the slice and the country code tag of the relevant country, and these are be added to the relevant staged `CompleteEntity` map * At the end of this process, the original parent entity will be removed from the relevant staged `CompleteEntity` map and a `FeatureChange.remove` is added to the `changes` `ChangeSet` to ensure it is removed from the final `Atlas` ### MultiPolygon Relation Slicing This operation has some added complexity that is worth explaining in-depth. While the overall approach follows the approach described above, the changes are as follows: 1. The `Relation` is filtered of [invalid members](https://wiki.openstreetmap.org/wiki/Relation:multipolygon#Members) 2. The geometry is built using the *raw member line geometry*, not the sliced `Lines` in the staged `CompleteEntity` map * This choice ensures that the geometry will build if the raw `Relation` data builds a valid multipolygon, and avoids any possible small stitching errors resultant from data gaps introduced during `Line` slicing 3. After the multipolygon is sliced against the intersecting country boundary polygons and new sliced `Relations` are created, an additional step occurs: `createSyntheticRelationMembers()` * This method takes the sliced multipolygon for a country, subtracts out the existing sliced `Line` members for that country from that geometry, then generates new `Line` entities to cover the remaining geometry * This operation preserves the ability to build a valid multipolygon out of the sliced `Relation`-- without it, the sliced `Line` members would have major gaps and fail to build into geometry * Additionally, in rare circumstances a member that was previously tagged as an `inner` role will now overlap with the exterior ring of the sliced geometry * In this case, that member will still be preserved but its role will be switched to an `outer` ## Synthetic Tags There are synthetic tags generated by the country-slicing process: 1. `SyntheticRelationMemberAdded` - indicates a Relation that had an added member as a result of country-slicing. An example includes closing a water body MultiPolygon relation with a new `Line` member that runs along a country boundary 2. `SyntheticRelationRoleUpdated` - indicates a Relation member role update, any time that some combination of inner/outer relation members was merged 3. `SyntheticBoundaryNodeTag` -- indicates a Point/Node along a Line/Edge that has been created due to the slicing of Line/Edge geometry at a country boundary 4. `SytheticInvalidGeometryTag` -- indicates a geometry went through the slicing process, but could not be sliced due to not meeting OGC compliance for the geometry type 5. `SyntheticInvalidMultiPolygonRelationMembersRemovedTag` -- indicates any multipolygon Relation members that were removed during slicing due to not meeting the OSM requirements for members (i.e. not a Line with a role of "inner" or "outer") 6. `SyntheticSyntheticRelationMemberTag` -- indicates an entity is a synthetic addition to a multipolygon Relation added during slicing 7. `SyntheticGeometrySlicedTag` -- indicates an entity had its geometry changed during slicing ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/raw/slicing/RawAtlasSlicer.java ================================================ package org.openstreetmap.atlas.geography.atlas.raw.slicing; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.prep.PreparedGeometry; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.geom.prep.PreparedPolygon; import org.locationtech.jts.operation.linemerge.LineMerger; import org.locationtech.jts.operation.overlayng.OverlayNG; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.change.ChangeAtlas; import org.openstreetmap.atlas.geography.atlas.change.ChangeBuilder; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.complete.CompleteArea; import org.openstreetmap.atlas.geography.atlas.complete.CompleteLine; import org.openstreetmap.atlas.geography.atlas.complete.CompletePoint; import org.openstreetmap.atlas.geography.atlas.complete.CompleteRelation; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.items.complex.MultiPolygonRelationToMemberConverter.Ring; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.AbstractIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.CountrySlicingIdentifierFactory; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPrecisionManager; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.tags.RelationTypeTag; import org.openstreetmap.atlas.tags.SyntheticBoundaryNodeTag; import org.openstreetmap.atlas.tags.SyntheticGeometrySlicedTag; import org.openstreetmap.atlas.tags.SyntheticInvalidGeometryTag; import org.openstreetmap.atlas.tags.SyntheticInvalidMultiPolygonRelationMembersRemovedTag; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.identifiers.EntityIdentifierGenerator; import org.openstreetmap.atlas.utilities.scalars.Duration; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Takes a raw Atlas (i.e. only Points, Lines, and Relations, all lacking country code tags) and * "slices" it * * @author samg */ public class RawAtlasSlicer { // Buffer values for slicing operation. If the remaining piece turns to be smaller than // buffer, we'll just ignore them. public static final double LINE_BUFFER = 0.000001; public static final double AREA_BUFFER = 0.000000005; public static final double BUFFER_PERCENTAGE = 0.005; public static final double PERCENTAGE = 100; // JTS converters private static final JtsPolygonConverter JTS_POLYGON_CONVERTER = new JtsPolygonConverter(); private static final JtsPolyLineConverter JTS_POLYLINE_CONVERTER = new JtsPolyLineConverter(); private static final Logger logger = LoggerFactory.getLogger(RawAtlasSlicer.class); private static final double MINIMUM_CONSOLIDATE_THRESHOLD = 0.8; private static final long SLICING_DURATION_WARN = 10; // Logging constants private static final String MULTIPOLYGON_RELATION_INVALID_GEOMETRY = "Relation {} for Atlas {} had invalid multipolygon geometry {}!"; private static final String MULTIPOLYGON_RELATION_HAD_NO_SLICED_GEOMETRY = "Relation {} for Atlas {} had no valid sliced geometry!"; private static final String MULTIPOLYGON_RELATION_HAD_EQUIVALENT_SLICED_GEOMETRY = "Relation {} for Atlas {} had sliced geometry equal to original geometry, not slicing!"; private static final String MULTIPOLYGON_RELATION_SLICING_DURATION_EXCEEDED = "Relation {} for Atlas {} took {} to slice!"; private static final String MULTIPOLYGON_RELATION_INVALID_MEMBER_REMOVED = "Purging invalid member {} from relation {}"; private static final String MULTIPOLYGON_RELATION_INVALID_SLICED_GEOMETRY = "Relation {} sliced for country {} produced invalid geometry {}!"; private static final String MULTIPOLYGON_RELATION_OVERLAPPING_INNERS = "Relation {} for Atlas {} had overlapping inners, but slicing will continue!"; private static final String LINE_HAD_MULTIPOLYGON_SLICE = "Line {} for Atlas {} had multipolygon slicing result when sliced as polygon, will slice as line instead!"; private static final String LINE_SLICING_DURATION_EXCEEDED = "Line {} for Atlas {} took {} to slice!"; private static final String LINE_HAD_INVALID_GEOMETRY = "Line {} for Atlas {} had invalid geometry {}, removing instead of slicing!"; private static final String LINE_EXCEEDED_SLICING_IDENTIFIER_SPACE = "Country slicing exceeded maximum line identifier name space of {} for line {} for Atlas {}. It will be added as is, with two or more country codes"; private static final String STARTED_SLICING = "Starting slicing for Atlas {}"; private static final String FINISHED_SLICING = "Finished slicing for Atlas {} in {}"; private static final String STARTED_LINE_SLICING = "Starting line slicing for Atlas {}"; private static final String FINISHED_LINE_SLICING = "Finished line slicing for Atlas {} in {}"; private static final String STARTED_RELATION_SLICING = "Starting relation slicing for Atlas {}"; private static final String FINISHED_RELATION_SLICING = "Finished relation slicing for Atlas {} in {}"; private static final String STARTED_POINT_SLICING = "Starting point slicing for Atlas {}"; private static final String FINISHED_POINT_SLICING = "Finished point slicing for Atlas {} in {}"; private static final String STARTED_RELATION_FILTERING = "Starting relation filtering for Atlas {}"; private static final String FINISHED_RELATION_FILTERING = "Finished relation filtering for Atlas {} in {}"; private final Atlas inputAtlas; private final Shard initialShard; private final Predicate consolidatePredicate; private final Predicate isInCountry; private final Predicate isAtlasEdge; private final CountryBoundaryMap boundary; private final String shardOrAtlasName; private final String country; /** See {@link AtlasLoadingOption#isKeepAll} */ private final boolean keepAll; private final Map stagedAreas = new ConcurrentHashMap<>(); private final Map stagedRelations = new ConcurrentHashMap<>(); private final Map stagedLines = new ConcurrentHashMap<>(); private final Map stagedPoints = new ConcurrentHashMap<>(); private final AtlasLoadingOption loadingOption; private final Map> splitRelations = new ConcurrentHashMap<>(); private final Set changes = Collections .newSetFromMap(new ConcurrentHashMap()); private final Set pointsBelongingToEdge = Collections .newSetFromMap(new ConcurrentHashMap()); private final PreparedGeometryFactory preparer = new PreparedGeometryFactory(); /** * This constructor will build a RawAtlasSlicer for use on a single Atlas with no dynamic * expansion on Relations. Primarily for tests, please consider using the alternate constructor * for most use cases! * * @param loadingOption * An AtlasLoadingOption with a minimum of a CountryBoundaryMap included * @param startingAtlas * The raw Atlas to slice */ public RawAtlasSlicer(final AtlasLoadingOption loadingOption, final Atlas startingAtlas) { this(loadingOption, startingAtlas, SlippyTile.forName("1-1-1")); } /** * This constructor will build a RawAtlasSlicer for use on a single Atlas with no dynamic * expansion on Relations. Primarily for tests, please consider using the alternate constructor * for most use cases! * * @param loadingOption * An AtlasLoadingOption with a minimum of a CountryBoundaryMap included * @param startingAtlas * The raw Atlas to slice * @param initialShard * The initial shard to keep points for */ public RawAtlasSlicer(final AtlasLoadingOption loadingOption, final Atlas startingAtlas, final Shard initialShard) { this.inputAtlas = startingAtlas; this.initialShard = initialShard; this.loadingOption = loadingOption; this.country = loadingOption.getCountryCode(); this.consolidatePredicate = entity -> entity.getType().equals(ItemType.RELATION) && loadingOption.getRelationSlicingConsolidateFilter().test(entity); if (loadingOption.getCountryCode() == null || loadingOption.getCountryCode().isEmpty()) { this.isInCountry = entity -> true; } else { this.isInCountry = entity -> ISOCountryTag.isIn(loadingOption.getCountryCode()) .test(entity); } this.isAtlasEdge = entity -> loadingOption.getEdgeFilter().test(entity); this.boundary = loadingOption.getCountryBoundaryMap(); this.shardOrAtlasName = startingAtlas.metaData().getShardName() .orElse(startingAtlas.getName()); this.inputAtlas.areas().forEach( area -> this.stagedAreas.put(area.getIdentifier(), CompleteArea.from(area))); this.inputAtlas.points().forEach( point -> this.stagedPoints.put(point.getIdentifier(), CompletePoint.from(point))); this.inputAtlas.lines().forEach( line -> this.stagedLines.put(line.getIdentifier(), CompleteLine.from(line))); this.inputAtlas.relations().forEach(relation -> this.stagedRelations .put(relation.getIdentifier(), CompleteRelation.from(relation))); this.keepAll = loadingOption.isKeepAll(); } /** * Calculates the changes needed to slice the Atlas, then builds a ChangeAtlas out of that and * cuts it down to match the boundaries of the initial shard * * @return An Atlas with only entities that lay inside the original Shard bounds and matching * the country code set provided in the AtlasLoadingOption, all with ISOCountryTags * properly set */ public Atlas slice() { final Time overallTime = Time.now(); Time time = Time.now(); logger.info(STARTED_SLICING, this.shardOrAtlasName); logger.info(STARTED_LINE_SLICING, this.shardOrAtlasName); this.inputAtlas.areas().forEach(this::sliceArea); final Set linesToSlice = new HashSet<>(); linesToSlice.addAll(this.stagedLines.values()); linesToSlice.forEach(this::sliceLine); logger.info(FINISHED_LINE_SLICING, this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); time = Time.now(); logger.info(STARTED_RELATION_SLICING, this.shardOrAtlasName); this.inputAtlas.relationsLowerOrderFirst().forEach(relation -> { if (relation.isGeometric()) { sliceRelation(this.stagedRelations.get(relation.getIdentifier())); } }); logger.info(FINISHED_RELATION_SLICING, this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); time = Time.now(); logger.info(STARTED_POINT_SLICING, this.shardOrAtlasName); this.inputAtlas.points().forEach(this::slicePoint); logger.info(FINISHED_POINT_SLICING, this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); logger.info(STARTED_RELATION_FILTERING, this.shardOrAtlasName); this.inputAtlas.relationsLowerOrderFirst().forEach(relation -> { if (this.stagedRelations.containsKey(relation.getIdentifier()) && !Validators.hasValuesFor(this.stagedRelations.get(relation.getIdentifier()), ISOCountryTag.class)) { filterRelation(this.stagedRelations.get(relation.getIdentifier())); } }); logger.info(FINISHED_RELATION_FILTERING, this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); this.stagedLines.values() .forEach(line -> this.changes.add(FeatureChange.add(line, this.inputAtlas))); this.stagedPoints.values() .forEach(point -> this.changes.add(FeatureChange.add(point, this.inputAtlas))); this.stagedRelations.values().forEach( relation -> this.changes.add(FeatureChange.add(relation, this.inputAtlas))); this.stagedAreas.values() .forEach(area -> this.changes.add(FeatureChange.add(area, this.inputAtlas))); logger.info(FINISHED_SLICING, this.shardOrAtlasName, overallTime.elapsedSince().asMilliseconds()); return new ChangeAtlas(this.inputAtlas, new ChangeBuilder().addAll(this.changes).get()) { private static final long serialVersionUID = -1379576156041355921L; @Override public synchronized AtlasMetaData metaData() { // Override meta-data here so the country code is properly included. final AtlasMetaData metaData = super.metaData(); final var originalTags = metaData.getTags(); // Remove the country shards to keep old behavior where they were dropped, but keep // other tags originalTags.remove("countryShards"); return new AtlasMetaData(metaData.getSize(), false, metaData.getCodeVersion().orElse(null), metaData.getDataVersion().orElse(null), RawAtlasSlicer.this.country, RawAtlasSlicer.this.shardOrAtlasName, originalTags); } }; } /** * Given a new split Relation, filter its original Relation's members by the country code for * the new split Relation and add only them * * @param newRelation * A new Relation containing a subset of members based on country code * @param oldRelation * The original Relation for the new split Relation * @param countryMultiPolygon */ private void addCountryMembersToSplitRelation(final CompleteRelation newRelation, final CompleteRelation oldRelation, final PreparedPolygon countryMultiPolygon) { oldRelation.membersMatching(member -> member.getEntity().getType().equals(ItemType.LINE) && Validators.isOfSameType(this.stagedLines.get(member.getEntity().getIdentifier()), newRelation, ISOCountryTag.class)) .forEach(member -> { final CompleteLine lineForMember = this.stagedLines .get(member.getEntity().getIdentifier()); // quirk of line slicing-- check to see that either we never altered the // geometry, in which case it definitely belongs in the relation, or that it was // sliced but intersects the output multipolygon if (lineForMember.getTag(SyntheticGeometrySlicedTag.KEY).isEmpty() || countryMultiPolygon.covers( JTS_POLYLINE_CONVERTER.convert(lineForMember.asPolyLine()))) { newRelation.withAddedMember(lineForMember, member.getRole()); lineForMember.withAddedRelationIdentifier(newRelation.getIdentifier()); } lineForMember.withRemovedRelationIdentifier(oldRelation.getIdentifier()); }); oldRelation .membersMatching(member -> member.getEntity().getType().equals(ItemType.AREA) && Validators.isOfSameType( this.stagedAreas.get(member.getEntity().getIdentifier()), newRelation, ISOCountryTag.class) && (member.getRole().equalsIgnoreCase(Ring.OUTER.toString()) || this.stagedAreas.get(member.getEntity().getIdentifier()) .getTag(SyntheticGeometrySlicedTag.KEY).isEmpty())) .forEach(member -> { final CompleteArea areaForMember = this.stagedAreas .get(member.getEntity().getIdentifier()); newRelation.withAddedMember(areaForMember, member.getRole()); areaForMember.withAddedRelationIdentifier(newRelation.getIdentifier()); areaForMember.withRemovedRelationIdentifier(oldRelation.getIdentifier()); }); } /** * Given a Line and its slice segment, ensure that any new coordinates where the Line has been * split either have an existing Point that has an updated SyntheticBoundaryNodeTag.EXISTING * value, or a new Point is created with the SyntheticBoundaryNodeTag.NEW value * * @param line * A Line being sliced that meets the criteria for an Edge (see isAtlasEdge method) * @param slice * A slice for that Line */ private void addSyntheticBoundaryNodesForSlice(final Line line, final PolyLine slice) { if (!slice.first().equals(line.asPolyLine().first())) { final Iterable pointsAtFirstLocation = this.inputAtlas.pointsAt(slice.first()); if (Iterables.isEmpty(pointsAtFirstLocation)) { final EntityIdentifierGenerator pointIdentifierGenerator = new EntityIdentifierGenerator(); final SortedSet countries = new TreeSet<>(); final Map tags = new HashMap<>(); countries.addAll(Arrays.asList(this.boundary.getCountryCodeISO3(slice.first()) .getIso3CountryCode().split(ISOCountryTag.COUNTRY_DELIMITER))); tags.put(ISOCountryTag.KEY, String.join(ISOCountryTag.COUNTRY_DELIMITER, countries)); tags.put(SyntheticBoundaryNodeTag.KEY, SyntheticBoundaryNodeTag.YES.toString()); final CompletePoint syntheticBoundaryNode = new CompletePoint(1L, slice.first(), tags, new HashSet<>()); syntheticBoundaryNode.withIdentifier( pointIdentifierGenerator.generateIdentifier(syntheticBoundaryNode)); this.stagedPoints.put(syntheticBoundaryNode.getIdentifier(), syntheticBoundaryNode); } else { this.stagedPoints.get(pointsAtFirstLocation.iterator().next().getIdentifier()) .withAddedTag(SyntheticBoundaryNodeTag.KEY, SyntheticBoundaryNodeTag.EXISTING.toString()); } } if (!slice.last().equals(line.asPolyLine().last())) { final Iterable pointsAtLastLocation = this.inputAtlas.pointsAt(slice.last()); if (Iterables.isEmpty(pointsAtLastLocation)) { final EntityIdentifierGenerator pointIdentifierGenerator = new EntityIdentifierGenerator(); final SortedSet countries = new TreeSet<>(); final Map tags = new HashMap<>(); countries.addAll(Arrays.asList(this.boundary.getCountryCodeISO3(slice.last()) .getIso3CountryCode().split(ISOCountryTag.COUNTRY_DELIMITER))); tags.put(ISOCountryTag.KEY, String.join(ISOCountryTag.COUNTRY_DELIMITER, countries)); tags.put(SyntheticBoundaryNodeTag.KEY, SyntheticBoundaryNodeTag.YES.toString()); final CompletePoint syntheticBoundaryNode = new CompletePoint(1L, slice.last(), tags, new HashSet<>()); syntheticBoundaryNode.withIdentifier( pointIdentifierGenerator.generateIdentifier(syntheticBoundaryNode)); this.stagedPoints.put(syntheticBoundaryNode.getIdentifier(), syntheticBoundaryNode); } else { this.stagedPoints.get(pointsAtLastLocation.iterator().next().getIdentifier()) .withAddedTag(SyntheticBoundaryNodeTag.KEY, SyntheticBoundaryNodeTag.EXISTING.toString()); } } } /** * Checks the slices of a Line to see if we can process them succesfully. If there's just one * country, go ahead and tag the line, and if it's too many slices tag the line as well. * * @param slicesKeySet * The set of country codes for all of the slices * @param totalSlicesCount * The total number of slices for the line * @return false if the line shouldn't be sliced with these slices, true otherwise */ private boolean checkSlices(final Set slicesCountryCodes, final long totalSlicesCount, final AtlasEntity entity) { if (slicesCountryCodes.size() == 1) { final String countryCode = slicesCountryCodes.iterator().next(); if (entity instanceof Line) { this.stagedLines.get(entity.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCode); if (!this.isInCountry.test(this.stagedLines.get(entity.getIdentifier()))) { removeLine((Line) entity); } } else if (entity instanceof Area) { this.stagedAreas.get(entity.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCode); if (!this.isInCountry.test(this.stagedAreas.get(entity.getIdentifier()))) { removeArea((Area) entity); } } return false; } else if (totalSlicesCount >= AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT) { // this should be rare, but we can't slice the line if we don't have the identifier // space for the slices logger.error(LINE_EXCEEDED_SLICING_IDENTIFIER_SPACE, AbstractIdentifierFactory.IDENTIFIER_SCALE_DEFAULT, entity.getOsmIdentifier(), this.shardOrAtlasName); final String countryString = String.join(",", slicesCountryCodes); if (entity instanceof Line) { this.stagedLines.get(entity.getIdentifier()).withTags(entity.getTags()) .withAddedTag(ISOCountryTag.KEY, countryString); } else if (entity instanceof Area) { this.stagedAreas.get(entity.getIdentifier()).withTags(entity.getTags()) .withAddedTag(ISOCountryTag.KEY, countryString); } return false; } return true; } /** * Given an Area and its slices, create the new Area entities and put them in the stagedAreas * map * * @param area * The Area being sliced * @param slices * The map representing the portions of its geometry in each country, mapped to * country code */ private void createNewSlicedAreas(final Area area, final SortedMap> slices) { final CountrySlicingIdentifierFactory areaIdentifierFactory = new CountrySlicingIdentifierFactory( area.getIdentifier()); slices.keySet().forEach(countryCode -> { for (final org.locationtech.jts.geom.Polygon slice : slices.get(countryCode)) { final CompleteArea newAreaSlice = CompleteArea.from(area) .withIdentifier(areaIdentifierFactory.nextIdentifier()) .withAddedTag(SyntheticGeometrySlicedTag.KEY, SyntheticGeometrySlicedTag.YES.toString()) .withAddedTag(ISOCountryTag.KEY, countryCode); if (this.isInCountry.test(newAreaSlice)) { final Polygon newAreaGeometry = new Polygon(processSlice(slice, area)); newAreaSlice.withGeometry(newAreaGeometry); area.relations().forEach(relation -> newAreaSlice .withAddedRelationIdentifier(relation.getIdentifier())); this.stagedAreas.put(newAreaSlice.getIdentifier(), newAreaSlice); for (final Relation relation : area.relations()) { if (this.stagedRelations.containsKey(relation.getIdentifier())) { this.stagedRelations.get(relation.getIdentifier()) .withAddedMember(newAreaSlice, area); } } } } }); removeArea(area); } /** * Given a non-Area Line and its slices, create the new Line entities and put them in the * stagedLines map * * @param line * The Line being sliced * @param slices * The map representing the portions of its geometry in each country, mapped to * country code */ private void createNewSlicedLines(final Line line, // NOSONAR final SortedMap> slices) { final CountrySlicingIdentifierFactory lineIdentifierFactory = new CountrySlicingIdentifierFactory( line.getIdentifier()); slices.keySet().forEach(countryCode -> { for (final LineString slice : slices.get(countryCode)) { final CompleteLine newLineSlice = CompleteLine.from(line) .withIdentifier(lineIdentifierFactory.nextIdentifier()) .withAddedTag(SyntheticGeometrySlicedTag.KEY, SyntheticGeometrySlicedTag.YES.toString()) .withAddedTag(ISOCountryTag.KEY, countryCode); if (this.isInCountry.test(newLineSlice)) { newLineSlice.withGeometry(processSlice(slice, line)); if (this.isAtlasEdge.test(line)) { addSyntheticBoundaryNodesForSlice(line, newLineSlice.asPolyLine()); } this.stagedLines.put(newLineSlice.getIdentifier(), newLineSlice); for (final Relation relation : line.relations()) { if (this.stagedRelations.containsKey(relation.getIdentifier())) { final String role = this.stagedRelations.get(relation.getIdentifier()) .membersMatching(member -> member.getEntity().getType() .equals(ItemType.LINE) && member.getEntity().getIdentifier() == line .getIdentifier()) .iterator().next().getRole(); this.stagedRelations.get(relation.getIdentifier()) .withAddedMember(newLineSlice, role); } } } } }); removeLine(line); } /** * For any Relation that we can't slice (non-multipolygon, didn't meet the Relation predicate * criteria, or bad geometry), instead filter out any members that aren't in the country code * set, and update the ISOCountryTag for the Relation * * @param relation */ private void filterRelation(final CompleteRelation relation) { final Set countryList = new HashSet<>(); for (final RelationMember member : relation.members()) { final AtlasEntity stagedRelationMember = getStagedEntityForMember(member); if (stagedRelationMember == null) { relation.withRemovedMember(member.getEntity()); if (member.getEntity().getType().equals(ItemType.RELATION) && this.splitRelations.containsKey(member.getEntity().getIdentifier())) { this.splitRelations.get(member.getEntity().getIdentifier()).keySet() .forEach(countryCode -> { final CompleteRelation splitRelation = this.splitRelations .get(member.getEntity().getIdentifier()).get(countryCode); if (this.isInCountry.test(splitRelation)) { relation.withAddedMember(splitRelation, member.getRole()); countryList.add(countryCode); } }); } continue; } final Optional countryCodeTag = stagedRelationMember.getTag(ISOCountryTag.KEY); if (countryCodeTag.isEmpty()) { throw new CoreException( "Untagged country value for entity {} for relation {} for Atlas {}", stagedRelationMember, relation.getIdentifier(), this.shardOrAtlasName); } Collections.addAll(countryList, countryCodeTag.get().split(",")); } if (countryList.isEmpty()) { this.stagedRelations.remove(relation.getIdentifier()); this.changes.add(FeatureChange.remove(relation)); return; } // compute the value of the final country tag relation.withAddedTag(ISOCountryTag.KEY, ISOCountryTag.join(countryList)); if (relation.isGeometric() && relation.asMultiPolygon().isPresent()) { relation.withMultiPolygonGeometry(null); } } /** * Given a JTS geometry, return all Polygons from the country boundary map that intersect its * internal envelope * * @param targetGeometry * The geometry being queried * @return All Polygons from the country boundary map that intersect its internal envelope */ private Set getIntersectingBoundaryPolygons(final Geometry targetGeometry) { return this.boundary.query(targetGeometry.getEnvelopeInternal()).stream().distinct() .filter(preparedPolygon -> preparedPolygon.intersects(targetGeometry)) .collect(Collectors.toSet()); } /** * Given a RelationMember, find its staged CompleteEntity * * @param member * A RelationMember to find * @return Its staged CompleteEntity */ private AtlasEntity getStagedEntityForMember(final RelationMember member) { final long identifier = member.getEntity().getIdentifier(); if (member.getEntity() instanceof Point) { if (this.stagedPoints.containsKey(identifier)) { return this.stagedPoints.get(identifier); } return null; } else if (member.getEntity() instanceof Line) { if (this.stagedLines.containsKey(identifier)) { return this.stagedLines.get(identifier); } return null; } else if (member.getEntity() instanceof Area) { if (this.stagedAreas.containsKey(identifier)) { return this.stagedAreas.get(identifier); } return null; } else { if (this.stagedRelations.containsKey(identifier)) { return this.stagedRelations.get(identifier); } return null; } } /** * Checks two sets of geometries to see if one contains any geometries that are covered by or * equals to any geometries in the second set * * @param geometries * A Set of Geometries to check * @param geometriesComparison * A second Set of Geometries to compare to * @return True if any geometry in geometries is equal to or is covered by a geometry in * geometryComparison, false otherwise */ private boolean isCoveredBy(final Set geometries, final Set geometriesComparison) { for (final PreparedGeometry comparisonGeometry : geometriesComparison) { for (final Geometry geometry : geometries) { if (geometry.equals(comparisonGeometry.getGeometry()) || comparisonGeometry.coveredBy(geometry) || comparisonGeometry.intersects(geometry) && geometry .intersection(comparisonGeometry.getGeometry()).getDimension() > 0) { return true; } } } return false; } /** * A filter to ensure trivial pieces of geometry aren't preserved from the slicing operation * * @param geometry * The geometry to check * @return True if the geometry is valid and larger than either the * CountryBoundaryMap.LINE_BUFFER or CountryBoundaryMap.AREA_BUFFER */ private boolean isSignificantGeometry(final Geometry original, final Geometry clipped) { if (!clipped.isValid() && logger.isWarnEnabled()) { logger.warn("Found invalid geometry {} during slicing Atlas {}", clipped.toText(), this.shardOrAtlasName); return false; } if (clipped.getDimension() == 1) { return clipped.getLength() > LINE_BUFFER || clipped.getLength() / original.getLength() > BUFFER_PERCENTAGE; } else if (clipped.getDimension() == 2) { return clipped.getArea() > AREA_BUFFER || clipped.getArea() / original.getArea() > BUFFER_PERCENTAGE; } return false; } /** * Given a slice for a Line entity, construct an appropriate PolyLine for the new sliced Line * member. In the case of a Polygon (i.e. Area), the PolyLine will be constructed such that the * winding represents the winding of the original entity * * @param slice * The slice for a Line entity * @param entity * The Line being sliced * @return The best PolyLine to represent that slice for the Line */ private PolyLine processSlice(final Geometry slice, final AtlasEntity entity) { PolyLine polylineForSlice; if (slice instanceof LineString) { polylineForSlice = JTS_POLYLINE_CONVERTER.backwardConvert((LineString) slice); } else if (slice instanceof org.locationtech.jts.geom.Polygon) { polylineForSlice = JTS_POLYLINE_CONVERTER .backwardConvert(((org.locationtech.jts.geom.Polygon) slice).getExteriorRing()); } else { throw new CoreException("Unexpected geometry when slicing line {} for Atlas {}", entity.getOsmIdentifier(), this.shardOrAtlasName); } // JTS frequently reverses the winding during slicing-- this checks the winding of the // original geometry, and reverse the winding of the slice if needed if (entity instanceof Area) { final boolean originalClockwise = ((Area) entity).asPolygon().isClockwise(); final boolean sliceClockwise = new Polygon(polylineForSlice.truncate(0, 1)) .isClockwise(); if (originalClockwise != sliceClockwise) { polylineForSlice = polylineForSlice.reversed(); } } return polylineForSlice; } /** * Given a geometric Relation, purge any members that don't meet the criteria in the OSM * specification * * @param relation * The Relation to check */ private void purgeInvalidGeometricRelationMembers(final CompleteRelation relation) { final Set memberIdentifiersRemoved = new HashSet<>(); relation.membersMatching(member -> member.getEntity().getType() != ItemType.LINE || !(member.getRole().equals(RelationTypeTag.MULTIPOLYGON_ROLE_OUTER) || member.getRole().equals(RelationTypeTag.MULTIPOLYGON_ROLE_INNER))) .forEach(invalidMember -> { final long identifier = invalidMember.getEntity().getIdentifier(); logger.warn(MULTIPOLYGON_RELATION_INVALID_MEMBER_REMOVED, invalidMember, relation.getOsmIdentifier()); if (invalidMember.getEntity().getType().equals(ItemType.LINE)) { // if it was in the atlas, remove the OG version, else it must have been // sliced so remove that if (this.inputAtlas.line(invalidMember.getEntity().getIdentifier()) != null) { relation.withRemovedMember(this.inputAtlas .line(invalidMember.getEntity().getIdentifier())); } else { relation.withRemovedMember(this.stagedLines .get(invalidMember.getEntity().getIdentifier())); } this.stagedLines.get(identifier) .withRemovedRelationIdentifier(relation.getIdentifier()); memberIdentifiersRemoved.add(Long.toString(identifier)); } else if (invalidMember.getEntity().getType().equals(ItemType.AREA) && this.stagedAreas.containsKey(identifier) && this.stagedAreas.get(identifier) .getTag(SyntheticGeometrySlicedTag.KEY).isPresent() && !(invalidMember.getRole() .equals(RelationTypeTag.MULTIPOLYGON_ROLE_OUTER) || invalidMember.getRole() .equals(RelationTypeTag.MULTIPOLYGON_ROLE_INNER))) { relation.withRemovedMember(invalidMember.getEntity()); this.stagedAreas.get(identifier) .withRemovedRelationIdentifier(relation.getIdentifier()); } else if (invalidMember.getEntity().getType().equals(ItemType.POINT)) { if (Validators.isOfType(relation, RelationTypeTag.class, RelationTypeTag.BOUNDARY) && (invalidMember.getRole().equals("admin_centre") || invalidMember.getRole().equals("label"))) { // keep admin centre or lable nodes } else { relation.withRemovedMember(invalidMember.getEntity()); this.stagedPoints.get(identifier) .withRemovedRelationIdentifier(relation.getIdentifier()); memberIdentifiersRemoved.add(Long.toString(identifier)); } } else { if (invalidMember.getEntity().getType().equals(ItemType.RELATION) && this.stagedRelations.containsKey(identifier)) { relation.withRemovedMember(invalidMember.getEntity()); this.stagedRelations.get(identifier) .withRemovedRelationIdentifier(relation.getIdentifier()); memberIdentifiersRemoved.add(Long.toString(identifier)); } else if (invalidMember.getEntity().getType().equals(ItemType.RELATION) && this.splitRelations.containsKey(identifier)) { relation.withRemovedMember(invalidMember.getEntity()); this.splitRelations.get(identifier).values().forEach( childRelation -> childRelation.withRemovedRelationIdentifier( relation.getIdentifier())); memberIdentifiersRemoved.add(Long.toString(identifier)); } } }); if (!memberIdentifiersRemoved.isEmpty()) { relation.withAddedTag(SyntheticInvalidMultiPolygonRelationMembersRemovedTag.KEY, String.join( SyntheticInvalidMultiPolygonRelationMembersRemovedTag.MEMBER_DELIMITER, memberIdentifiersRemoved)); } } /** * Remove an Area from the Atlas, and remove it from any Relations that may contain it * * @param area * The Line to remove */ private void removeArea(final Area area) { final CompleteArea removedArea = this.stagedAreas.remove(area.getIdentifier()); this.changes.add(FeatureChange.remove(removedArea, this.inputAtlas)); removedArea.relationIdentifiers().forEach(relationIdentifier -> { if (this.stagedRelations.containsKey(relationIdentifier)) { this.stagedRelations.get(relationIdentifier).withRemovedMember(removedArea); } }); } /** * Remove a Line from the Atlas, and remove it from any Relations that may contain it * * @param line * The Line to remove */ private void removeLine(final Line line) { final CompleteLine removedLine = this.stagedLines.remove(line.getIdentifier()); if (this.inputAtlas.line(line.getIdentifier()) == null) { // in rare cases, we're slicing a line that technically never existed in the original // atlas, e.g. multipolygon areas now being sliced as lines, or areas that failed // polygonal slicing. in these cases, simple remove the staged line, don't make a // FeatureChange to remove the line from the Atlas removedLine.relationIdentifiers().forEach(relationIdentifier -> { if (this.stagedRelations.containsKey(relationIdentifier)) { this.stagedRelations.get(relationIdentifier).withRemovedMember(removedLine); } }); return; } this.changes.add(FeatureChange.remove(removedLine, this.inputAtlas)); removedLine.relationIdentifiers().forEach(relationIdentifier -> { if (this.stagedRelations.containsKey(relationIdentifier)) { this.stagedRelations.get(relationIdentifier).withRemovedMember(removedLine); } }); } private org.locationtech.jts.geom.MultiPolygon removeOsmValidOverlappingInners( final Relation relation, final org.locationtech.jts.geom.MultiPolygon multipolygon) { final Set slicedInnerLines = new HashSet<>(); relation.membersMatching( member -> member.getRole().equals(RelationTypeTag.MULTIPOLYGON_ROLE_INNER) && member.getEntity().getType().equals(ItemType.LINE)) .forEach(member -> { final Line innerLine = this.stagedLines.get(member.getEntity().getIdentifier()); if (innerLine.getTag(SyntheticGeometrySlicedTag.KEY).isPresent()) { slicedInnerLines.add(this.preparer .create(JTS_POLYLINE_CONVERTER.convert(innerLine.asPolyLine()))); } }); final org.locationtech.jts.geom.Polygon[] modifiedPolygons = new org.locationtech.jts.geom.Polygon[multipolygon .getNumGeometries()]; for (int i = 0; i < multipolygon.getNumGeometries(); i++) { final org.locationtech.jts.geom.Polygon currentPolygon = (org.locationtech.jts.geom.Polygon) multipolygon .getGeometryN(i); final List holes = new ArrayList<>(); for (int j = 0; j < currentPolygon.getNumInteriorRing(); j++) { final PreparedGeometry currentInner = this.preparer .create(currentPolygon.getInteriorRingN(j)); boolean remove = false; for (int k = j + 1; k < currentPolygon.getNumInteriorRing(); k++) { final Geometry comparisonInner = currentPolygon.getInteriorRingN(k); if (currentInner.intersects(comparisonInner)) { final Set inners = new HashSet<>(); inners.add(currentInner.getGeometry()); inners.add(comparisonInner); if (isCoveredBy(inners, slicedInnerLines)) { remove = false; break; } remove = true; } } if (!remove) { holes.add((LinearRing) currentInner.getGeometry()); } } modifiedPolygons[i] = new org.locationtech.jts.geom.Polygon( currentPolygon.getExteriorRing(), holes.toArray(new LinearRing[holes.size()]), JtsPrecisionManager.getGeometryFactory()); } return new org.locationtech.jts.geom.MultiPolygon(modifiedPolygons, JtsPrecisionManager.getGeometryFactory()); } /** * Slice a Line that qualifies as an Area by converting it to 2d geometry, calculating its * slices, and creating the new sliced Lines. If it belongs to just one country or cannot be * sliced, update the ISOCountryTag appropriately instead * * @param line * The Line to slice as an Area */ private void sliceArea(final Area area) { final Time time = Time.now(); final org.locationtech.jts.geom.Polygon jtsPolygon = JTS_POLYGON_CONVERTER .convert(area.asPolygon()); final Set intersectingBoundaryPolygons = getIntersectingBoundaryPolygons( jtsPolygon); if (intersectingBoundaryPolygons.size() == 1 || CountryBoundaryMap.isSameCountry(intersectingBoundaryPolygons)) { final String countryCode = CountryBoundaryMap.getGeometryProperty( intersectingBoundaryPolygons.iterator().next().getGeometry(), ISOCountryTag.KEY); this.stagedAreas.get(area.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCode); if (!this.isInCountry.test(this.stagedAreas.get(area.getIdentifier()))) { removeArea(area); } return; } // only check this once we know it's likely to be sliced if (jtsPolygon.isEmpty() || !jtsPolygon.isValid()) { if (logger.isErrorEnabled()) { logger.error(LINE_HAD_INVALID_GEOMETRY, area.getOsmIdentifier(), this.shardOrAtlasName, jtsPolygon.toText()); } final SortedSet countries = new TreeSet<>(); intersectingBoundaryPolygons.forEach(polygon -> countries.add(CountryBoundaryMap .getGeometryProperty(polygon.getGeometry(), ISOCountryTag.KEY))); final String countryCodes = String.join(",", countries); this.stagedAreas.get(area.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCodes); this.stagedAreas.get(area.getIdentifier()).withAddedTag(SyntheticInvalidGeometryTag.KEY, SyntheticInvalidGeometryTag.YES.toString()); return; } final SortedMap> slices; try { slices = slicePolygonGeometry(area.getOsmIdentifier(), jtsPolygon, intersectingBoundaryPolygons); } catch (final CoreException exception) { logger.error(LINE_HAD_MULTIPOLYGON_SLICE, area.getOsmIdentifier(), this.shardOrAtlasName); final Set relationIds = new HashSet<>(); area.relations().forEach(relation -> relationIds.add(relation.getIdentifier())); final CompleteLine lineFromArea = new CompleteLine(area.getIdentifier(), JTS_POLYLINE_CONVERTER.backwardConvert( JTS_POLYGON_CONVERTER.convert(area.asPolygon()).getExteriorRing()), area.getTags(), relationIds); lineFromArea.withGeometricRelationIdentifiers( CompleteArea.from(area).geometricRelationIdentifiers()); area.relations().forEach(relation -> this.stagedRelations.get(relation.getIdentifier()) .withAddedMember(lineFromArea, area)); removeArea(area); this.stagedLines.put(lineFromArea.getIdentifier(), lineFromArea); return; } long numSlices = 0; for (final Set sliceSet : slices.values()) { numSlices += sliceSet.size(); } logger.info("Way {} was sliced into {} slices", area.getOsmIdentifier(), numSlices); if (!checkSlices(slices.keySet(), numSlices, area)) { return; } createNewSlicedAreas(area, slices); if (time.elapsedSince().isMoreThan(Duration.minutes(SLICING_DURATION_WARN))) { logger.warn(LINE_SLICING_DURATION_EXCEEDED, area.getOsmIdentifier(), this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); } } /** * Given a geometry and a set of intersecting country boundary Polygons, calculate the portion * of that geometry contained by each boundary polygon and return those mapped to country code * * @param geometry * The JTS geometry to slice * @param countryBoundaryPolygons * All country boundary polygons intersecting it * @return A map of country codes to the portions of JTS geometry inside those country boundary * polygons */ private Map> sliceGeometry( final Geometry geometry, final Set countryBoundaryPolygons, final long identifier) { final Set filteredPieces = new HashSet<>(); final Map> results = new HashMap<>(); for (final PreparedPolygon boundaryPolygon : countryBoundaryPolygons) { final String countryCode = CountryBoundaryMap .getGeometryProperty(boundaryPolygon.getGeometry(), ISOCountryTag.KEY); // occasionally, the geometry's internal envelope will intersect multiple boundary // polygons but the geometry doesn't. in this case, just tag the geometry entirely if (boundaryPolygon.contains(geometry)) { CountryBoundaryMap.setGeometryProperty(geometry, ISOCountryTag.KEY, countryCode); results.clear(); results.put(countryCode, new HashSet<>()); results.get(countryCode).add(geometry); return results; } else if (boundaryPolygon.intersects(geometry)) { if (!results.containsKey(countryCode)) { results.put(countryCode, new HashSet<>()); } final Geometry clipped = OverlayNG.overlay(geometry, boundaryPolygon.getGeometry(), OverlayNG.INTERSECTION, JtsPrecisionManager.getPrecisionModel()); if (clipped instanceof GeometryCollection) { CountryBoundaryMap.geometries((GeometryCollection) clipped) .filter(result -> isSignificantGeometry(geometry, result)) .forEach(result -> { CountryBoundaryMap.setGeometryProperty(result, ISOCountryTag.KEY, countryCode); results.get(countryCode).add(result); }); filteredPieces .addAll(CountryBoundaryMap.geometries((GeometryCollection) clipped) .filter(result -> !isSignificantGeometry(geometry, result)) .collect(Collectors.toSet())); } else if (isSignificantGeometry(geometry, clipped)) { CountryBoundaryMap.setGeometryProperty(clipped, ISOCountryTag.KEY, countryCode); results.get(countryCode).add(clipped); } else { filteredPieces.add(clipped); } } } if (!filteredPieces.isEmpty()) { if (geometry.getDimension() == 1) { long length = 0; for (final Geometry filtered : filteredPieces) { length += filtered.getLength(); } logger.warn("Removed {} slices from way {} for being trivial, summing to {} length", filteredPieces.size(), identifier, length); } else if (geometry.getDimension() == 2) { long area = 0; for (final Geometry filtered : filteredPieces) { area += filtered.getArea(); } logger.warn( "Removed {} slices from OSM entity {} for being trivial, summing to {} area", filteredPieces.size(), identifier, area); } } return results; } /** * Slice a Line by converting it to 1d geometry, calculating its slices, and creating the new * sliced Lines. If it belongs to just one country or cannot be sliced, update the ISOCountryTag * appropriately instead * * @param line * The Line to slice as an Line */ private void sliceLine(final Line line) { final Time time = Time.now(); final LineString jtsLine = JTS_POLYLINE_CONVERTER.convert(line.asPolyLine()); if (this.isAtlasEdge.test(line)) { final Set selfIntersections = line.asPolyLine().selfIntersects() ? line.asPolyLine().selfIntersections() : new HashSet<>(); line.forEach(location -> { if (line.asPolyLine().first().equals(location) || line.asPolyLine().last().equals(location) || line.isClosed() || selfIntersections.contains(location) || !this.initialShard.bounds().fullyGeometricallyEncloses(location) || AtlasSectionProcessor.shouldSectionAtLocation(location, line, this.loadingOption, this.inputAtlas)) { this.inputAtlas.pointsAt(location).forEach( point -> this.pointsBelongingToEdge.add(point.getIdentifier())); } }); } final Set intersectingBoundaryPolygons = getIntersectingBoundaryPolygons( jtsLine); if (CountryBoundaryMap.isSameCountry(intersectingBoundaryPolygons)) { final String countryCode = CountryBoundaryMap.getGeometryProperty( intersectingBoundaryPolygons.iterator().next().getGeometry(), ISOCountryTag.KEY); this.stagedLines.get(line.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCode); if (!this.isInCountry.test(this.stagedLines.get(line.getIdentifier()))) { removeLine(line); } return; } // we only want to do this validation if we're going to slice it if (jtsLine.isEmpty() || !jtsLine.isValid()) { if (logger.isErrorEnabled()) { logger.error(LINE_HAD_INVALID_GEOMETRY, line.getOsmIdentifier(), this.shardOrAtlasName, jtsLine.toText()); } final SortedSet countries = new TreeSet<>(); intersectingBoundaryPolygons.forEach(polygon -> countries.add(CountryBoundaryMap .getGeometryProperty(polygon.getGeometry(), ISOCountryTag.KEY))); final String countryCodes = String.join(",", countries); this.stagedLines.get(line.getIdentifier()).withAddedTag(ISOCountryTag.KEY, countryCodes); this.stagedLines.get(line.getIdentifier()).withAddedTag(SyntheticInvalidGeometryTag.KEY, SyntheticInvalidGeometryTag.YES.toString()); return; } final SortedMap> slices = sliceLineStringGeometry(jtsLine, intersectingBoundaryPolygons, line.getOsmIdentifier()); long numSlices = 0; for (final Set sliceSet : slices.values()) { numSlices += sliceSet.size(); } if (this.isAtlasEdge.test(line)) { logger.info("Edge {} was sliced into {} slices", line.getOsmIdentifier(), numSlices); } else { logger.info("Way {} was sliced into {} slices", line.getOsmIdentifier(), numSlices); } if (!checkSlices(slices.keySet(), numSlices, line)) { return; } createNewSlicedLines(line, slices); if (time.elapsedSince().isMoreThan(Duration.minutes(SLICING_DURATION_WARN))) { logger.warn(LINE_SLICING_DURATION_EXCEEDED, line.getOsmIdentifier(), this.shardOrAtlasName, time.elapsedSince().asMilliseconds()); } } /** * Take a LineString and find its slices, also attempt to merge all LineSlices for a country * together to reduce artifacting * * @param line * The Line being sliced * @param intersectingBoundaryPolygons * All intersecting country boundary polygons * @return A SortedMap of country codes to the portions of the Line inside those country * boundary polygons */ @SuppressWarnings("unchecked") private SortedMap> sliceLineStringGeometry(final LineString line, final Set intersectingBoundaryPolygons, final long identifier) { final Map> currentResults = sliceGeometry(line, intersectingBoundaryPolygons, identifier); final SortedMap> results = new TreeMap<>(); for (final Map.Entry> entry : currentResults .entrySet()) { final Set lineSlices = new HashSet<>(); final LineMerger lineMerger = new LineMerger(); entry.getValue().stream() .filter(polygon -> polygon instanceof org.locationtech.jts.geom.LineString) .forEach(lineMerger::add); final String countryCode = entry.getKey(); lineMerger.add(lineSlices); lineMerger.getMergedLineStrings().forEach(mergedLineSlice -> { if (mergedLineSlice instanceof LineString) { lineSlices.add((LineString) mergedLineSlice); } }); results.put(countryCode, lineSlices); } return results; } /** * Take a MultiPolygon and find its slices, while also checking for validity and geometric * consistency (i.e. only Polygonal results) * * @param identifier * The identifier for the Relation being sliced * @param geometry * The multipolygon being sliced * @param intersectingBoundaryPolygons * All intersecting country boundary polygons * @return A SortedMap of country codes to the portions of the MultiPolygon inside those country * boundary polygons */ private SortedMap sliceMultiPolygonGeometry( final long identifier, final org.locationtech.jts.geom.MultiPolygon geometry, final Set intersectingBoundaryPolygons) { final Map> currentResults = sliceGeometry( geometry, intersectingBoundaryPolygons, identifier); final SortedMap results = new TreeMap<>(); for (final Map.Entry> entry : currentResults .entrySet()) { if (entry.getValue().size() == 1 && entry.getValue().iterator() .next() instanceof org.locationtech.jts.geom.MultiPolygon) { results.put(entry.getKey(), (org.locationtech.jts.geom.MultiPolygon) entry .getValue().iterator().next()); continue; } final Set polygonClippings = new HashSet<>(); entry.getValue().stream() .filter(polygon -> polygon instanceof org.locationtech.jts.geom.Polygon) .forEach(polygon -> polygonClippings .add((org.locationtech.jts.geom.Polygon) polygon)); final String countryCode = entry.getKey(); final org.locationtech.jts.geom.MultiPolygon multipolygon = new org.locationtech.jts.geom.MultiPolygon( polygonClippings.toArray( new org.locationtech.jts.geom.Polygon[polygonClippings.size()]), JtsPrecisionManager.getGeometryFactory()); if (multipolygon.isEmpty() || !multipolygon.isValid()) { if (logger.isErrorEnabled()) { logger.warn(MULTIPOLYGON_RELATION_INVALID_SLICED_GEOMETRY, identifier, countryCode, multipolygon.toText()); } } else { results.put(countryCode, multipolygon); } } return results; } /** * Given a point, either remove it from the Atlas if it has no pre-existing tags and doesn't * belong to an Edge, OR update its ISOCountryTag value * * @param point * The point to slice */ private void slicePoint(final Point point) { if (point.getOsmTags().isEmpty() && !this.pointsBelongingToEdge.contains(point.getIdentifier()) && this.stagedPoints .get(point.getIdentifier()).getTag(SyntheticBoundaryNodeTag.KEY).isEmpty() && point.relations().isEmpty() && !this.keepAll) { // we care about a point if and only if it has pre-existing OSM tags OR it belongs // to a future edge OR we are keeping all points for QC this.stagedPoints.remove(point.getIdentifier()); this.changes.add(FeatureChange.remove(CompletePoint.shallowFrom(point))); } else { final CompletePoint updatedPoint = this.stagedPoints.get(point.getIdentifier()); final SortedSet countries = new TreeSet<>(); countries.addAll(Arrays.asList(this.boundary.getCountryCodeISO3(point.getLocation()) .getIso3CountryCode().split(ISOCountryTag.COUNTRY_DELIMITER))); updatedPoint.withAddedTag(ISOCountryTag.KEY, String.join(ISOCountryTag.COUNTRY_DELIMITER, countries)); if (countries.size() > 1) { updatedPoint.withAddedTag(SyntheticBoundaryNodeTag.KEY, SyntheticBoundaryNodeTag.EXISTING.toString()); } if (!this.isInCountry.test(updatedPoint) && !this.keepAll) { this.stagedPoints.remove(point.getIdentifier()); this.changes.add(FeatureChange.remove(updatedPoint, this.inputAtlas)); } } } /** * Take a polygon and find its slices, while also checking for validity and geometric * consistency (i.e. only Polygonal results) * * @param identifier * the OSM identifier for the Line being sliced * @param polygon * The Polygon being sliced * @param intersectingBoundaryPolygons * All intersecting country boundary polygons * @return A SortedMap of country codes to the portions of the Polygon inside those country * boundary polygons */ private SortedMap> slicePolygonGeometry( final long identifier, final org.locationtech.jts.geom.Polygon polygon, final Set intersectingBoundaryPolygon) { final Map> currentResults = sliceGeometry(polygon, intersectingBoundaryPolygon, identifier); final SortedMap> results = new TreeMap<>(); for (final Map.Entry> entry : currentResults.entrySet()) { final String countryCode = entry.getKey(); final Set slicedPolygons = new HashSet<>(); entry.getValue().forEach(geometry -> { if (geometry instanceof org.locationtech.jts.geom.Polygon) { // this can happen when a country boundary introduces a hole into a polygon that // was previously simple. in this case, we *must* discard the result. we surface // the log as an error so it should be easy to find if (((org.locationtech.jts.geom.Polygon) geometry).getNumInteriorRing() > 0) { throw new CoreException( "Line {} for Atlas {} had multipolygon geometry {}!", identifier, this.shardOrAtlasName, geometry.toText()); } else { slicedPolygons.add((org.locationtech.jts.geom.Polygon) geometry); } } }); results.put(countryCode, slicedPolygons); } return results; } /** * Slice a geometric Relation by constructing its geometry out of the valid raw Atlas members, * calculating its slices, and creating the new sliced Relations with valid geometry. If it * belongs to just one country or cannot be sliced, update the ISOCountryTag appropriately * instead * * @param Relation * The Relation to slice geometrically */ private void sliceRelation(final CompleteRelation relation) { final Time time = Time.now(); purgeInvalidGeometricRelationMembers(relation); final Optional geom = relation.asMultiPolygon(); if (geom.isEmpty()) { return; } org.locationtech.jts.geom.MultiPolygon jtsMp = geom.get(); // Check to see if the relation is one country only, short circuit if it is final Set polygons = this.boundary.query(jtsMp.getEnvelopeInternal()) .stream().distinct().collect(Collectors.toSet()); if (CountryBoundaryMap.isSameCountry(polygons)) { final String country = CountryBoundaryMap.getGeometryProperty( polygons.iterator().next().getGeometry(), ISOCountryTag.KEY); // just tag with the country code and move on, no slicing needed relation.withAddedTag(ISOCountryTag.KEY, country); relation.withMultiPolygonGeometry(jtsMp); return; } if (!jtsMp.isValid()) { jtsMp = removeOsmValidOverlappingInners(relation, jtsMp); if (!jtsMp.isValid()) { if (logger.isErrorEnabled()) { logger.error(MULTIPOLYGON_RELATION_INVALID_GEOMETRY, relation.getOsmIdentifier(), this.shardOrAtlasName, jtsMp.toText()); } relation.withAddedTag(SyntheticInvalidGeometryTag.KEY, SyntheticInvalidGeometryTag.YES.toString()); return; } else { logger.warn(MULTIPOLYGON_RELATION_OVERLAPPING_INNERS, relation.getOsmIdentifier(), this.shardOrAtlasName); } } final SortedMap clippedMultiPolygons = sliceMultiPolygonGeometry( relation.getIdentifier(), jtsMp, polygons); if (clippedMultiPolygons.isEmpty()) { logger.error(MULTIPOLYGON_RELATION_HAD_NO_SLICED_GEOMETRY, relation.getOsmIdentifier(), this.shardOrAtlasName); return; } final SortedMap preparedClippedPolygons = new TreeMap<>(); for (final Map.Entry entry : clippedMultiPolygons .entrySet()) { final String country = entry.getKey(); final org.locationtech.jts.geom.MultiPolygon countryMultipolygon = entry.getValue(); if (countryMultipolygon.equals(jtsMp)) { logger.info(MULTIPOLYGON_RELATION_HAD_EQUIVALENT_SLICED_GEOMETRY, relation.getOsmIdentifier(), this.shardOrAtlasName); // just tag with the country code and move on, no slicing needed relation.withAddedTag(ISOCountryTag.KEY, country); relation.withMultiPolygonGeometry(jtsMp); return; } preparedClippedPolygons.put(country, (PreparedPolygon) this.preparer.create(countryMultipolygon)); } if (this.consolidatePredicate.test(relation)) { final Set removedCountries = new HashSet<>(); double size = 0; double percentSize = 0; org.locationtech.jts.geom.MultiPolygon largest = null; for (final PreparedPolygon polygon : preparedClippedPolygons.values()) { if (largest == null || polygon.getGeometry().getArea() > largest.getArea()) { largest = (org.locationtech.jts.geom.MultiPolygon) polygon.getGeometry(); } } if (largest.getArea() / jtsMp.getArea() > MINIMUM_CONSOLIDATE_THRESHOLD) { final Set countrySlices = new HashSet<>(); countrySlices.addAll(preparedClippedPolygons.keySet()); for (final String country : countrySlices) { if (!preparedClippedPolygons.get(country).getGeometry().equals(largest)) { size += preparedClippedPolygons.get(country).getGeometry().getArea(); percentSize += preparedClippedPolygons.get(country).getGeometry().getArea() / jtsMp.getArea() * PERCENTAGE; logger.info( "Removing sliced relation {} with size {} and percentage size {}", relation.getOsmIdentifier(), preparedClippedPolygons.get(country).getGeometry().getArea(), preparedClippedPolygons.get(country).getGeometry().getArea() / jtsMp.getArea()); preparedClippedPolygons.remove(country); removedCountries.add(country); } } logger.info( "Consolidated relation {} to country {}, ignoring countries {} with sliced size {} and percent size {}", relation.getOsmIdentifier(), preparedClippedPolygons.keySet().iterator().next(), String.join(ISOCountryTag.COUNTRY_DELIMITER, removedCountries), size, percentSize); } else { logger.info( "Relation {} met tagging criteria for consolidation, but largest piece with size {} did not meet percentage threshold {}", relation.getOsmIdentifier(), largest.getArea() / jtsMp.getArea(), MINIMUM_CONSOLIDATE_THRESHOLD); } } // because this is a SortedSet, iterating over the keys guarantees that we will split our // relation into a deterministic identifier each time. note that this is why we put all // countries that this relation spans into the key set for the map, even if it's associated // with empty polygons. that way, if a relation spans country A and country B, country A // will get 001000 and country B will get 002000, no matter what the country parameter for // slicing is final CountrySlicingIdentifierFactory relationIdentifierFactory = new CountrySlicingIdentifierFactory( relation.getIdentifier()); final List newRelationIds = new ArrayList<>(); final Map newRelations = new HashMap<>(); preparedClippedPolygons.keySet().forEach(countryCode -> { final CompleteRelation newRelation = CompleteRelation.shallowFrom(relation) .withIdentifier(relationIdentifierFactory.nextIdentifier()) .withTags(relation.getTags()) .withAddedTag(SyntheticGeometrySlicedTag.KEY, SyntheticGeometrySlicedTag.YES.toString()) .withAddedTag(ISOCountryTag.KEY, countryCode) .withRelationIdentifiers(relation.relationIdentifiers()) .withOsmRelationIdentifier(relation.getOsmIdentifier()); newRelationIds.add(newRelation.getIdentifier()); if (this.isInCountry.test(newRelation)) { final Geometry clipped = preparedClippedPolygons.get(countryCode).getGeometry(); final MultiPolygon slicedGeometry; if (clipped instanceof org.locationtech.jts.geom.Polygon) { slicedGeometry = new MultiPolygon( new org.locationtech.jts.geom.Polygon[] { (org.locationtech.jts.geom.Polygon) preparedClippedPolygons .get(countryCode).getGeometry() }, JtsPrecisionManager.getGeometryFactory()); } else if (clipped instanceof MultiPolygon) { slicedGeometry = (MultiPolygon) preparedClippedPolygons.get(countryCode) .getGeometry(); } else { throw new CoreException( "Cannot have non-multipolygon relation geometry {} for relation {}", clipped, relation.getIdentifier()); } newRelation.withMultiPolygonGeometry(slicedGeometry); addCountryMembersToSplitRelation(newRelation, relation, preparedClippedPolygons.get(countryCode)); if (newRelation.members() != null && !newRelation.members().isEmpty()) { newRelations.put(countryCode, newRelation); } } }); final HashMap relationByCountry = new HashMap<>(); newRelations.values().forEach(newRelation -> { // update so each split relation knows about the other split relations newRelation.withAllRelationsWithSameOsmIdentifier(new ArrayList<>()); newRelation.withAllKnownOsmMembers(relation.getBean()); relationByCountry.put(newRelation.getTag(ISOCountryTag.KEY).get(), newRelation); this.stagedRelations.put(newRelation.getIdentifier(), newRelation); }); // remove the old relation this.changes.add(FeatureChange.remove(relation, this.inputAtlas)); this.stagedRelations.remove(relation.getIdentifier()); this.splitRelations.put(relation.getIdentifier(), relationByCountry); if (time.elapsedSince().isMoreThan(Duration.minutes(SLICING_DURATION_WARN))) { logger.warn(MULTIPOLYGON_RELATION_SLICING_DURATION_EXCEEDED, relation.getOsmIdentifier(), this.shardOrAtlasName, time.elapsedSince()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/routing/AStarRouter.java ================================================ package org.openstreetmap.atlas.geography.atlas.routing; import java.util.HashSet; import java.util.PriorityQueue; import java.util.Set; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * Router that follows the simple A* algorithm * * @author matthieun */ public class AStarRouter extends AbstractRouter { /** * A heuristic for an {@link AStarRouter} * * @author matthieun */ public interface Heuristic { /** * Compute the cost of a candidate, given the start and end point. * * @param start * The start of the route * @param candidate * The candidate point of the route * @param end * The end of the route * @return The cost */ double cost(Node start, Node candidate, Node end); } /** * A candidate for an {@link AStarRouter} * * @author matthieun */ private static class Candidate implements Comparable { private final Route route; private final double cost; Candidate(final Route route, final double cost) { this.route = route; this.cost = cost; } @Override public int compareTo(final Candidate other) { return this.getCost() > other.getCost() ? 1 : this.getCost() == other.getCost() ? 0 : -1; } public double getCost() { return this.cost; } public Route getRoute() { return this.route; } public Edge lastEdge() { return getRoute().end(); } public Node lastNode() { return lastEdge().end(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("[Candidate: "); builder.append(this.route.toString()); builder.append(", Cost: "); builder.append(this.cost); builder.append("]"); return builder.toString(); } public Candidate withNewEdge(final Edge edge, final double edgeCost) { return new Candidate(getRoute().append(edge), getCost() + edgeCost); } } private final Heuristic heuristic; /** * @param atlas * The {@link Atlas} on which the router works * @param threshold * The threshold to look for edges in case of routing between locations * @return A balanced A* Router, which gives 75% cost to the distance from the end and 25% cost * to the distance from the start. */ public static AStarRouter balanced(final Atlas atlas, final Distance threshold) { final double distanceFromStartCostRatio = 0.25; return new AStarRouter(atlas, threshold, (start, candidate, end) -> distanceFromStartCostRatio * start.getLocation().distanceTo(candidate.getLocation()).asMeters() + (1 - distanceFromStartCostRatio) * candidate.getLocation().distanceTo(end.getLocation()).asMeters()); } /** * @param atlas * The {@link Atlas} on which the router works * @param threshold * The threshold to look for edges in case of routing between locations * @return A Dijkstra router (the heuristic looks at the distance from the start only) */ public static AStarRouter dijkstra(final Atlas atlas, final Distance threshold) { return new AStarRouter(atlas, threshold, (start, candidate, end) -> start.getLocation() .distanceTo(candidate.getLocation()).asMeters()); } /** * @param atlas * The {@link Atlas} on which the router works * @param threshold * The threshold to look for edges in case of routing between locations * @return A fast A* Router, which gives all cost to the distance from the end. The route will * be found faster, but the result will be non-optimal */ public static AStarRouter fastComputationAndSubOptimalRoute(final Atlas atlas, final Distance threshold) { return new AStarRouter(atlas, threshold, (start, candidate, end) -> candidate.getLocation() .distanceTo(end.getLocation()).asMeters()); } /** * Construct * * @param atlas * The map * @param threshold * The threshold to look for edges in case of routing between locations * @param heuristic * The heuristic of the {@link AStarRouter} */ public AStarRouter(final Atlas atlas, final Distance threshold, final Heuristic heuristic) { super(atlas, threshold); this.heuristic = heuristic; } @Override public Route route(final Node start, final Node end) { if (start.equals(end)) { return null; } if (start.outEdges().isEmpty()) { return null; } if (end.inEdges().isEmpty()) { return null; } // Real Routing final PriorityQueue candidates = new PriorityQueue<>(); final Set explored = new HashSet<>(); // Initialize for (final Edge edge : start.outEdges()) { if (end.equals(edge.end())) { return Route.forEdge(edge); } candidates.add(new Candidate(Route.forEdge(edge), this.heuristic.cost(start, edge.end(), end))); } // Cycle while (!candidates.isEmpty()) { final Candidate best = candidates.poll(); if (end.equals(best.lastNode())) { return best.getRoute(); } for (final Edge edge : best.lastNode().outEdges()) { if (!explored.contains(edge)) { candidates.add( best.withNewEdge(edge, this.heuristic.cost(start, edge.end(), end))); } } explored.add(best.lastEdge()); } return null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/routing/AbstractRouter.java ================================================ package org.openstreetmap.atlas.geography.atlas.routing; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.openstreetmap.atlas.geography.atlas.items.SnappedEdge; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Abstract implementation of a {@link Router}. * * @author matthieun */ public abstract class AbstractRouter implements Router { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(AbstractRouter.class); private final Atlas atlas; private final Distance threshold; /** * Construct * * @param atlas * The map * @param threshold * The threshold to look for edges in case of routing between locations */ public AbstractRouter(final Atlas atlas, final Distance threshold) { this.atlas = atlas; this.threshold = threshold; } @Override public Route route(final Edge start, final Edge end) { if (start == null || end == null) { throw new CoreException( "Cannot compute route on null arguments: start = {} and end = {}", start, end); } if (start.equals(end)) { // Same edge return Route.forEdge(start); } if (start.end().equals(end.start())) { // Directly connected edges return Route.forEdge(start).append(end); } final Route result = route(start.end(), end.start()); if (result != null) { // Re-populate the result with the start and end edges return Route.forEdge(start).append(result).append(end); } return null; } @Override public Route route(final Location start, final Location end) { if (start == null || end == null) { throw new CoreException( "Cannot compute route on null arguments: start = {} and end = {}", start, end); } final List startEdges = this.atlas.snaps(start, this.threshold); final List endEdges = this.atlas.snaps(end, this.threshold); if (startEdges.isEmpty()) { // logger.warn("Could not find a snap for start location {}", start); return null; } if (endEdges.isEmpty()) { // logger.warn("Could not find a snap for end location {}", end); return null; } final Iterator startIterator = startEdges.iterator(); for (int i = 0; i < startEdges.size(); i++) { final Edge startEdge = startIterator.next().getEdge(); final Iterator endIterator = endEdges.iterator(); for (int j = 0; j < endEdges.size(); j++) { final Edge endEdge = endIterator.next().getEdge(); final Route route = route(startEdge, endEdge); if (route != null) { return route; } } } return null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/routing/AllPathsRouter.java ================================================ package org.openstreetmap.atlas.geography.atlas.routing; import java.util.Comparator; import java.util.HashSet; import java.util.Set; import java.util.Stack; import java.util.TreeSet; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Route; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Iterables; /** * Router that returns all possible paths between two given {@link Edge}s, using DFS. Note: this is * very inefficient for larger {@link Atlas} files. * * @author mgostintsev */ public final class AllPathsRouter { public static final int MAXIMUM_ALLOWED_PATHS = 20000; private static final Logger logger = LoggerFactory.getLogger(AllPathsRouter.class); private static final int MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL = 1000; // The default filter to use when we want to include all edges. private static final Predicate ALL_EDGES = edge -> true; /** * Find all possible routes from the start {@link Edge} to the end {@link Edge}, using DFS. * * @param start * The {@link Edge} to start from * @param end * The {@link Edge} to end at * @return all possible {@link Route}s from start to end */ public static Set allRoutes(final Edge start, final Edge end) { if (start.getAtlas() != end.getAtlas()) { throw new CoreException("Supplied start and end edges must come from the same atlas!"); } if (Iterables.size(start.getAtlas().edges()) > MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL) { throw new CoreException( "Atlas has too many edges for an efficient traversal - aborting!"); } final Stack path = new Stack<>(); final Set onPath = new HashSet<>(); final Set routes = new HashSet<>(); allRoutes(start, end, path, onPath, routes, ALL_EDGES, MAXIMUM_ALLOWED_PATHS); return routes; } /** * Find all possible routes from the start {@link Edge} to the end {@link Edge}, using DFS and * return in an order determined by the given {@link Comparator}. * * @param start * The {@link Edge} to start from * @param end * The {@link Edge} to end at * @param comparator * Used to order the found routes * @return all possible {@link Route}s from start to end */ public static Set allRoutes(final Edge start, final Edge end, final Comparator comparator) { if (start.getAtlas() != end.getAtlas()) { throw new CoreException("Supplied start and end edges must come from the same atlas!"); } if (Iterables.size(start.getAtlas().edges()) > MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL) { throw new CoreException( "Atlas has too many edges for an efficient traversal - aborting!"); } final Stack path = new Stack<>(); final Set onPath = new HashSet<>(); final Set routes = new TreeSet<>(comparator); allRoutes(start, end, path, onPath, routes, ALL_EDGES, MAXIMUM_ALLOWED_PATHS); return routes; } /** * Find all possible routes from the start {@link Edge} to the end {@link Edge}, using DFS. Only * {@link Edge}s that meet the given filter will be included in the resulting {@link Route}s. * * @param start * The {@link Edge} to start from * @param end * The {@link Edge} to end at * @param filter * The filter to use when including {@link Edge}s that make up the route * @return all possible {@link Route}s from start to end */ public static Set allRoutes(final Edge start, final Edge end, final Predicate filter) { if (start.getAtlas() != end.getAtlas()) { throw new CoreException("Supplied start and end edges must come from the same atlas!"); } if (Iterables.size(start.getAtlas().edges()) > MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL) { throw new CoreException( "Atlas has too many edges for an efficient traversal - aborting!"); } final Stack path = new Stack<>(); final Set onPath = new HashSet<>(); final Set routes = new HashSet<>(); allRoutes(start, end, path, onPath, routes, filter, MAXIMUM_ALLOWED_PATHS); return routes; } /** * Find all possible routes from the start {@link Edge} to the end {@link Edge}, using DFS. Only * {@link Edge}s that meet the given filter will be included in the resulting {@link Route}s. * The supplied {@link Comparator} will be used to order the {@link Route}s. * * @param start * The {@link Edge} to start from * @param end * The {@link Edge} to end at * @param filter * The filter to use when including {@link Edge}s that make up the route * @param comparator * Used to order the found routes * @return all possible {@link Route}s from start to end */ public static Set allRoutes(final Edge start, final Edge end, final Predicate filter, final Comparator comparator) { if (start.getAtlas() != end.getAtlas()) { throw new CoreException("Supplied start and end edges must come from the same atlas!"); } if (Iterables.size(start.getAtlas().edges()) > MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL) { throw new CoreException( "Atlas has too many edges for an efficient traversal - aborting!"); } final Stack path = new Stack<>(); final Set onPath = new HashSet<>(); final Set routes = new TreeSet<>(comparator); allRoutes(start, end, path, onPath, routes, filter, MAXIMUM_ALLOWED_PATHS); return routes; } public static Set allRoutes(final Edge start, final Edge end, final Predicate filter, final int maximumAllowedPaths) { if (start.getAtlas() != end.getAtlas()) { throw new CoreException("Supplied start and end edges must come from the same atlas!"); } if (Iterables.size(start.getAtlas().edges()) > MAXIMUM_ALLOWED_EDGES_FOR_TRAVERSAL) { throw new CoreException( "Atlas has too many edges for an efficient traversal - aborting!"); } final Stack path = new Stack<>(); final Set onPath = new HashSet<>(); final Set routes = new HashSet<>(); allRoutes(start, end, path, onPath, routes, filter, maximumAllowedPaths); return routes; } private static void allRoutes(final Edge start, final Edge end, final Stack path, final Set onPath, final Set routes, final Predicate filter, final int maximumAllowedPaths) { if (routes.size() >= maximumAllowedPaths) { return; } // Add start edge to the path path.push(start); // This will avoid adding same edge both in forward and reverse direction onPath.add(start.end().getIdentifier()); if (start.equals(end)) { // Found a path from start to end, save this route routes.add(Route.forEdges(path)); if (routes.size() > maximumAllowedPaths) { logger.warn("Too many paths found - aborting! Path so far: {}", path.stream().map(edge -> String.valueOf(edge.getMainEdgeIdentifier())) .collect(Collectors.toList())); } } else { // Consider all outgoing, non-zero length edges that can continue the current path, // without repeating an edge that's already part of the current path, and that meet the // given filter for (final Edge candidate : start.outEdges()) { // Proceed if we have not yet visited the edge in any direction (It would be really // weired // to revisit an edge in a BigNode, both in positive and negative directions. if (!candidate.isZeroLength() && !onPath.contains(candidate.end().getIdentifier()) && (filter.test(candidate) || candidate.equals(end))) { allRoutes(candidate, end, path, onPath, routes, filter, maximumAllowedPaths); } } } // We've explored all paths that go through this edge. Remove it from consideration path.pop(); onPath.remove(start.end().getIdentifier()); } private AllPathsRouter() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/routing/README.md ================================================ # Routing This package contains simple routers mostly used to test the Atlas connectivity features. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/routing/Router.java ================================================ package org.openstreetmap.atlas.geography.atlas.routing; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Route; /** * A router that routes between two points * * @author matthieun */ public interface Router { /** * Route from a start to an end * * @param start * The start edge * @param end * The end edge * @return The route corresponding, null if it can't find any */ Route route(Edge start, Edge end); /** * Route from a start to an end * * @param start * The start location * @param end * The end location * @return The route corresponding, null if it can't find any */ Route route(Location start, Location end); /** * Route from a start to an end * * @param start * The start node * @param end * The end node * @return The route corresponding, null if it can't find any */ Route route(Node start, Node end); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/AtlasStatistics.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics; import java.io.BufferedWriter; import java.io.OutputStreamWriter; import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import org.apache.commons.lang3.StringEscapeUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; /** * @author matthieun */ public class AtlasStatistics implements Iterable, Serializable { /** * @author matthieun */ public static class StatisticKey implements Serializable { private static final long serialVersionUID = -4581103889690712256L; private final String tag; private final String type; private final String subType; public StatisticKey(final String tag, final String type, final String subType) { this.tag = tag; this.type = type; this.subType = subType; } @Override public boolean equals(final Object other) { if (other instanceof StatisticKey) { final StatisticKey that = (StatisticKey) other; return this.getTag().equalsIgnoreCase(that.getTag()) && this.getType().equals(that.getType()) && this.getSubType().equals(that.getSubType()); } return false; } public String getSubType() { return this.subType; } public String getTag() { return this.tag; } public String getType() { return this.type; } @Override public int hashCode() { return this.tag.toLowerCase().hashCode() + this.type.hashCode() + this.subType.hashCode(); } @Override public String toString() { return this.tag + "," + this.type + "," + StringEscapeUtils.escapeCsv(this.subType); } } /** * @author matthieun */ public static class StatisticValue implements Serializable { private static final long serialVersionUID = 1693304125915623196L; private final double count; private final double totalCount; private static String format(final double value) { return String.format("%.2f", value); } private static void validate(final double value) { if (Math.abs(value) > Double.MAX_VALUE / 2.0) { throw new CoreException("Invalid count/totalCount value: {}", format(value)); } } public StatisticValue(final double count, final double totalCount) { validate(count); validate(totalCount); this.count = count; this.totalCount = totalCount; } @Override public boolean equals(final Object other) { if (other instanceof StatisticValue) { final StatisticValue that = (StatisticValue) other; return this.getCount() == that.getCount() && this.getTotalCount() == that.getTotalCount(); } return false; } public double getCount() { return this.count; } public double getTotalCount() { return this.totalCount; } @Override public int hashCode() { return Double.hashCode(this.count) + Double.hashCode(this.totalCount); } public StatisticValue merge(final StatisticValue other) { return new StatisticValue(getCount() + other.getCount(), getTotalCount() + other.getTotalCount()); } @Override public String toString() { return format(this.count) + "," + format(this.totalCount); } } private static final long serialVersionUID = 1564587872667339612L; private static final String LINE_SEPARATOR = System.getProperty("line.separator"); private static final int TAG = 0; private static final int TYPE = 1; private static final int SUB_TYPE = 2; private static final int COUNT = 3; private static final int TOTAL = 4; private final Map data; public static String csvHeader() { final StringList header = new StringList(); header.add("# tag"); header.add("type"); header.add("sub_type"); header.add("count"); header.add("total_count"); return header.join(","); } public static AtlasStatistics fromResource(final Resource resource) { final AtlasStatistics result = new AtlasStatistics(); resource.lines().forEach(line -> { if (line.startsWith("#")) { return; } final StringList split = StringList.split(line, ","); final StatisticKey key = new StatisticKey(split.get(TAG), split.get(TYPE), split.get(SUB_TYPE)); final StatisticValue value = new StatisticValue(Double.valueOf(split.get(COUNT)), Double.valueOf(split.get(TOTAL))); result.add(key, value); }); return result; } public static AtlasStatistics merge(final AtlasStatistics... statistics) { return merge(Iterables.asList(statistics)); } public static AtlasStatistics merge(final Iterable statistics) { final Map mergedData = new HashMap<>(); for (final AtlasStatistics source : statistics) { for (final StatisticKey key : source) { StatisticValue value = source.get(key); if (mergedData.containsKey(key)) { value = value.merge(mergedData.get(key)); mergedData.remove(key); } mergedData.put(key, value); } } return new AtlasStatistics(mergedData); } public AtlasStatistics() { this.data = new HashMap<>(); } private AtlasStatistics(final Map data) { this.data = data; } @Override public boolean equals(final Object other) { if (other instanceof AtlasStatistics) { return ((AtlasStatistics) other).getData().equals(getData()); } return false; } public StatisticValue get(final StatisticKey key) { return this.data.get(key); } public StatisticValue get(final String tag, final String type, final String subType) { return this.data.get(new StatisticKey(tag, type, subType)); } public Map getData() { return this.data; } @Override public int hashCode() { return this.data.hashCode(); } @Override public Iterator iterator() { return this.data.keySet().iterator(); } public void save(final WritableResource writableResource) { BufferedWriter out = null; try { out = new BufferedWriter( new OutputStreamWriter(writableResource.write(), StandardCharsets.UTF_8)); out.write(toString()); Streams.close(out); } catch (final Exception e) { Streams.close(out); throw new CoreException("Could not save AtlasStatistics to {}", e, writableResource); } } @Override public String toString() { final StringList result = new StringList(); result.add(csvHeader()); this.data.forEach((key, value) -> result.add(key + "," + value)); return result.join(LINE_SEPARATOR); } protected void add(final StatisticKey key, final StatisticValue value) { this.data.put(key, value); } protected void append(final Map statistics) { this.data.putAll(statistics); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/AtlasStatisticsMerger.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.maps.MultiMap; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * @author matthieun */ public class AtlasStatisticsMerger extends Command { private static final Switch> INPUT = new Switch<>("input", "The input folder containing all the shard stat files", value -> { final File folder = new File(value); return folder.listFilesRecursively().stream() .filter(file -> file.getName().endsWith(".csv.gz")) .collect(Collectors.toList()); }); private static final Switch OUTPUT = new Switch<>("output", "The output folder for the country stats", File::new); public static void main(final String[] args) { new AtlasStatisticsMerger().run(args); } @Override protected int onRun(final CommandMap command) { @SuppressWarnings("unchecked") final List inputs = (List) command.get(INPUT); final File output = (File) command.get(OUTPUT); output.mkdirs(); final MultiMap stats = new MultiMap<>(); for (final File input : inputs) { final String country = input.getName().split("_")[0]; final AtlasStatistics stat = AtlasStatistics.fromResource(input); stats.add(country, stat); } final Map countryStats = stats.reduceByKey(AtlasStatistics::merge); countryStats.forEach((country, stat) -> stat.save(output.child(country + ".csv"))); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(INPUT, OUTPUT); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/Counter.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.Crawler; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.AtlasItem; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.Coverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.area.LakeAreaCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.area.RiverAreaCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.BusRouteLinearCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.AllHighwayTagEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.BridgeEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.FerryEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.FreshnessEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.LanesEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.NameEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.NoNameEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.OneWayEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.PrivateAccessEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.ReferenceEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.SpeedLimitEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.SurfaceEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.TollEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge.TunnelEdgeCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.line.RailLineCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.line.RiverLineCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.line.TransitRailLineCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.poi.EdgesCountCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.poi.LastUserNameCountCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.poi.OneWayEdgesCountCoverage; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.poi.SimpleCoverage; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Iterables; /** * @author matthieun */ public class Counter extends Crawler { private static final Logger logger = LoggerFactory.getLogger(Counter.class); private static final long LAST_USER_EDITS_CUTOFF = 1_000L; public static final Switch POI_COUNTS_DEFINITION = new Switch<>("poiCounts", "file containing all the poi counts definition", value -> { final Resource defaultResource = new InputStreamResource( () -> SimpleCoverage.class.getResourceAsStream("counts.txt")); if ("".equals(value)) { return defaultResource; } else { try { return new File(value); } catch (final Exception e) { return defaultResource; } } }, Optionality.OPTIONAL, ""); private Resource countsDefinition = POI_COUNTS_DEFINITION.getDefault(); private Sharding sharding; public static void main(final String[] args) { new Counter().run(args); } public Counter() { super(logger); } public List> generateCoverages(final Atlas atlas) { final List> coverages = new ArrayList<>(); // Areas coverages.add(new LakeAreaCoverage(atlas)); coverages.add(new RiverAreaCoverage(atlas)); // Edges coverages.add(new SpeedLimitEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add(new LanesEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add(new SurfaceEdgeCoverage(atlas, HighwayTag::isMetricHighway)); coverages.add(new NameEdgeCoverage(atlas, HighwayTag::isMetricHighway, "length_named")); coverages.add(new NameEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway, "length_roads_named")); coverages.add(new NoNameEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add(new OneWayEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add( new AllHighwayTagEdgeCoverage(atlas, HighwayTag::isMetricHighway, "length_total")); coverages.add(new AllHighwayTagEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway, "length_roads_total")); coverages.add(new BridgeEdgeCoverage(atlas)); coverages.add(new TunnelEdgeCoverage(atlas)); coverages.add(new FerryEdgeCoverage(atlas)); coverages.add(new AllHighwayTagEdgeCoverage(atlas, HighwayTag::isPedestrianNavigableHighway, "length_roads_pedestrian")); coverages.add(new ReferenceEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add(new TollEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); coverages.add(new PrivateAccessEdgeCoverage(atlas, HighwayTag::isCarNavigableHighway)); // LineItems coverages.add(new BusRouteLinearCoverage(atlas)); // Lines coverages.add(new RiverLineCoverage(atlas)); coverages.add(new RailLineCoverage(atlas)); coverages.add(new TransitRailLineCoverage(atlas)); // POIs SimpleCoverage.parseSimpleCoverages(atlas, this.countsDefinition.lines()) .forEach(coverages::add); coverages.add(new EdgesCountCoverage(atlas, edge -> HighwayTag.isMetricHighway(edge) && edge.length().isGreaterThan(Distance.ZERO) && edge.asPolyLine().size() > 1)); coverages.add(new OneWayEdgesCountCoverage(atlas, edge -> HighwayTag.isMetricHighway(edge) && edge.length().isGreaterThan(Distance.ZERO) && edge.asPolyLine().size() > 1)); coverages.add(new LastUserNameCountCoverage(atlas, LAST_USER_EDITS_CUTOFF)); // Freshness coverages.add(new FreshnessEdgeCoverage(atlas)); // Sharding related adjustments if (this.sharding != null) { coverages.forEach(coverage -> coverage.setShardDivisor(entity -> { if (entity instanceof AtlasItem && !(entity instanceof LocationItem)) { final PolyLine geometry; if (entity instanceof LineItem) { geometry = ((LineItem) entity).asPolyLine(); } else if (entity instanceof Area) { geometry = ((Area) entity).asPolygon(); } else { throw new CoreException("Unknown entity type: {}", entity.getClass().getCanonicalName()); } return Iterables.size(this.sharding.shardsIntersecting(geometry)); } else { // Skip relations, points and nodes } return 1; })); } return coverages; } public AtlasStatistics processAtlas(final Atlas atlas) { final AtlasStatistics result = new AtlasStatistics(); generateCoverages(atlas).forEach(coverage -> { coverage.run(); result.append(coverage.getStatistic()); }); return result; } public void setCountsDefinition(final Resource countsDefinition) { this.countsDefinition = countsDefinition; } public Counter withSharding(final Sharding sharding) { this.sharding = sharding; return this; } @Override protected void initialize(final CommandMap command) { this.countsDefinition = (Resource) command.get(POI_COUNTS_DEFINITION); logger.info("Using {} for POI counts", this.countsDefinition); } @Override protected void processAtlas(final String atlasName, final Atlas atlas, final String folder) { final File file = new File(folder).child(atlasName + "-statistics.csv"); final AtlasStatistics statistics = processAtlas(atlas); file.writeAndClose(statistics.toString()); } @Override protected SwitchList switches() { return super.switches().with(POI_COUNTS_DEFINITION); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/Coverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics.StatisticKey; import org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics.StatisticValue; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Ratio; import org.openstreetmap.atlas.utilities.statistic.storeless.CounterWithStatistic; import org.slf4j.Logger; /** * Coverage of an Atlas feature. * * @param * The type of {@link AtlasEntity} * @author matthieun */ public abstract class Coverage { /** * @author matthieun */ public enum CoverageType { DISTANCE, SURFACE, COUNT; public static CoverageType forName(final String name) { if (DISTANCE.name().equalsIgnoreCase(name)) { return DISTANCE; } else if (SURFACE.name().equalsIgnoreCase(name)) { return SURFACE; } else if (COUNT.name().equalsIgnoreCase(name)) { return COUNT; } else { throw new CoreException("Unknown coverage type {}", name); } } } private static final int REPORT_FREQUENCY = 100_000; public static final String AGGREGATE_KEY = "all"; public static final String NULL_KEY = ""; private final Logger logger; private final Atlas atlas; private final Predicate filter; private Map counted; private Map total; private Map validCount; private Map totalCount; private CounterWithStatistic statistic; // In case an item is spanning multiple count entities (ex. shards), supply a divisor that will // under-count the item. private Function shardDivisor = null; private Comparator keyComparator; public Coverage(final Logger logger, final Atlas atlas) { this(logger, atlas, item -> true); } /** * Construct. * * @param atlas * The {@link Atlas} to crawl * @param filter * The filter to apply to the crawled items. * @param logger * The logger to use */ public Coverage(final Logger logger, final Atlas atlas, final Predicate filter) { this.logger = logger; this.atlas = atlas; this.filter = filter; this.counted = new HashMap<>(); this.total = new HashMap<>(); this.validCount = new HashMap<>(); this.totalCount = new HashMap<>(); this.statistic = null; this.keyComparator = null; } /** * @param key * The key to get the ratio from * @return The count {@link Ratio} of the specified key. */ public Ratio getCountCoverage(final String key) { if (!this.counted.containsKey(key)) { throw new CoreException("Key {} is not valid."); } if (this.validCount.get(key) > this.totalCount.get(key)) { throw new CoreException("Invalid Ratio: {} / {}", this.validCount.get(key), this.totalCount.get(key)); } if (this.totalCount.get(key) <= 0) { return Ratio.percentage(0); } return Ratio.ratio((double) this.validCount.get(key) / this.totalCount.get(key)); } /** * @param key * The key to get the ratio from * @return The coverage {@link Ratio} of the specified key. */ public Ratio getCoverage(final String key) { if (!this.counted.containsKey(key)) { throw new CoreException("Key {} is not valid."); } if (this.counted.get(key) > this.total.get(key)) { throw new CoreException("Invalid Ratio: {} / {}", this.counted.get(key), this.total.get(key)); } if (this.total.get(key) <= 0.0) { throw new CoreException("Invalid Total: {}", this.total.get(key)); } return Ratio.ratio(this.counted.get(key) / this.total.get(key)); } public Map getStatistic() { final Map result = new HashMap<>(); for (final String key : getKeys()) { final StatisticKey statisticKey = new StatisticKey(key, type(), subType()); final double count; final double totalCount; switch (coverageType()) { case DISTANCE: case SURFACE: count = this.counted.get(key); totalCount = this.total.get(key); break; case COUNT: count = this.validCount.get(key); totalCount = this.totalCount.get(key); break; default: throw new CoreException("Unknown coverage type {}", coverageType()); } final StatisticValue statisticValue = new StatisticValue(count, totalCount); result.put(statisticKey, statisticValue); } return result; } /** * @param key * The key to test for * @return True if this {@link Coverage} covers the specified key */ public boolean hasKey(final String key) { return this.counted.containsKey(key); } /** * Execute this {@link Coverage} */ public void run() { this.statistic = new CounterWithStatistic(this.logger, REPORT_FREQUENCY, this.getClass().getSimpleName()); items().forEach(item -> { this.statistic.increment(); keys(item).forEach(key -> { final double value = getValue(item); // The adjusted value is the value divided by the divisor. The divisor can be the // number of shards the feature crosses. For example, an Edge that spans three // shards will be counted one third. Then when aggregating the counts for all the // shards, this value will not have been counted three times too many. final double adjustedValue = this.shardDivisor == null ? value : value / this.shardDivisor.apply(item); increment(key, adjustedValue, isCounted(item)); }); }); } public void setShardDivisor(final Function shardDivisor) { this.shardDivisor = shardDivisor; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("["); builder.append(this.getClass().getSimpleName()); builder.append(":\n"); for (final String key : getKeys()) { builder.append("\t"); builder.append(key); builder.append(" = \n\t{\n\t\t"); builder.append(getCoverage(key)); builder.append(" of "); builder.append(String.format("%,.2f", this.total.get(key))); builder.append(" "); builder.append(getUnit()); builder.append(",\n\t\t"); builder.append(getCountCoverage(key)); builder.append(" of "); builder.append(String.format("%,d", this.totalCount.get(key))); builder.append(" features\n\t},\n"); } builder.append("]"); return builder.toString(); } protected abstract CoverageType coverageType(); protected Atlas getAtlas() { return this.atlas; } /** * @return All the wanted {@link AtlasEntity}s */ protected abstract Iterable getEntities(); protected Set getKeys() { final Set keySet; if (this.keyComparator != null) { keySet = new TreeSet<>(this.keyComparator); } else { keySet = new HashSet<>(); } keySet.addAll(this.total.keySet()); return keySet; } /** * Get all the categories (or "keys") for which an item can be accounted for. For example, an * Edge that is a highway would be categorized as "highway" and a trunk road edge would be in * the different category "trunk". * * @param item * The item to test for * @return All the categories (or "keys") for which the item can be accounted for */ protected abstract Set getKeys(T item); /** * @return The unit (if any, empty {@link String} otherwise) that is used to meter each item. */ protected abstract String getUnit(); /** * The value used to meter an item. * * @param item * The item to meter * @return The value of the item to meter. */ protected abstract double getValue(T item); /** * Increment counts for an item with a specific key * * @param key * One of the item's keys. * @param value * The item's value * @param valid * True if the item is valid within the requirements of the coverage metric. */ protected void increment(final String key, final double value, final boolean valid) { if (value < 0.0) { throw new CoreException("Invalid value {}", value); } if (!this.total.containsKey(key)) { this.total.put(key, 0.0); this.totalCount.put(key, 0L); this.counted.put(key, 0.0); this.validCount.put(key, 0L); } this.total.put(key, this.total.get(key) + value); this.totalCount.put(key, this.totalCount.get(key) + 1); if (valid) { this.counted.put(key, this.counted.get(key) + value); this.validCount.put(key, this.validCount.get(key) + 1); } } /** * @param item * The item to test * @return True if the item is valid within the requirements of the coverage metric. */ protected abstract boolean isCounted(T item); /** * @param keyComparator * The String comparator to order the keys when printing */ protected void setKeyComparator(final Comparator keyComparator) { this.keyComparator = keyComparator; } protected abstract String subType(); protected abstract String type(); /** * @return The filtered {@link Iterable} of items to measure. */ private Iterable items() { return Iterables.filter(getEntities(), this.filter); } /** * The keys for an item. This method wraps the abstract getKeys() method and adds an all, which * will represent an aggregate of all the coverages. If the keys provided by the sub class is * empty, there is no need to add an "all" key. Instead a null key is added. * * @param item * The item to get the keys from. * @return The keys from the sub-class, plus the "All" key. */ private Set keys(final T item) { final Set result = getKeys(item); if (result.isEmpty()) { result.add(NULL_KEY); } else { result.add(AGGREGATE_KEY); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/area/AreaCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.area; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.Coverage; import org.slf4j.Logger; /** * @author matthieun */ public abstract class AreaCoverage extends Coverage { public AreaCoverage(final Logger logger, final Atlas atlas) { super(logger, atlas); } public AreaCoverage(final Logger logger, final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected CoverageType coverageType() { return CoverageType.DISTANCE; } @Override protected Iterable getEntities() { return getAtlas().areas(); } @Override protected Set getKeys(final Area area) { return new HashSet<>(); } @Override protected String getUnit() { return "kilometer squared"; } @Override protected double getValue(final Area area) { return area.asPolygon().surface().asKilometerSquared(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/area/LakeAreaCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.area; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class LakeAreaCoverage extends AreaCoverage { private static final Logger logger = LoggerFactory.getLogger(LakeAreaCoverage.class); private static final StringList NATURAL_MATCHES = new StringList(new String[] { "water" }); private static final StringList WATER_MATCHES = new StringList( new String[] { "lake", "pond", "reflecting_pool", "reservoir" }); private static final StringList LANDUSE_MATCHES = new StringList(new String[] { "basin" }); public LakeAreaCoverage(final Atlas atlas) { super(logger, atlas); } public LakeAreaCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected boolean isCounted(final Area item) { return item.containsValue("natural", NATURAL_MATCHES) && (item.containsValue("water", WATER_MATCHES) || item.tag("water") == null) || item.containsValue("landuse", LANDUSE_MATCHES); } @Override protected String subType() { return "true"; } @Override protected String type() { return "lakes_area"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/area/RiverAreaCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.area; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.line.RiverLineCoverage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class RiverAreaCoverage extends AreaCoverage { private static final Logger logger = LoggerFactory.getLogger(RiverAreaCoverage.class); public RiverAreaCoverage(final Atlas atlas) { super(logger, atlas); } public RiverAreaCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected boolean isCounted(final Area item) { return item.containsValue("waterway", RiverLineCoverage.WATERWAY_MATCHES) || item.containsValue("natural", RiverLineCoverage.NATURAL_MATCHES) && item.containsValue("water", RiverLineCoverage.WATER_MATCHES); } @Override protected String subType() { return "true"; } @Override protected String type() { return "rivers_area"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/BusRouteLinearCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class BusRouteLinearCoverage extends LinearCoverage { private static final Logger logger = LoggerFactory.getLogger(BusRouteLinearCoverage.class); private static final StringList RELATION_TYPE_MATCHES = new StringList( new String[] { "route" }); private static final StringList RELATION_ROUTE_MATCHES = new StringList(new String[] { "bus" }); public BusRouteLinearCoverage(final Atlas atlas) { super(logger, atlas); } public BusRouteLinearCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected Iterable getEntities() { return getAtlas().lineItems(); } @Override protected Set getKeys(final LineItem item) { return new HashSet<>(); } @Override protected boolean isCounted(final LineItem item) { for (final Relation relation : item.relations()) { if (relation.containsValue("type", RELATION_TYPE_MATCHES) && relation.containsValue("route", RELATION_ROUTE_MATCHES)) { return true; } } return false; } @Override protected String subType() { return "true"; } @Override protected String type() { return "transit_bus_length"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/LinearCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.Coverage; import org.slf4j.Logger; /** * @param * The type of {@link LineItem} * @author matthieun */ public abstract class LinearCoverage extends Coverage { public LinearCoverage(final Logger logger, final Atlas atlas) { super(logger, atlas); } public LinearCoverage(final Logger logger, final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected CoverageType coverageType() { return CoverageType.DISTANCE; } @Override protected String getUnit() { return "kilometers"; } @Override protected double getValue(final T item) { return item.asPolyLine().length().asKilometers(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/edge/AllHighwayTagEdgeCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class AllHighwayTagEdgeCoverage extends EdgeCoverage { private static final Logger logger = LoggerFactory.getLogger(AllHighwayTagEdgeCoverage.class); private final String type; public AllHighwayTagEdgeCoverage(final Atlas atlas, final Predicate filter, final String type) { super(logger, atlas, filter); this.type = type; } public AllHighwayTagEdgeCoverage(final Atlas atlas, final String type) { super(logger, atlas); this.type = type; } @Override protected boolean isCounted(final Edge item) { return true; } @Override protected String type() { return this.type; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/edge/BridgeEdgeCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.tags.BridgeTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Coverage for the Bridge tag * * @author pmi */ public class BridgeEdgeCoverage extends EdgeCoverage { private static final Logger logger = LoggerFactory.getLogger(BridgeEdgeCoverage.class); public BridgeEdgeCoverage(final Atlas atlas) { super(logger, atlas); } public BridgeEdgeCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected boolean isCounted(final Edge edge) { return BridgeTag.isBridge(edge); } @Override protected String type() { return BridgeTag.KEY; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/edge/EdgeCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge; import java.util.Comparator; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.LinearCoverage; import org.openstreetmap.atlas.tags.HighwayTag; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.collections.Maps; import org.slf4j.Logger; /** * Highway type separated {@link Edge} coverage * * @author matthieun */ public abstract class EdgeCoverage extends LinearCoverage { private static final Comparator KEY_COMPARATOR = (key1, key2) -> { final Taggable taggable1 = Taggable.with(Maps.hashMap(HighwayTag.KEY, key1)); final Taggable taggable2 = Taggable.with(Maps.hashMap(HighwayTag.KEY, key2)); final Optional tag1 = HighwayTag.highwayTag(taggable1); final Optional tag2 = HighwayTag.highwayTag(taggable2); if (tag1.isPresent() && tag2.isPresent()) { return tag1.get().compareTo(tag2.get()); } else { return key1.compareTo(key2); } }; public EdgeCoverage(final Logger logger, final Atlas atlas) { super(logger, atlas); this.setKeyComparator(KEY_COMPARATOR); } public EdgeCoverage(final Logger logger, final Atlas atlas, final Predicate filter) { super(logger, atlas, filter.and(Edge::isMainEdge)); this.setKeyComparator(KEY_COMPARATOR); } @Override public String toString() { return super.toString(); } @Override protected Iterable getEntities() { return getAtlas().edges(); } @Override protected Set getKeys(final Edge edge) { final Set result = new HashSet<>(); result.add(edge.highwayTag().getTagValue()); return result; } @Override protected String subType() { return "true"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/edge/FerryEdgeCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.tags.RouteTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Coverage for Route=Ferry tag * * @author pmi */ public class FerryEdgeCoverage extends EdgeCoverage { private static final Logger logger = LoggerFactory.getLogger(FerryEdgeCoverage.class); public FerryEdgeCoverage(final Atlas atlas) { super(logger, atlas); } public FerryEdgeCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override protected boolean isCounted(final Edge edge) { return RouteTag.isFerry(edge); } @Override protected String type() { return "ferry"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/atlas/statistics/coverage/linear/edge/FreshnessEdgeCoverage.java ================================================ package org.openstreetmap.atlas.geography.atlas.statistics.coverage.linear.edge; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics.StatisticKey; import org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics.StatisticValue; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author matthieun */ public class FreshnessEdgeCoverage extends EdgeCoverage { private static final Logger logger = LoggerFactory.getLogger(FreshnessEdgeCoverage.class); public FreshnessEdgeCoverage(final Atlas atlas) { super(logger, atlas); } public FreshnessEdgeCoverage(final Atlas atlas, final Predicate filter) { super(logger, atlas, filter); } @Override public Map getStatistic() { final Map old = super.getStatistic(); final Map totals = new HashMap<>(); final Map monthAll = new HashMap<>(); old.forEach((oldKey, oldValue) -> { final StringList split = StringList.split(oldKey.getTag(), "_"); if (split.size() == 2) { final String tag = split.get(0); final String month = split.get(1); Double allResult = oldValue.getCount(); if (monthAll.containsKey(month)) { allResult += monthAll.get(month); } monthAll.put(month, allResult); Double result = oldValue.getCount(); if (totals.containsKey(tag)) { result += totals.get(tag); } totals.put(tag, result); } }); final Map result = new HashMap<>(); old.forEach((oldKey, oldValue) -> { final StringList split = StringList.split(oldKey.getTag(), "_"); if (split.size() == 2) { final StatisticKey key = new StatisticKey(split.get(0), oldKey.getType(), split.get(1)); final StatisticValue value = new StatisticValue(oldValue.getCount(), totals.get(key.getTag())); result.put(key, value); } }); monthAll.forEach((month, count) -> result.put( new StatisticKey(AGGREGATE_KEY, type(), month), new StatisticValue(monthAll.get(month), monthAll.values().stream().reduce((left, right) -> left + right).get()))); return result; } @Override protected Set getKeys(final Edge edge) { final Optional here. * * @author mgostintsev */ public class GeodeticEarthCenteredEarthFixedConverter implements TwoWayConverter { @Override public GeodeticCoordinate backwardConvert(final EarthCenteredEarthFixedCoordinate coordinate) { final double semiMinor = Math.sqrt(WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS_SQUARED * (1 - WorldGeodeticStandardConstants.ECCENTRICITY_SQUARED)); final double semiMinorSquared = Math.pow(semiMinor, 2); final double secondEccentricity = Math .sqrt((WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS_SQUARED - semiMinorSquared) / semiMinorSquared); final double auxiliaryP = Math .sqrt(Math.pow(coordinate.getX(), 2) + Math.pow(coordinate.getY(), 2)); final double theta = Math.atan2( coordinate.getZ() * WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS.asMeters(), auxiliaryP * semiMinor); final double longitude = Math.atan2(coordinate.getY(), coordinate.getX()); final double latitude = Math.atan2( coordinate.getZ() + Math.pow(secondEccentricity, 2) * semiMinor * Math.pow(Math.sin(theta), 3), auxiliaryP - WorldGeodeticStandardConstants.ECCENTRICITY_SQUARED * WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS.asMeters() * Math.pow(Math.cos(theta), 3)); final double radiusOfCurviture = WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS.asMeters() / Math.sqrt(1 - WorldGeodeticStandardConstants.ECCENTRICITY_SQUARED * Math.pow(Math.sin(latitude), 2)); final double altitude = auxiliaryP / Math.cos(latitude) - radiusOfCurviture; return new GeodeticCoordinate(Latitude.radians(latitude), Longitude.radians(longitude), Altitude.meters(altitude)); } @Override public EarthCenteredEarthFixedCoordinate convert(final GeodeticCoordinate coordinate) { final double radiusOfCurviture = WorldGeodeticStandardConstants.SEMI_MAJOR_AXIS.asMeters() / Math.sqrt(1 - WorldGeodeticStandardConstants.ECCENTRICITY_SQUARED * Math.pow(Math.sin(coordinate.getLatitude().asPositiveRadians()), 2)); final double height = coordinate.getAltitude().asMeters(); // getting positive angles for latitude/longitude is okay as the angles would be the same. // For example, cos(-30) == cos(330) and sin(-30) == sin(330) final double xValue = (radiusOfCurviture + height) * Math.cos(coordinate.getLatitude().asPositiveRadians()) * Math.cos(coordinate.getLongitude().asPositiveRadians()); final double yValue = (radiusOfCurviture + height) * Math.cos(coordinate.getLatitude().asPositiveRadians()) * Math.sin(coordinate.getLongitude().asPositiveRadians()); final double zValue = ((1 - WorldGeodeticStandardConstants.ECCENTRICITY_SQUARED) * radiusOfCurviture + height) * Math.sin(coordinate.getLatitude().asPositiveRadians()); return new EarthCenteredEarthFixedCoordinate(xValue, yValue, zValue); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/MultiPolygonStringConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayStringConverter; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * Convert a {@link MultiPolygon} back and forth to a {@link String} * * @author matthieun */ public class MultiPolygonStringConverter implements TwoWayStringConverter { // For some reason, splitting the String fails when OUTER_SEPARATOR is "|" public static final String OUTER_SEPARATOR = "&"; public static final String OUTER_INNERS_SEPARATOR = "#"; public static final String INNER_SEPARATOR = "+"; private static final PolygonStringConverter POLYGON_STRING_CONVERTER = new PolygonStringConverter(); @Override public String backwardConvert(final MultiPolygon object) { final StringList outers = new StringList(); for (final Polygon outer : object.outers()) { final StringList inners = new StringList(); for (final Polygon inner : object.innersOf(outer)) { inners.add(inner.toCompactString()); } outers.add(outer.toCompactString() + OUTER_INNERS_SEPARATOR + inners.join(INNER_SEPARATOR)); } return outers.join(OUTER_SEPARATOR); } @Override public MultiPolygon convert(final String object) { final MultiMap result = new MultiMap<>(); final StringList outers = StringList.split(object, OUTER_SEPARATOR); for (final String outerString : outers) { final StringList outerInners = StringList.split(outerString, OUTER_INNERS_SEPARATOR); final Polygon outer = POLYGON_STRING_CONVERTER.convert(outerInners.get(0)); if (outerInners.size() > 1) { final StringList inners = StringList.split(outerInners.get(1), INNER_SEPARATOR); for (final String innerString : inners) { result.add(outer, POLYGON_STRING_CONVERTER.convert(innerString)); } } else { result.put(outer, new ArrayList<>()); } } return new MultiPolygon(result); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/MultiplePolyLineToMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.geom.prep.PreparedPolygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.items.Relation.Ring; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.Converter; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * From a map of {@link PolyLine}s for {@link Ring} types, try to stitch all the {@link PolyLine}s * together to form a single {@link MultiPolygon}. * * @author samg **/ public class MultiplePolyLineToMultiPolygonConverter implements Converter>, MultiPolygon> { private static final MultiplePolyLineToPolygonsConverter MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER = new MultiplePolyLineToPolygonsConverter(); private static List buildOuters(final Iterable outers) { final List outerPolygons = new ArrayList<>(); Iterables.stream(outers).filter(line -> line instanceof Polygon) .forEach(polygon -> outerPolygons.add((Polygon) polygon)); final List outerPolyLines = new ArrayList<>(); Iterables.stream(outers).filter(line -> !(line instanceof Polygon)) .forEach(outerPolyLines::add); MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER.convert(outerPolyLines) .forEach(outerPolygons::add); return outerPolygons; } private static MultiMap buildOutersToInnersMap(final List outers, final Iterable inners) { final MultiMap outersToInners = new MultiMap<>(); outers.forEach(outer -> outersToInners.put(outer, new ArrayList<>())); final List innerPolygons = new ArrayList<>(); final List innerPolyLines = new ArrayList<>(); Iterables.stream(inners).filter(line -> line instanceof Polygon) .forEach(polygon -> innerPolygons.add((Polygon) polygon)); Iterables.stream(inners).filter(line -> !(line instanceof Polygon)) .forEach(innerPolyLines::add); MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER.convert(innerPolyLines) .forEach(innerPolygons::add); final JtsPolygonConverter converter = new JtsPolygonConverter(); final Map preparedOuters = new HashMap<>(); outersToInners.keySet().forEach(outer -> preparedOuters.put(outer, (PreparedPolygon) PreparedGeometryFactory.prepare(converter.convert(outer)))); innerPolygons.forEach(inner -> { boolean added = false; final org.locationtech.jts.geom.Polygon inner2 = converter.convert(inner); for (final Map.Entry entry : preparedOuters.entrySet()) { if (entry.getValue().containsProperly(inner2)) { outersToInners.add(entry.getKey(), inner); added = true; break; } } if (!added) { throw new CoreException("Malformed MultiPolygon: inner has no outer host: {}", inner); } }); return outersToInners; } @Override public MultiPolygon convert(final Map> outersAndInners) { final List outers = buildOuters(outersAndInners.get(Ring.OUTER)); if (outers.isEmpty()) { throw new CoreException("Unable to find outer polygon."); } final MultiMap outersToInners = buildOutersToInnersMap(outers, outersAndInners.get(Ring.INNER)); return new MultiPolygon(outersToInners); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/MultiplePolyLineToPolygonsConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.operation.polygonize.Polygonizer; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.MultiIterable; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * From a set of {@link PolyLine}s, try to stitch all the {@link PolyLine}s together to form * {@link Polygon}s. * * @author matthieun * @author Sid */ public class MultiplePolyLineToPolygonsConverter implements Converter, Iterable> { /** * @author Sid */ public static class OpenPolygonException extends CoreException { private static final long serialVersionUID = -278028096455310936L; public static final String OPEN_LOCATIONS_ARE = " Open Locations are: "; private final List openLocations; public OpenPolygonException(final String message, final List openLocations) { super(message + OPEN_LOCATIONS_ARE + openLocations.toString()); this.openLocations = openLocations; } public OpenPolygonException(final String message, final List openLocations, final Object... arguments) { super(message + OPEN_LOCATIONS_ARE + openLocations.toString(), arguments); this.openLocations = openLocations; } public OpenPolygonException(final String message, final List openLocations, final Throwable cause, final Object... arguments) { super(message + OPEN_LOCATIONS_ARE + openLocations.toString(), cause, arguments); this.openLocations = openLocations; } public List getOpenLocations() { return this.openLocations; } } /** * Simple object containing connectivity information about two {@link PolyLine}s. "connected" * set to "true" means the two {@link PolyLine}s connect at some end. "reversed" set to true * means that one of the two {@link PolyLine}s had to be reversed to connect. * * @author matthieun */ private static class ConnectResult { private final boolean connected; private final boolean reversed; ConnectResult(final boolean connected, final boolean reversed) { this.connected = connected; this.reversed = reversed; } public boolean isConnected() { return this.connected; } public boolean isReversed() { return this.reversed; } @Override public String toString() { final String isReversed = this.reversed ? " and reversed" : ""; return this.connected ? "Connected" + isReversed : "Not connected"; } } /** * A {@link Polygon} in construction, with many other {@link PolyLine}s * * @author matthieun */ private static class PossiblePolygon { private boolean completed; // An ordered list of polylines, based on connectivity private final List polyLines = new ArrayList<>(); PossiblePolygon(final PolyLine first) { this.completed = first instanceof Polygon || first.first().equals(first.last()); this.polyLines.add(first); } /** * @param candidate * A polyLine to attach to that possible polygon * @return True if the polyLine was successfully attached. */ public boolean attach(final PolyLine candidate) { boolean result = false; final ConnectResult canAppendCandidateToLine = canAppendSecondToFirst(lastPolyLine(), candidate); final ConnectResult canPrependCandidateToLine = canPrependFirstToSecond(candidate, firstPolyLine()); PolyLine toAdd = candidate; if (canAppendCandidateToLine.isConnected()) { if (canAppendCandidateToLine.isReversed()) { toAdd = toAdd.reversed(); } if (toAdd.size() > 1) { toAdd = trimFirst(toAdd); } else { if (canPrependCandidateToLine.isConnected()) { this.completed = true; } return true; } } if (canPrependCandidateToLine.isConnected()) { if (canPrependCandidateToLine.isReversed()) { if (canAppendCandidateToLine.isConnected()) { // Already reversed previously } else { toAdd = toAdd.reversed(); } } if (toAdd.size() > 1) { toAdd = trimLast(toAdd); } else { if (canAppendCandidateToLine.isConnected()) { this.completed = true; } return true; } } if (canAppendCandidateToLine.isConnected()) { this.polyLines.add(toAdd); result = true; } else if (canPrependCandidateToLine.isConnected()) { this.polyLines.add(0, toAdd); result = true; } if (canPrependCandidateToLine.isConnected() && canAppendCandidateToLine.isConnected()) { this.completed = true; } return result; } public Location firstLocation() { return this.polyLines.get(0).first(); } public boolean isCompleted() { return this.completed; } public Location lastLocation() { return this.polyLines.get(this.polyLines.size() - 1).last(); } public int size() { return this.polyLines.size(); } public Polygon toPolygon() { if (!this.isCompleted() && this.size() >= 1) { // If that method is called and the PossiblePolygon is not closed (i.e. completed) // we gather the first and end point of the partially completed polyline and throw // an exception. final List openLocations = new ArrayList<>(); final Location firstLocation = this.polyLines.get(0).first(); final Location lastLocation = this.polyLines.get(this.size() - 1).last(); if (firstLocation != null && lastLocation != null) { openLocations.add(firstLocation); openLocations.add(lastLocation); throw new OpenPolygonException( "Cannot build polygon with multiple polylines. Loop is not closed.", openLocations); } } return new Polygon(new MultiIterable<>(this.polyLines)); } @Override public String toString() { final StringList list = new StringList(); this.polyLines.forEach(polyLine -> list.add(polyLine.first() + " -> ")); list.add(this.lastLocation()); return list.join(""); } /** * Test if two {@link PolyLine}s connect by appending the second {@link PolyLine} (straight * or reversed) to the first one (unchanged). * * @param one * The {@link PolyLine} from which the end will be considered * @param two * The {@link PolyLine} from which the start will be considered * @return ConnectResult: connected = true if the end of one is the same as the start of two * and reversed = true if the {@link PolyLine} two had to be reversed to be able to * connect the end of one to the beginning of two. */ private ConnectResult canAppendSecondToFirst(final PolyLine one, final PolyLine two) { if (one.last().equals(two.first())) { return new ConnectResult(true, false); } else if (one.last().equals(two.last())) { return new ConnectResult(true, true); } else { return new ConnectResult(false, false); } } /** * Test if two {@link PolyLine}s connect by prepending the first {@link PolyLine} (straight * or reversed) to the second one (unchanged). * * @param one * The {@link PolyLine} from which the end will be considered * @param two * The {@link PolyLine} from which the start will be considered * @return ConnectResult: connected = true if the end of one is the same as the start of two * and reversed = true if the {@link PolyLine} one had to be reversed to be able to * connect the end of one to the beginning of two. */ private ConnectResult canPrependFirstToSecond(final PolyLine one, final PolyLine two) { if (one.last().equals(two.first())) { return new ConnectResult(true, false); } else if (one.first().equals(two.first())) { return new ConnectResult(true, true); } else { return new ConnectResult(false, false); } } private PolyLine firstPolyLine() { return this.polyLines.get(0); } private PolyLine lastPolyLine() { return this.polyLines.get(this.polyLines.size() - 1); } /** * Remove the first point of this {@link PolyLine} to append it to another {@link PolyLine} * * @param current * The {@link PolyLine} to trim * @return The {@link PolyLine} trimmed of its first point. */ private PolyLine trimFirst(final PolyLine current) { final List result = new ArrayList<>(); for (final Location location : current) { result.add(location); } result.remove(0); return new PolyLine(result); } /** * Remove the last point of this {@link PolyLine} to prepend it to another {@link PolyLine} * * @param current * The {@link PolyLine} to trim * @return The {@link PolyLine} trimmed of its last point. */ private PolyLine trimLast(final PolyLine current) { final List result = new ArrayList<>(); for (final Location location : current) { result.add(location); } result.remove(result.size() - 1); return new PolyLine(result); } } private static final JtsPolyLineConverter JTS_POLY_LINE_CONVERTER = new JtsPolyLineConverter(); private static final JtsPolygonConverter JTS_POLYGON_CONVERTER = new JtsPolygonConverter(); private final boolean usePolygonizer; public MultiplePolyLineToPolygonsConverter() { this.usePolygonizer = false; } public MultiplePolyLineToPolygonsConverter(final boolean usePolygonizer) { this.usePolygonizer = usePolygonizer; } @Override public Iterable convert(final Iterable candidates) { if (this.usePolygonizer) { return convertAttemptPolygonizer(candidates); } else { return convertLegacy(candidates); } } public Iterable convertAttemptPolygonizer(final Iterable candidates) { final Polygonizer polygonizer = new Polygonizer(); candidates.forEach(polyLine -> polygonizer.add(JTS_POLY_LINE_CONVERTER.convert(polyLine))); // Check for missing parts final List errors = new ArrayList<>(); Exception potentialException = null; try { errors.addAll(polygonizer.getDangles()); errors.addAll(polygonizer.getCutEdges()); errors.addAll(polygonizer.getInvalidRingLines()); } catch (final Exception e) { potentialException = e; } if (errors.isEmpty() && potentialException == null) { // Get results final List result = (List) polygonizer .getPolygons(); return result.stream().map(polygon -> { polygon.normalize(); return JTS_POLYGON_CONVERTER.backwardConvert(polygon); }).collect(Collectors.toList()); } else { final List locations = errors.stream() .map(JTS_POLY_LINE_CONVERTER::backwardConvert) .flatMap(dangle -> Iterables .asList(Iterables.from(dangle.first(), dangle.last())).stream()) .collect(Collectors.toList()); final OpenPolygonException jtsException; final String errorMessage = "Unable to close all the polygons!"; if (potentialException != null) { jtsException = new OpenPolygonException(errorMessage, locations, potentialException); } else { jtsException = new OpenPolygonException(errorMessage, locations); } // Try the legacy convert try { return convertLegacy(candidates); } catch (final Exception e) { throw new OpenPolygonException( "Failed second legacy attempt. JTS Exception was: \"{}\"", jtsException.getOpenLocations(), jtsException.getMessage(), e); } } } public Iterable convertLegacy(final Iterable candidates) // NOSONAR { // The complete polygons final List completes = new ArrayList<>(); // The polygons that have been started, but that are incomplete. final List incompletes = new ArrayList<>(); // The polyLines that have not found a match yet final LinkedList remainingPolyLines = new LinkedList<>(); candidates.forEach(remainingPolyLines::add); int iterationsSinceLastPolyLineTaken = 0; while (!remainingPolyLines.isEmpty() && iterationsSinceLastPolyLineTaken <= remainingPolyLines.size()) { final PolyLine candidate = remainingPolyLines.removeFirst(); boolean added = false; if (!incompletes.isEmpty()) { // There are some incompletes. Always try to fill the incompletes to the end until // they are complete before creating new incomplete polygons. boolean completed = false; int index = -1; // Try the candidate polyline with all the incomplete polygons for (final PossiblePolygon incomplete : incompletes) { index++; if (incomplete.attach(candidate)) { added = true; completed = incomplete.isCompleted(); break; } } if (completed) { final PossiblePolygon increased = incompletes.get(index); incompletes.remove(index); completes.add(increased); } } else { // There are no incomplete polygons, just create one. final PossiblePolygon incompleteCandidate = new PossiblePolygon(candidate); if (incompleteCandidate.isCompleted()) { completes.add(incompleteCandidate); } else { incompletes.add(incompleteCandidate); } added = true; } if (!added) { // Could not add the polyline to any incomplete polygon, so adding it back to the // end of the list. It might get better luck once those incomplete polygons have // grown a bit more. remainingPolyLines.addLast(candidate); iterationsSinceLastPolyLineTaken++; } else { iterationsSinceLastPolyLineTaken = 0; } } if (!incompletes.isEmpty()) { throw new OpenPolygonException("Unable to close all the polygons!", Iterables .stream(incompletes).flatMap(incomplete -> Iterables .from(incomplete.firstLocation(), incomplete.lastLocation())) .collectToList()); } return completes.stream().map(PossiblePolygon::toPolygon).collect(Collectors.toList()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/MultiplePolyLineToPolygonsConverterCommand.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.io.OutputStreamWriter; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.SafeBufferedWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * This command reads a file with delimited lists of {@link PolyLine}s in WKT format, and applies * logic to stitch those {@link PolyLine}s together to form one or more {@link Polygon}s. * * @author matthieun */ public class MultiplePolyLineToPolygonsConverterCommand extends Command { private static final Switch POLYLINES = new Switch<>("polylines", "File containing lines of semicolon separated list of WKT polylines", File::new, Optionality.REQUIRED); private static final Switch POLYGONS = new Switch<>("polygons", "Output file that will contain lines of semicolon separated list of reconstructed polygons", File::new, Optionality.OPTIONAL); private static final Switch DELIMITER = new Switch<>("delimiter", "The string delimiter between groups of polylines, and polygons in the output.", StringConverter.IDENTITY, Optionality.OPTIONAL, ";"); private static final WktPolyLineConverter WKT_POLY_LINE_CONVERTER = new WktPolyLineConverter(); private static final WktMultiPolyLineConverter WKT_MULTI_POLY_LINE_CONVERTER = new WktMultiPolyLineConverter(); private static final MultiplePolyLineToPolygonsConverter MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER = new MultiplePolyLineToPolygonsConverter(); public static void main(final String[] args) { new MultiplePolyLineToPolygonsConverterCommand().run(args); } @Override protected int onRun(final CommandMap command) { final String delimiter = (String) command.get(DELIMITER); final File input = (File) command.get(POLYLINES); final File output = (File) command.get(POLYGONS); translate(input, output, delimiter); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(POLYLINES, POLYGONS, DELIMITER); } protected void translate(final Resource input, final WritableResource output, final String delimiter) { final Iterable> inputs = Iterables.stream(input.lines()) .map(line -> StringList.split(line, delimiter).stream() .filter(wkt -> wkt != null && !wkt.isEmpty()).flatMap(wkt -> { final List result = new ArrayList<>(); if (wkt.toLowerCase().contains("multi")) { WKT_MULTI_POLY_LINE_CONVERTER.backwardConvert(wkt) .forEach(result::add); } else { result.add(WKT_POLY_LINE_CONVERTER.backwardConvert(wkt)); } return result.stream(); }).collect(Collectors.toList())); try (SafeBufferedWriter writer = writer(output)) { for (final List inputPolyLines : inputs) { writer.writeLine(new StringList(Iterables .stream(MULTIPLE_POLY_LINE_TO_POLYGONS_CONVERTER.convert(inputPolyLines)) .map(Polygon::toWkt)).join(delimiter)); } } catch (final Exception e) { throw new CoreException("Unable to convert polylines from {}", input.getName(), e); } } private SafeBufferedWriter writer(final WritableResource output) { return output != null ? output.writer() : new SafeBufferedWriter(new OutputStreamWriter(System.out)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/PolyLineStringConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayStringConverter; /** * @author matthieun */ public class PolyLineStringConverter implements TwoWayStringConverter { @Override public String backwardConvert(final PolyLine object) { return object.toString(); } @Override public PolyLine convert(final String object) { final StringList split = StringList.split(object, PolyLine.SEPARATOR); final List locations = new ArrayList<>(); for (final String location : split) { locations.add(Location.forString(location)); } return new PolyLine(locations); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/PolygonStringConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayStringConverter; /** * @author matthieun */ public class PolygonStringConverter implements TwoWayStringConverter { @Override public String backwardConvert(final Polygon object) { return object.toCompactString(); } @Override public Polygon convert(final String object) { final StringList split = StringList.split(object, PolyLine.SEPARATOR); final List locations = new ArrayList<>(); for (final String location : split) { locations.add(Location.forString(location)); } return new Polygon(locations); } public Polygon convertLongitudeLatitude(final String object) { final StringList split = StringList.split(object, PolyLine.SEPARATOR); final List locations = new ArrayList<>(); for (final String location : split) { locations.add(Location.forStringLongitudeLatitude(location)); } return new Polygon(locations); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/PolygonStringFormat.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; import org.locationtech.jts.io.WKBReader; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.streaming.readers.GeoJsonReader; import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The enum for supported Polygon and Multipolygon string formats. Contains functions to return * {@link Polygon} and {@link MultiPolygon} {@link StringConverter}s. * * @author jklamer */ public enum PolygonStringFormat { ATLAS("atlas"), GEOJSON("geojson"), WKT("wkt"), WKB("wkb"), UNSUPPORTED("UNSUPPORTED"); private static final Logger logger = LoggerFactory.getLogger(PolygonStringFormat.class); private final String format; public static PolygonStringFormat getEnumForFormat(final String format) { for (final PolygonStringFormat polygonStringFormat : values()) { if (polygonStringFormat.getFormat().equalsIgnoreCase(format)) { return polygonStringFormat; } } return UNSUPPORTED; } PolygonStringFormat(final String format) { this.format = format; } public String getFormat() { return this.format; } public StringConverter>> getMultiPolygonConverter() { switch (this) { case ATLAS: return string -> Optional.of(Collections .singletonList(new MultiPolygonStringConverter().convert(string))); case GEOJSON: return string -> { final List multiPolygons = new ArrayList<>(); final GeoJsonReader reader = new GeoJsonReader(new StringResource(string)); while (reader.hasNext()) { final PropertiesLocated propertiesLocated = reader.next(); if (propertiesLocated.getItem() instanceof MultiPolygon) { multiPolygons.add((MultiPolygon) propertiesLocated.getItem()); } else { logger.warn("MultiPolygon Filter does not support item {}", propertiesLocated.toString()); } } return Optional.of(multiPolygons).filter(list -> !list.isEmpty()); }; case WKB: return string -> Optional .of(Collections.singletonList(new WkbMultiPolygonConverter() .backwardConvert(WKBReader.hexToBytes(string)))); case WKT: return string -> Optional.of(Collections .singletonList(new WktMultiPolygonConverter().backwardConvert(string))); case UNSUPPORTED: default: logger.warn("No converter set up for {} format. Supported formats are {}", this.format, Arrays.copyOf(values(), values().length - 1)); return string -> Optional.empty(); } } public StringConverter>> getPolygonConverter() { switch (this) { case ATLAS: return string -> Optional.of( Collections.singletonList(new PolygonStringConverter().convert(string))); case GEOJSON: return string -> { final List polygons = new ArrayList<>(); final GeoJsonReader reader = new GeoJsonReader(new StringResource(string)); while (reader.hasNext()) { final PropertiesLocated propertiesLocated = reader.next(); if (propertiesLocated.getItem() instanceof Polygon) { polygons.add((Polygon) propertiesLocated.getItem()); } else { logger.warn("Polygon Filter does not support item {}", propertiesLocated.toString()); } } return Optional.of(polygons).filter(list -> !list.isEmpty()); }; case WKT: return string -> Optional.of(Collections .singletonList(new WktPolygonConverter().backwardConvert(string))); case WKB: return string -> Optional.of(Collections.singletonList( new WkbPolygonConverter().backwardConvert(WKBReader.hexToBytes(string)))); case UNSUPPORTED: default: logger.warn("No converter set up for {} format. Supported formats are {}", this.format, Arrays.copyOf(values(), values().length - 1)); return string -> Optional.empty(); } } @Override public String toString() { return getFormat(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Abstract converter that underpins {@link WktMultiPolygonConverter} and * {@link WkbMultiPolygonConverter} to reduce code duplication. * * @author matthieun * @param * The type to convert to: {@link String} or byte[] */ public abstract class WkMultiPolygonConverter implements TwoWayConverter { private static final JtsPolygonToMultiPolygonConverter POLYGON_TO_MULTI_POLYGON_CONVERTER = new JtsPolygonToMultiPolygonConverter(); private static final JtsMultiPolygonToMultiPolygonConverter MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); @Override public MultiPolygon backwardConvert(final T wkt) { try { final Geometry result = getGeometryConverter().convert(wkt); if (result instanceof org.locationtech.jts.geom.Polygon) { final org.locationtech.jts.geom.Polygon jtsPolygon = (org.locationtech.jts.geom.Polygon) result; return POLYGON_TO_MULTI_POLYGON_CONVERTER.convert(jtsPolygon); } else if (result instanceof org.locationtech.jts.geom.MultiPolygon) { return MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER .convert((org.locationtech.jts.geom.MultiPolygon) result); } else { throw new CoreException("Unknown type: {}", result.getClass().getCanonicalName()); } } catch (final Exception e) { throw new CoreException("Cannot parse wkt : {}", wkt, e); } } @Override public T convert(final MultiPolygon multiPolygon) { final org.locationtech.jts.geom.Geometry geometry; if (multiPolygon.outers().size() > 1) { geometry = MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(multiPolygon); } else { // JTS Polygons can have one outer and some holes geometry = POLYGON_TO_MULTI_POLYGON_CONVERTER.backwardConvert(multiPolygon); } return getGeometryConverter().backwardConvert(geometry); } abstract TwoWayConverter getGeometryConverter(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkbLocationConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Location} to/from Well Known Binary (WKB). * * @author matthieun */ public class WkbLocationConverter implements TwoWayConverter { @Override public Location backwardConvert(final byte[] wkb) { Point geometry = null; final WKBReader myReader = new WKBReader(); try { geometry = (Point) myReader.read(wkb); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkb : {}", WKBWriter.toHex(wkb), e); } return new JtsPointConverter().backwardConvert(geometry); } @Override public byte[] convert(final Location location) { final Geometry geometry = new JtsPointConverter().convert(location); final byte[] wkb = new WKBWriter().write(geometry); return wkb; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkbMultiPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolyLineConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * A class for converting between MultiLineStrings in WKB format and {@link MultiPolyLine} * * @author jklamer */ public class WkbMultiPolyLineConverter implements TwoWayConverter { private static final WKBReader WKB_READER = new WKBReader(); @Override public MultiPolyLine backwardConvert(final byte[] wkb) { MultiLineString geometry = null; try { geometry = (MultiLineString) WKB_READER.read(wkb); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkb : {}", WKBWriter.toHex(wkb), e); } return new JtsMultiPolyLineConverter().backwardConvert(geometry); } @Override public byte[] convert(final MultiPolyLine multiPolyLine) { final Geometry geometry = new JtsMultiPolyLineConverter().convert(multiPolyLine); return new WKBWriter().write(geometry); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkbMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Converter class for conversion between Wkb byte array and {@link MultiPolygon} * * @author jklamer * @author matthieun */ public class WkbMultiPolygonConverter extends WkMultiPolygonConverter { private static final TwoWayConverter CONVERTER = new TwoWayConverter() { @Override public byte[] backwardConvert(final Geometry geometry) { return new WKBWriter().write(geometry); } @Override public Geometry convert(final byte[] kyte) { try { return new WKBReader().read(kyte); } catch (final ParseException e) { throw new CoreException("Unable to parse WKB", e); } } }; @Override TwoWayConverter getGeometryConverter() { return CONVERTER; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkbPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import java.util.ArrayList; import java.util.List; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Conversion from a {@link PolyLine} into a Well Known Binary (WKB) byte array. Uses JTS for * Coordinate creation and WKB creation and writing. * * @author robert_stack */ public class WkbPolyLineConverter implements TwoWayConverter { @Override public PolyLine backwardConvert(final byte[] wkb) { PolyLine polyLine = null; Geometry geometry = null; final WKBReader myReader = new WKBReader(); try { geometry = myReader.read(wkb); } catch (final ParseException e) { throw new CoreException("Cannot parse wkb : {}", WKBWriter.toHex(wkb), e); } final Coordinate[] coordinates = geometry.getCoordinates(); final List locations = new ArrayList<>(); for (int i = 0; i < coordinates.length; i++) { // y = latitude, x = longitude from JTS Coordinate format locations.add(new Location(Latitude.degrees(coordinates[i].y), Longitude.degrees(coordinates[i].x))); } polyLine = new PolyLine(locations); return polyLine; } @Override public byte[] convert(final PolyLine polyLine) { final Geometry geometry; final List cooordinates = new ArrayList<>(); for (final Location location : polyLine) { // swap latitude/longitude for JTS Coordinate format cooordinates.add(new Coordinate(location.getLongitude().asDegrees(), location.getLatitude().asDegrees())); } final Coordinate[] coordinateArray = cooordinates .toArray(new Coordinate[cooordinates.size()]); if (coordinateArray.length == 1) { geometry = new GeometryFactory().createPoint(coordinateArray[0]); } else { geometry = new GeometryFactory().createLineString(coordinateArray); } final byte[] wkb = new WKBWriter().write(geometry); return wkb; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WkbPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert Polygons to a Well Known Binary (WKB) byte array. Polygons retain only one of first * versus last point (which are the same for a closed loop). * * @author Sid * @author cstaylor */ public class WkbPolygonConverter implements TwoWayConverter { @Override public Polygon backwardConvert(final byte[] wkb) { org.locationtech.jts.geom.Polygon geometry = null; final WKBReader myReader = new WKBReader(); try { geometry = (org.locationtech.jts.geom.Polygon) myReader.read(wkb); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkb : {}", WKBWriter.toHex(wkb), e); } return new JtsPolygonConverter().backwardConvert(geometry); } @Override public byte[] convert(final Polygon polygon) { final Geometry geometry = new JtsPolygonConverter().convert(polygon); final byte[] wkb = new WKBWriter().write(geometry); return wkb; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WktLocationConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Location} to/from Well Known Text (WKT). * * @author matthieun */ public class WktLocationConverter implements TwoWayConverter { @Override public Location backwardConvert(final String wkt) { Point geometry = null; final WKTReader myReader = new WKTReader(); try { geometry = (Point) myReader.read(wkt); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkt : {}", wkt, e); } return new JtsPointConverter().backwardConvert(geometry); } @Override public String convert(final Location location) { final Geometry geometry = new JtsPointConverter().convert(location); final String wkt = new WKTWriter().write(geometry); return wkt; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WktMultiPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolyLineConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Given an WKT string generate a {@link MultiLineString} and vice-versa * * @author yalimu */ public class WktMultiPolyLineConverter implements TwoWayConverter { @Override public MultiPolyLine backwardConvert(final String wkt) { MultiLineString geometry = null; final WKTReader myReader = new WKTReader(); try { geometry = (MultiLineString) myReader.read(wkt); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkt : {}", wkt, e); } return new JtsMultiPolyLineConverter().backwardConvert(geometry); } @Override public String convert(final MultiPolyLine multiPolyLine) { final MultiLineString geometry = new JtsMultiPolyLineConverter().convert(multiPolyLine); return new WKTWriter().write(geometry); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WktMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Given an WKT string generate a {@link MultiPolygon} and vice-versa * * @author matthieun */ public class WktMultiPolygonConverter extends WkMultiPolygonConverter { private static final TwoWayConverter CONVERTER = new TwoWayConverter() { @Override public String backwardConvert(final Geometry geometry) { return new WKTWriter().write(geometry); } @Override public Geometry convert(final String wkt) { try { return new WKTReader().read(wkt); } catch (final ParseException e) { throw new CoreException("Unable to parse WKT", e); } } }; @Override TwoWayConverter getGeometryConverter() { return CONVERTER; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WktPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Given an WKT string generate a {@link PolyLine} and vice-versa * * @author matthieun */ public class WktPolyLineConverter implements TwoWayConverter { @Override public PolyLine backwardConvert(final String wkt) { LineString geometry = null; final WKTReader myReader = new WKTReader(); try { geometry = (LineString) myReader.read(wkt); } catch (final ParseException | ClassCastException e) { throw new CoreException("Cannot parse wkt : {}", wkt, e); } return new JtsPolyLineConverter().backwardConvert(geometry); } @Override public String convert(final PolyLine polyLine) { final Geometry geometry = new JtsPolyLineConverter().convert(polyLine); return new WKTWriter().write(geometry); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/WktPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Given an WKT string generate a Polygon and vice-versa * * @author cstaylor * @author matthieun */ public class WktPolygonConverter implements TwoWayConverter { private static final JtsMultiPolygonToMultiPolygonConverter JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER = new JtsMultiPolygonToMultiPolygonConverter(); @Override public Polygon backwardConvert(final String wkt) { final WKTReader myReader = new WKTReader(); try { final Geometry result = myReader.read(wkt); if (result instanceof org.locationtech.jts.geom.Polygon) { return new JtsPolygonConverter() .backwardConvert((org.locationtech.jts.geom.Polygon) result); } else if (result instanceof org.locationtech.jts.geom.MultiPolygon) { final MultiPolygon castResult = JTS_MULTI_POLYGON_TO_MULTI_POLYGON_CONVERTER .convert((org.locationtech.jts.geom.MultiPolygon) result); if (castResult.outers().size() == 1 && castResult.inners().isEmpty()) { return castResult.outers().iterator().next(); } } throw new CoreException( "Cannot convert wkt which is not a Polygon or single-outer MultiPolygon: {}", wkt); } catch (final ParseException e) { throw new CoreException("Cannot parse wkt: {}", wkt, e); } } @Override public String convert(final Polygon polygon) { final Geometry geometry = new JtsPolygonConverter().convert(polygon); return new WKTWriter().write(geometry); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/GeometryStreamer.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Transform a JTS {@link GeometryCollection} into a real {@link Iterable}. * * @author matthieun */ public final class GeometryStreamer { /** * @param collection * The collection to stream * @return The JTS {@link Geometry} {@link Iterable} */ public static Iterable stream(final GeometryCollection collection) { final int size = collection.getNumGeometries(); return Iterables.indexBasedIterable(size, index -> collection.getGeometryN((int) index)); } /** * @param collection * The collection to stream * @return The JTS {@link Geometry} {@link Iterable} as JTS {@link Polygon}s */ public static Iterable streamPolygons(final GeometryCollection collection) { return Iterables.translate(stream(collection), geometry -> (Polygon) geometry); } private GeometryStreamer() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsCoordinateArrayConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert an {@link Iterable} of {@link Location} to a {@link CoordinateSequence} from the JTS * library. * * @author matthieun */ public class JtsCoordinateArrayConverter implements TwoWayConverter, CoordinateSequence> { private static final JtsLocationConverter LOCATION_CONVERTER = new JtsLocationConverter(); public static CoordinateSequence empty() { final Coordinate[] emptyCoordinateArray = new Coordinate[0]; return new CoordinateArraySequence(emptyCoordinateArray); } @Override public Iterable backwardConvert(final CoordinateSequence coordinateSequence) { final List result = new ArrayList<>(); for (final Coordinate coordinate : coordinateSequence.toCoordinateArray()) { result.add(LOCATION_CONVERTER.backwardConvert(coordinate)); } return result; } @Override public CoordinateSequence convert(final Iterable locations) { final int size; if (locations instanceof Collection) { size = ((Collection) locations).size(); } else { size = (int) Iterables.size(locations); } final Coordinate[] result = new Coordinate[size]; int index = 0; for (final Location location : locations) { result[index++] = LOCATION_CONVERTER.convert(location); } return new CoordinateArraySequence(result); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsLinearRingConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.List; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Polygon} to a {@link LinearRing} from the JTS library. It is worth noting that a * {@link Polygon} assumes that the last point is connected to the first point, and hence does not * supply the first point twice. However the JTS {@link LinearRing} implementation assumes that the * ring has to be closed, i.e. that the last point and the first point have to be the same. If this * is not the case, creating the {@link LinearRing} will fail. This converter accounts for that both * ways. * * @author matthieun */ public class JtsLinearRingConverter implements TwoWayConverter { private static final JtsCoordinateArrayConverter COORDINATE_ARRAY_CONVERTER = new JtsCoordinateArrayConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); // Protect from: java.lang.IllegalArgumentException: Invalid number of points in LinearRing // (found x - must be 0 or >= 4) private static final int MINIMUM_LINEAR_RING_SIZE = 4; public static LinearRing empty() { return new LinearRing(JtsCoordinateArrayConverter.empty(), FACTORY); } @Override public Polygon backwardConvert(final LinearRing object) { final CoordinateSequence sequence = object.getCoordinateSequence(); if (sequence.size() <= 0) { // Cannot have an empty polygon. return null; } final Coordinate[] newArray = new Coordinate[sequence.size() - 1]; for (int i = 0; i < newArray.length; i++) { newArray[i] = sequence.getCoordinate(i); } return new Polygon( COORDINATE_ARRAY_CONVERTER.backwardConvert(new CoordinateArraySequence(newArray))); } @Override public LinearRing convert(final Polygon object) { final List locations = new ArrayList<>(object); // Hack to close the loop, as JTS expects it... locations.add(locations.get(0)); while (locations.size() < MINIMUM_LINEAR_RING_SIZE) { locations.add(locations.get(0)); } return new LinearRing(COORDINATE_ARRAY_CONVERTER.convert(locations), FACTORY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsLocationConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.Coordinate; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Location} to a {@link Coordinate} from the JTS library. The {@link Location}'s * {@link Latitude} dm7 value becomes the {@link Coordinate}'s y value, and the {@link Location}'s * {@link Longitude} dm7 value becomes the the {@link Coordinate}'s x value. * * @author matthieun */ public class JtsLocationConverter implements TwoWayConverter { @Override public Location backwardConvert(final Coordinate coordinate) { return new Location(Latitude.degrees(coordinate.y), Longitude.degrees(coordinate.x)); } @Override public Coordinate convert(final Location location) { return new Coordinate(location.getLongitude().asDegrees(), location.getLatitude().asDegrees()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsMultiPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.List; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.MultiLineString; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link MultiPolyLine} to a JTS {@link MultiLineString}. * * @author yalimu */ public class JtsMultiPolyLineConverter implements TwoWayConverter { private static final JtsCoordinateArrayConverter COORDINATE_ARRAY_CONVERTER = new JtsCoordinateArrayConverter(); private static final JtsPolyLineConverter POLYLINE_CONVERTER = new JtsPolyLineConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public MultiPolyLine backwardConvert(final MultiLineString multiLineString) { final List polyLineList = new ArrayList<>(); for (int i = 0; i < multiLineString.getNumGeometries(); i++) { final LineString lineString = (LineString) multiLineString.getGeometryN(i); final PolyLine polyLine = new PolyLine( COORDINATE_ARRAY_CONVERTER.backwardConvert(lineString.getCoordinateSequence())); // No duplicated polyline is allowed to add. if (!polyLineList.contains(polyLine)) { polyLineList.add(polyLine); } } return new MultiPolyLine(polyLineList); } @Override public MultiLineString convert(final MultiPolyLine multiPolyLine) { final List lineStringList = Iterables.stream(multiPolyLine) .map(POLYLINE_CONVERTER::convert).collectToList(); return new MultiLineString(lineStringList.toArray(new LineString[lineStringList.size()]), FACTORY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * Convert a {@link MultiPolygon} to a {@link Set} of {@link org.locationtech.jts.geom.Polygon} form * the JTS library. As a {@link MultiPolygon} can contain many outer bounds, each outer bound is * translated to a {@link org.locationtech.jts.geom.Polygon}. A JTS * {@link org.locationtech.jts.geom.Polygon} is one single outer bound and many inner bounds. * * @author matthieun */ public class JtsMultiPolygonConverter implements TwoWayConverter> { private static final JtsLinearRingConverter LINEAR_RING_CONVERTER = new JtsLinearRingConverter(); private static final JtsPolygonConverter POLYGON_CONVERTER = new JtsPolygonConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public MultiPolygon backwardConvert(final Set object) { final MultiMap result = new MultiMap<>(); for (final org.locationtech.jts.geom.Polygon polygon : object) { final Polygon outer = POLYGON_CONVERTER.backwardConvert(polygon); if (outer == null) { continue; } for (int n = 0; n < polygon.getNumInteriorRing(); n++) { final LinearRing ring = new LinearRing( polygon.getInteriorRingN(n).getCoordinateSequence(), FACTORY); final Polygon inner = LINEAR_RING_CONVERTER.backwardConvert(ring); result.add(outer, inner); } if (polygon.getNumInteriorRing() == 0) { // Make sure the outer still exists if the inners are not there. result.put(outer, new ArrayList<>()); } } return new MultiPolygon(result); } @Override public Set convert(final MultiPolygon object) { final Set result = new HashSet<>(); for (final Polygon outer : object.outers()) { final List inners = object.innersOf(outer); final LinearRing[] holes = new LinearRing[inners.size()]; int index = 0; for (final Polygon inner : inners) { holes[index++] = LINEAR_RING_CONVERTER.convert(inner); } result.add(new org.locationtech.jts.geom.Polygon(LINEAR_RING_CONVERTER.convert(outer), holes, FACTORY)); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsMultiPolygonToMultiLineStringConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.List; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.geography.atlas.change.ChangeRelation; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * this is a somewhat specific use case converter used by {@link ChangeRelation} * * @author samuelgass */ public class JtsMultiPolygonToMultiLineStringConverter implements TwoWayConverter { @Override public MultiPolygon backwardConvert(final GeometryCollection object) { final Polygon[] polygons = new Polygon[object.getNumGeometries()]; for (int i = 0; i < polygons.length; i++) { polygons[i] = (Polygon) object.getGeometryN(i); } return new MultiPolygon(polygons, JtsPrecisionManager.getGeometryFactory()); } @Override public GeometryCollection convert(final MultiPolygon object) { final List linestrings = new ArrayList<>(); for (int i = 0; i < object.getNumGeometries(); i++) { final Polygon part = (Polygon) object.getGeometryN(i); linestrings.add(part.getExteriorRing()); for (int j = 0; j < part.getNumInteriorRing(); j++) { linestrings.add(part.getInteriorRingN(j)); } } Geometry[] geometries = new Geometry[linestrings.size()]; geometries = linestrings.toArray(geometries); return new GeometryCollection(geometries, JtsPrecisionManager.getGeometryFactory()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsMultiPolygonToMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.HashSet; import java.util.Set; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * @author matthieun */ public class JtsMultiPolygonToMultiPolygonConverter implements TwoWayConverter { private static final JtsMultiPolygonConverter JTS_MULTI_POLYGON_CONVERTER = new JtsMultiPolygonConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public org.locationtech.jts.geom.MultiPolygon backwardConvert(final MultiPolygon object) { final Polygon[] polygons = JTS_MULTI_POLYGON_CONVERTER.convert(object) .toArray(new Polygon[0]); return new org.locationtech.jts.geom.MultiPolygon(polygons, FACTORY); } @Override public MultiPolygon convert(final org.locationtech.jts.geom.MultiPolygon object) { final int numberGeometries = object.getNumGeometries(); final Set polygons = new HashSet<>(); for (int index = 0; index < numberGeometries; index++) { final Polygon polygon = (Polygon) object.getGeometryN(index); polygons.add(polygon); } return JTS_MULTI_POLYGON_CONVERTER.backwardConvert(polygons); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsPointConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Location} to a {@link Point} from the JTS library. The {@link Location}'s * {@link Latitude} dm7 value becomes the {@link Point}'s y value, and the {@link Location}'s * {@link Longitude} dm7 value becomes the the {@link Point}'s x value. * * @author mgostintsev */ public class JtsPointConverter implements TwoWayConverter { public static final GeometryFactory GEOMETRY_FACTORY = JtsPrecisionManager.getGeometryFactory(); private static final JtsLocationConverter LOCATION_CONVERTER = new JtsLocationConverter(); @Override public Location backwardConvert(final Point point) { return new Location(Latitude.degrees(point.getY()), Longitude.degrees(point.getX())); } @Override public Point convert(final Location location) { final Coordinate coordinate = LOCATION_CONVERTER.convert(location); final CoordinateArraySequence sequence = new CoordinateArraySequence( new Coordinate[] { coordinate }); return new Point(sequence, GEOMETRY_FACTORY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsPolyLineConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.LinearRing; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link PolyLine} to a JTS {@link LineString}. * * @author matthieun */ public class JtsPolyLineConverter implements TwoWayConverter { private static final JtsCoordinateArrayConverter COORDINATE_ARRAY_CONVERTER = new JtsCoordinateArrayConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public PolyLine backwardConvert(final LineString lineString) { return new PolyLine( COORDINATE_ARRAY_CONVERTER.backwardConvert(lineString.getCoordinateSequence())); } @Override public LineString convert(final PolyLine polyLine) { /* * Occasionally this method may be called with a PolyLine that is actually a Polygon. In * those cases, we want to properly return a LinearRing so as not to break downstream code * that expects Polygon-like closed-ness. */ if (polyLine instanceof Polygon) { final Polygon polygon = (Polygon) polyLine; return new LinearRing( COORDINATE_ARRAY_CONVERTER.convert(new PolyLine(polygon.closedLoop())), FACTORY); } return new LineString(COORDINATE_ARRAY_CONVERTER.convert(polyLine), FACTORY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a {@link Polygon} to a JTS {@link org.locationtech.jts.geom.Polygon}. Here the inner * bounds are left empty. When converting backwards, if there is an inner bound in the * {@link org.locationtech.jts.geom.Polygon}, it will be ignored. * * @author matthieun */ public class JtsPolygonConverter implements TwoWayConverter { private static final JtsLinearRingConverter LINEAR_RING_CONVERTER = new JtsLinearRingConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public Polygon backwardConvert(final org.locationtech.jts.geom.Polygon object) { return LINEAR_RING_CONVERTER.backwardConvert((LinearRing) object.getExteriorRing()); } @Override public org.locationtech.jts.geom.Polygon convert(final Polygon object) { return new org.locationtech.jts.geom.Polygon(LINEAR_RING_CONVERTER.convert(object), new LinearRing[0], FACTORY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsPolygonToMultiPolygonConverter.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.ArrayList; import java.util.List; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * @author matthieun */ public class JtsPolygonToMultiPolygonConverter implements TwoWayConverter { private static final JtsLinearRingConverter LINEAR_RING_CONVERTER = new JtsLinearRingConverter(); private static final GeometryFactory FACTORY = JtsPrecisionManager.getGeometryFactory(); @Override public org.locationtech.jts.geom.Polygon backwardConvert(final MultiPolygon object) { if (object.getOuterToInners().isEmpty()) { final LinearRing[] emptyInners = new LinearRing[0]; return new org.locationtech.jts.geom.Polygon(JtsLinearRingConverter.empty(), emptyInners, FACTORY); } if (object.getOuterToInners().keySet().size() != 1) { throw new CoreException( "A MultiPolygon can be converted to JTS Polygon only if it has no more than one outer ring."); } final Polygon outer = object.outers().iterator().next(); final LinearRing linearRingOuter = LINEAR_RING_CONVERTER.convert(outer); final List inners = object.getOuterToInners().get(outer); final LinearRing[] linearRingInners = new LinearRing[inners.size()]; for (int index = 0; index < inners.size(); index++) { linearRingInners[index] = LINEAR_RING_CONVERTER.convert(inners.get(index)); } return new org.locationtech.jts.geom.Polygon(linearRingOuter, linearRingInners, FACTORY); } @Override public MultiPolygon convert(final org.locationtech.jts.geom.Polygon object) { final MultiMap outersToInners = new MultiMap<>(); // Here we cannot use the reverse JtsLinearRingConverter because // jts.Polygon.getExteriorRing() returns a LineString instead of a LinearRing :( final List locationsOuter = (List) new JtsCoordinateArrayConverter() .backwardConvert(object.getExteriorRing().getCoordinateSequence()); final Polygon outer = new Polygon(locationsOuter.subList(0, locationsOuter.size() - 1)); outersToInners.put(outer, new ArrayList<>()); for (int index = 0; index < object.getNumInteriorRing(); index++) { final List locationsInner = (List) new JtsCoordinateArrayConverter() .backwardConvert(object.getInteriorRingN(index).getCoordinateSequence()); final Polygon inner = new Polygon(locationsInner.subList(0, locationsInner.size() - 1)); outersToInners.add(outer, inner); } return new MultiPolygon(outersToInners); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsPrecisionManager.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.PrecisionModel; /** * JTS Precision utility class. * * @author Yiqing Jin */ public final class JtsPrecisionManager { private static final int PRECISION_SCALE = 10_000_000; private static PrecisionModel precisionModel; private static GeometryFactory geometryFactory; static { precisionModel = new PrecisionModel(PRECISION_SCALE); geometryFactory = new GeometryFactory(precisionModel); } public static GeometryFactory getGeometryFactory() { return geometryFactory; } public static PrecisionModel getPrecisionModel() { return precisionModel; } private JtsPrecisionManager() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/converters/jts/JtsUtility.java ================================================ package org.openstreetmap.atlas.geography.converters.jts; import java.util.List; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.impl.CoordinateArraySequence; /** * A simple utility for working with JTS objects. * * @author Yiqing Jin */ public final class JtsUtility { public static final GeometryFactory GEOMETRY_FACTORY = JtsPrecisionManager.getGeometryFactory(); private static final int MININMUM_NUMBER_OF_POLYGON_POINTS = 4; public static LineString buildLineString(final Coordinate[] coordinates) { final CoordinateArraySequence sequence = new CoordinateArraySequence(coordinates); return new LineString(sequence, GEOMETRY_FACTORY); } public static LinearRing buildLinearRing(final List coordinates) { final Coordinate[] coordinateArray = coordinates .toArray(new Coordinate[coordinates.size()]); final CoordinateArraySequence sequence = new CoordinateArraySequence(coordinateArray); return new LinearRing(sequence, GEOMETRY_FACTORY); } public static LinearRing buildLinearRing(final CoordinateSequence sequence) { return new LinearRing(sequence, GEOMETRY_FACTORY); } public static Polygon toPolygon(final Coordinate[] coordinates) { if (coordinates.length < MININMUM_NUMBER_OF_POLYGON_POINTS && coordinates.length != 0) { // An invalid polygon. one example A->B->A return null; } final CoordinateArraySequence sequence = new CoordinateArraySequence(coordinates); final LinearRing shell = new LinearRing(sequence, GEOMETRY_FACTORY); return new Polygon(shell, new LinearRing[] {}, GEOMETRY_FACTORY); } public static Polygon toPolygon(final List coordinates) { return toPolygon(coordinates.toArray(new Coordinate[coordinates.size()])); } private JtsUtility() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/coordinates/EarthCenteredEarthFixedCoordinate.java ================================================ package org.openstreetmap.atlas.geography.coordinates; import java.io.Serializable; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.converters.GeodeticEarthCenteredEarthFixedConverter; /** * Earth-Fixed, Earth-Centered (ECEF) coordinate system representation. This is a right-handed * Cartesian coordinate system (think X, Y and Z) with the origin at the Earth's center. Note: Units * are generally expressed in meters. * * @author mgostintsev */ public class EarthCenteredEarthFixedCoordinate implements Serializable { private static final long serialVersionUID = -3091871010423428109L; private static final GeodeticEarthCenteredEarthFixedConverter COORDINATE_CONVERTER = new GeodeticEarthCenteredEarthFixedConverter(); private final double xValue; private final double yValue; private final double zValue; /** * Constructs an {@link EarthCenteredEarthFixedCoordinate} at (0,0,0). */ public EarthCenteredEarthFixedCoordinate() { this.xValue = 0; this.yValue = 0; this.zValue = 0; } /** * Constructs an {@link EarthCenteredEarthFixedCoordinate} at given (x,y,z). * * @param xValue * x-value * @param yValue * y-value * @param zValue * z-value */ public EarthCenteredEarthFixedCoordinate(final double xValue, final double yValue, final double zValue) { this.xValue = xValue; this.yValue = yValue; this.zValue = zValue; } /** * Constructs an {@link EarthCenteredEarthFixedCoordinate} at given {@link Location}. * * @param location * The {@link Location} of the coordinate */ public EarthCenteredEarthFixedCoordinate(final Location location) { final EarthCenteredEarthFixedCoordinate coordinate = COORDINATE_CONVERTER .apply(location.toGeodeticCoordinate()); this.xValue = coordinate.getX(); this.yValue = coordinate.getY(); this.zValue = coordinate.getZ(); } @Override public boolean equals(final Object other) { if (other instanceof EarthCenteredEarthFixedCoordinate) { final EarthCenteredEarthFixedCoordinate that = (EarthCenteredEarthFixedCoordinate) other; return this.getX() == that.getX() && this.getY() == that.getY() && this.getZ() == that.getZ(); } return false; } public double getX() { return this.xValue; } public double getY() { return this.yValue; } public double getZ() { return this.zValue; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getX()).append(this.getY()).append(this.getZ()) .hashCode(); } @Override public String toString() { return "(" + this.getX() + ", " + this.getY() + ", " + this.getZ() + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/coordinates/GeodeticCoordinate.java ================================================ package org.openstreetmap.atlas.geography.coordinates; import java.io.Serializable; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; /** * Geodetic coordinate system representation, consisting of a {@link Latitude}, {@link Longitude} * and {@link Altitude} - commonly referred to as LLA. Note: Units are generally expressed in polar * coordinates and meters (for {@link Altitude}}. * * @author mgostintsev */ public class GeodeticCoordinate implements Serializable { private static final long serialVersionUID = 4614378421580938085L; private final Latitude latitude; private final Longitude longitude; private final Altitude altitude; /** * Default constructor. * * @param latitude * latitude * @param longitude * longitude * @param altitude * altitude */ public GeodeticCoordinate(final Latitude latitude, final Longitude longitude, final Altitude altitude) { this.latitude = latitude; this.longitude = longitude; this.altitude = altitude; } /** * Creates a {@link GeodeticCoordinate} at the given {@link Location}, at mean sea level. * * @param location * The {@link Location} of the coordinate. */ public GeodeticCoordinate(final Location location) { this.latitude = location.getLatitude(); this.longitude = location.getLongitude(); this.altitude = Altitude.MEAN_SEA_LEVEL; } @Override public boolean equals(final Object other) { if (other instanceof GeodeticCoordinate) { final GeodeticCoordinate that = (GeodeticCoordinate) other; return this.getLatitude().equals(that.getLatitude()) && this.getLongitude().equals(that.getLongitude()) && this.getAltitude().equals(that.getAltitude()); } return false; } public Altitude getAltitude() { return this.altitude; } public Latitude getLatitude() { return this.latitude; } public Longitude getLongitude() { return this.longitude; } @Override public int hashCode() { return new HashCodeBuilder().append(this.getLatitude()).append(this.getLongitude()) .append(this.getAltitude()).hashCode(); } @Override public String toString() { return "(" + this.getLatitude() + ", " + this.getLongitude() + ", " + this.getAltitude() + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/ConcatenateGeoJsonCommand.java ================================================ package org.openstreetmap.atlas.geography.geojson; import java.util.Collections; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.readers.GeoJsonReader; import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Utility class to concatenate GeoJson objects. * * @author matthieun * @author sid */ public class ConcatenateGeoJsonCommand extends Command { /** * Mode specifies the format of the input geoJSON objects. Each file can be a GeoJSON object or * file can contain multiple geoJSON objects (one per line) * * @author Sid */ public enum Mode { // Each file is a geoJSON object FILE, // Each line of the file is a geoJSON object LINE; } public static final Switch PATH = new Switch<>("path", "The folder containing the geojson files to concatenate", File::new, Optionality.REQUIRED); public static final Switch OUTPUT = new Switch<>("output", "The file to write the concatenated geojson to", File::new, Optionality.REQUIRED); public static final Switch MODE = new Switch<>("mode", "The mode of the input geoJSON objects (FILE or LINE)", value -> Mode.valueOf(value), Optionality.REQUIRED); public static final Switch FILE_PREFIX = new Switch<>("filePrefix", "The prefix of the input geoJSON file in LINE mode", StringConverter.IDENTITY, Optionality.OPTIONAL, "part-"); public static void main(final String[] args) { new ConcatenateGeoJsonCommand().run(args); } @Override protected int onRun(final CommandMap command) { final File folder = (File) command.get(PATH); final File output = (File) command.get(OUTPUT); final Mode mode = (Mode) command.get(MODE); final String filePrefix = (String) command.get(FILE_PREFIX); // processing the files in sorted order makes testing easier final List files = folder.listFilesRecursively(); Collections.sort(files); final Iterable jsonItems = readGeoJsonItems(mode, files, filePrefix); final GeoJsonObject result = new GeoJsonBuilder() .createFeatureCollectionFromPropertiesLocated(jsonItems); final JsonWriter writer = new JsonWriter(output); writer.write(result.jsonObject()); writer.close(); return 0; } protected Iterable readGeoJsonItems(final Mode mode, final Iterable files, final String filePrefix) { switch (mode) { case FILE: return Iterables.stream(files) .filter(file -> file.getName().endsWith(FileSuffix.GEO_JSON.toString())) .flatMap(this::readGeoJsonItems); case LINE: return Iterables.stream(files).filter(file -> file.getName().startsWith(filePrefix)) .flatMap(file -> file.lines()).map(line -> line.trim()) .filter(line -> !line.isEmpty()).flatMap(this::readGeoJsonItems); default: throw new CoreException("Invalid Mode"); } } @Override protected SwitchList switches() { return new SwitchList().with(PATH, OUTPUT, MODE, FILE_PREFIX); } private Iterable readGeoJsonItems(final Resource resource) { final Iterable iterableOfPropertiesLocated = () -> new GeoJsonReader( resource); return iterableOfPropertiesLocated; } private Iterable readGeoJsonItems(final String line) { return readGeoJsonItems(new StringResource(line)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJson.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * For all classes with a GeoJson representation. From the spec * https://tools.ietf.org/html/rfc7946#section-3 * * 
 *      A GeoJSON object represents a Geometry, Feature, or collection of
 *    Features.
 *
 *    o  A GeoJSON object is a JSON object.
 *
 *    o  A GeoJSON object has a member with the name "type".  The value of
 *       the member MUST be one of the GeoJSON types.
 *
 *    o  A GeoJSON object MAY have a "bbox" member, the value of which MUST
 *       be a bounding box array (see Section 5).
 *
 *    o  A GeoJSON object MAY have other members (see Section 6).
 *
* * This interface is for all classes with a geojson object representation. * * @author jklamer */ public interface GeoJson { JsonObject asGeoJson(); GeoJsonType getGeoJsonType(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonBuilder.java ================================================ package org.openstreetmap.atlas.geography.geojson; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * @author matthieun * @author cuthbertm * @author mgostintsev * @author rmegraw * @author chunzhu */ public class GeoJsonBuilder { /** * Java bean to store geometry (as an {@link Iterable} of {@link Location}s) and properties as a * {@link String} to {@link Object} {@link Map} * * @author matthieun * @author rmegraw */ public static class GeometryWithProperties { private final Iterable geometry; private final Map properties; public GeometryWithProperties(final Iterable geometry, final Map properties) { this.geometry = geometry; this.properties = properties; } public Iterable getGeometry() { return this.geometry; } public Map getProperties() { return this.properties; } } /** * Java bean to store the geometry (as an {@link Iterable} of {@link Location}s) and all the * tags as a {@link String} to {@link String} {@link Map} * * @deprecated instead use {@link GeometryWithProperties} * @author matthieun */ @Deprecated public static class LocationIterableProperties { private final Iterable locations; private final Map properties; public LocationIterableProperties(final Iterable locations, final Map properties) { this.locations = locations; this.properties = properties; } public Iterable getLocations() { return this.locations; } public Map getProperties() { return this.properties; } } public static final String COORDINATES = "coordinates"; public static final String FEATURE = "Feature"; public static final String FEATURES = "features"; public static final String FEATURE_COLLECTION = "FeatureCollection"; public static final String GEOMETRIES = "geometries"; public static final String GEOMETRY = "geometry"; public static final String GEOMETRY_COLLECTION = "GeometryCollection"; public static final String PROPERTIES = "properties"; public static final String TYPE = "type"; private static final Logger logger = LoggerFactory.getLogger(GeoJsonBuilder.class); private final int logFrequency; /** * Converts iterable of deprecated LocationIterableProperties to iterable of * GeometryWithProperties. * * @param objects * iterable of LocationIterableProperties * @return iterable of GeometryWithProperties */ protected static final Iterable toGeometriesWithProperties( final Iterable objects) { final Iterable geometriesWithProperties = Iterables .translate(objects, locationIterableProperties -> { return toGeometryWithProperties(locationIterableProperties); }); return geometriesWithProperties; } /** * Converts deprecated LocationIterableProperties to GeometryWithProperties. * * @param locationIterableProperties * LocationIterableProperties object * @return GeometryWithProperties object */ protected static final GeometryWithProperties toGeometryWithProperties( final LocationIterableProperties locationIterableProperties) { final Map propertiesObjects = new HashMap<>(); propertiesObjects.putAll(locationIterableProperties.getProperties()); return new GeometryWithProperties(locationIterableProperties.getLocations(), propertiesObjects); } public GeoJsonBuilder() { this.logFrequency = -1; } public GeoJsonBuilder(final int logFrequency) { this.logFrequency = logFrequency; } public GeoJsonObject create(final GeoJsonType type, final Location... locations) { return this.create(Iterables.iterable(locations), type); } /** * Creates a Json Feature from a {@link GeometryWithProperties} * * @param geometryWithProperties * {@link GeometryWithProperties} * @return a GeoJson Feature */ public JsonObject create(final GeometryWithProperties geometryWithProperties) { final Iterable geometry = geometryWithProperties.getGeometry(); final Map properties = geometryWithProperties.getProperties(); if (geometry instanceof Location) { return create((Location) geometry).withNewProperties(properties).jsonObject(); } else if (geometry instanceof Polygon) { return create((Polygon) geometry).withNewProperties(properties).jsonObject(); } else if (geometry instanceof PolyLine) { return create((PolyLine) geometry).withNewProperties(properties).jsonObject(); } else { throw new CoreException("Unrecognized object type {}", geometry.getClass().getSimpleName()); } } /** * Creates a GeoJson Feature containing a Geometry * * @param locations * geometry coordinates * @param type * geometry type * @return a GeoJson Feature */ public GeoJsonObject create(final Iterable locations, final GeoJsonType type) { final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE); final JsonArray coordinates; switch (type) { case POINT: { final Location location = locations.iterator().next(); coordinates = new JsonArray(); coordinates.add(new JsonPrimitive(location.getLongitude().asDegrees())); coordinates.add(new JsonPrimitive(location.getLatitude().asDegrees())); break; } case LINESTRING: case MULTI_POINT: case MULTI_LINESTRING: case MULTI_POLYGON: { coordinates = GeoJsonUtils.locationsToCoordinates(locations); break; } case POLYGON: { coordinates = new JsonArray(); final JsonArray locationArray = GeoJsonUtils.locationsToCoordinates(locations); coordinates.add(locationArray); break; } default: throw new CoreException("Unrecognized object type {}", type); } final JsonObject geometry = new JsonObject(); geometry.addProperty(TYPE, type.getTypeString()); geometry.add(COORDINATES, coordinates); result.add(GEOMETRY, geometry); return new GeoJsonObject(result); } /** * Creates a GeoJson FeatureCollection containing a list of Features * * @param objects * used to build each Feature * @return a GeoJson FeatureCollection * @deprecated use {@link #createFromGeometriesWithProperties(Iterable)} instead */ @Deprecated public GeoJsonObject create(final Iterable objects) { return createFromGeometriesWithProperties(toGeometriesWithProperties(objects)); } /** * Creates a Point type GeoJson Feature * * @param location * geometry * @return a Feature */ public GeoJsonObject create(final Location location) { return this.create(location, GeoJsonType.POINT); } /** * Creates a Json Feature from a {@link LocationIterableProperties} * * @deprecated use {@link #create(GeometryWithProperties)} instead * @param object * {@link LocationIterableProperties} * @return a GeoJson Feature */ @Deprecated public JsonObject create(final LocationIterableProperties object) { final GeometryWithProperties geometryWithProperties = toGeometryWithProperties(object); return create(geometryWithProperties); } /** * Creates a Polygon type GeoJson Feature * * @param polygon * geometry * @return a GeoJson Feature */ public GeoJsonObject create(final Polygon polygon) { return this.create(polygon.closedLoop(), GeoJsonType.POLYGON); } /** * Creates a LineString type GeoJson Feature * * @param polyLine * geometry * @return a GeoJson Feature */ public GeoJsonObject create(final PolyLine polyLine) { return this.create(polyLine, GeoJsonType.LINESTRING); } /** * Creates a GeoJson FeatureCollection containing a list of GeoJsonObject Features * * @param objects * the features * @return a GeoJson FeatureCollection */ public GeoJsonObject createFeatureCollection(final Iterable objects) { final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE_COLLECTION); final JsonArray features = new JsonArray(); int counter = 0; for (final GeoJsonObject object : objects) { if (this.logFrequency > 0 && ++counter % this.logFrequency == 0) { logger.info("Processed {} features.", counter); } if (!Optional.ofNullable(object.jsonObject().get(TYPE)) .filter(jsonObject -> jsonObject.getAsString().equals(FEATURE)).isPresent()) { throw new CoreException("Illegal GeoJson Type for Feature collection"); } features.add(object.jsonObject()); } result.add(FEATURES, features); return new GeoJsonObject(result); } /** * Creates a GeoJson FeatureCollection containing a list of Features from an iterable of * PropertiesLocated. * * @param iterableOfPropertiesLocated * iterable of PropertiesLocated * @return a GeoJson FeatureCollection */ public GeoJsonObject createFeatureCollectionFromPropertiesLocated( final Iterable iterableOfPropertiesLocated) { final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE_COLLECTION); final JsonArray features = new JsonArray(); int counter = 0; for (final PropertiesLocated propertiesLocated : iterableOfPropertiesLocated) { if (this.logFrequency > 0 && ++counter % this.logFrequency == 0) { logger.info("Processed {} features.", counter); } final GeoJsonObject feature; final Located located = propertiesLocated.getItem(); if (located instanceof Location) { feature = create((Location) located); } else if (located instanceof PolyLine) { feature = create((PolyLine) located); } else if (located instanceof Polygon) { feature = create((Polygon) located); } else if (located instanceof MultiPolyLine) { feature = createMultiLineStrings((MultiPolyLine) located); } else { throw new CoreException("Unrecognized object type {}", located.getClass().getName()); } final JsonObject featureJsonObj = feature.jsonObject(); featureJsonObj.add(PROPERTIES, propertiesLocated.getProperties()); features.add(feature.jsonObject()); } result.add(FEATURES, features); return new GeoJsonObject(result); } /** * Creates a GeoJson FeatureCollection from an iterable of GeoJsonObject * * @param geoJsonObjects * a iterable of GeoJsonObject * @return a GeoJson FeatureCollection */ public GeoJsonObject createFromGeoJson(final Iterable geoJsonObjects) { final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE_COLLECTION); final JsonArray features = new JsonArray(); int counter = 0; for (final GeoJsonObject object : geoJsonObjects) { if (this.logFrequency > 0 && ++counter % this.logFrequency == 0) { logger.info("Processed {} features.", counter); } features.add(object.jsonObject()); } result.add(FEATURES, features); return new GeoJsonObject(result); } /** * Creates a GeoJson FeatureCollection containing a list of Features * * @param geometriesWithProperties * associated geometries and properties used to build each Feature * @return a GeoJson FeatureCollection */ public GeoJsonObject createFromGeometriesWithProperties( final Iterable geometriesWithProperties) { final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE_COLLECTION); final JsonArray features = new JsonArray(); int counter = 0; for (final GeometryWithProperties geometryWithProperties : geometriesWithProperties) { if (this.logFrequency > 0 && ++counter % this.logFrequency == 0) { logger.info("Processed {} features.", counter); } features.add(create(geometryWithProperties)); } result.add(FEATURES, features); return new GeoJsonObject(result); } /** * Creates a GeometryCollection type Feature containing geometries derived from a collection of * {@link LocationIterableProperties}. Note: feature parameters are not present * in the resulting GeometryCollection and must be handled separately to avoid data loss. * * @deprecated use {@link #createGeometryCollectionFeature(Iterable)} instead * @param objects * used to build each geometry * @return a GeoJson Feature */ @Deprecated public GeoJsonObject createGeometryCollection( final Iterable objects) { return createGeometryCollectionFeature(toGeometriesWithProperties(objects)); } /** * Creates a GeometryCollection type Feature containing geometries derived from a collection of * {@link GeometryWithProperties}. Note: feature parameters are not present in * the resulting GeometryCollection and must be handled separately to avoid data loss. * * @param geometriesWithProperties * used to build each geometry * @return a GeoJson Feature */ public GeoJsonObject createGeometryCollectionFeature( final Iterable geometriesWithProperties) { final JsonObject geometryCollection = new JsonObject(); geometryCollection.addProperty(TYPE, GEOMETRY_COLLECTION); final Map>> geometryMap = new HashMap<>(); int counter = 0; for (final GeometryWithProperties geometryWithProperties : geometriesWithProperties) { if (this.logFrequency > 0 && ++counter % this.logFrequency == 0) { logger.info("Processed {} geometries.", counter); } final Iterable geometry = geometryWithProperties.getGeometry(); final GeoJsonType geoJsonType; if (geometry instanceof Location) { geoJsonType = GeoJsonType.POINT; } else if (geometry instanceof Polygon) { geoJsonType = GeoJsonType.POLYGON; } else if (geometry instanceof PolyLine) { geoJsonType = GeoJsonType.LINESTRING; } else { throw new CoreException("Unrecognized object type {}", geometry.getClass().getSimpleName()); } geometryMap.computeIfAbsent(geoJsonType, key -> new ArrayList<>()).add(geometry); } final JsonArray geometries = new JsonArray(); // Point vs MultiPoint if (geometryMap.containsKey(GeoJsonType.POINT)) { final List> points = geometryMap.get(GeoJsonType.POINT); if (points.size() > 1) { geometries.add(create(Iterables.translate(points, Iterables::head), GeoJsonType.MULTI_POINT).jsonObject().getAsJsonObject(GEOMETRY)); } else if (points.size() == 1) { geometries.add(create(Iterables.head(points), GeoJsonType.POINT).jsonObject() .getAsJsonObject(GEOMETRY)); } } // Polygon vs MultiPolygon if (geometryMap.containsKey(GeoJsonType.POLYGON)) { final List> polygons = geometryMap.get(GeoJsonType.POLYGON); if (polygons.size() > 1) { geometries.add(createMultiPolygons(Iterables.stream(polygons).map(Polygon::new)) .jsonObject().getAsJsonObject(GEOMETRY)); } else if (polygons.size() == 1) { geometries.add(create(Iterables.head(polygons), GeoJsonType.POLYGON).jsonObject() .getAsJsonObject(GEOMETRY)); } } // LineString vs MultLineString if (geometryMap.containsKey(GeoJsonType.LINESTRING)) { final List> multiPolylines = geometryMap.get(GeoJsonType.LINESTRING); if (multiPolylines.size() > 1) { geometries.add( createMultiLineStrings(Iterables.stream(multiPolylines).map(PolyLine::new)) .jsonObject().getAsJsonObject(GEOMETRY)); } else if (multiPolylines.size() == 1) { geometries.add(create(Iterables.head(multiPolylines), GeoJsonType.LINESTRING) .jsonObject().getAsJsonObject(GEOMETRY)); } } geometryCollection.add(GEOMETRIES, geometries); final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE); result.add(GEOMETRY, geometryCollection); return new GeoJsonObject(result); } /** * Creates a MultiLineString type GeoJson Feature * * @param polyLines * geometry * @return a GeoJson Feature */ public GeoJsonObject createMultiLineStrings(final Iterable polyLines) { // Create the coordinates for each polyline final List objects = new ArrayList<>(); for (final PolyLine polygon : polyLines) { objects.add(this.create(polygon, GeoJsonType.MULTI_LINESTRING)); } final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE); final JsonArray coordinates = new JsonArray(); // Add the coordinates back for the entire object for (final GeoJsonObject object : objects) { coordinates .add(object.jsonObject().getAsJsonObject(GEOMETRY).getAsJsonArray(COORDINATES)); } final JsonObject geometry = new JsonObject(); geometry.addProperty(TYPE, GeoJsonType.MULTI_LINESTRING.getTypeString()); geometry.add(COORDINATES, coordinates); result.add(GEOMETRY, geometry); return new GeoJsonObject(result); } /** * Creates a MultiPolygon type GeoJson Feature * * @param polygons * geometries * @return a GeoJson Feature */ public GeoJsonObject createMultiPolygons(final Iterable polygons) { // Create the coordinates for each polygon final List objects = new ArrayList<>(); for (final Polygon polygon : polygons) { objects.add(this.create(polygon.closedLoop(), GeoJsonType.MULTI_POLYGON)); } final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE); final JsonArray coordinates = new JsonArray(); // Add the coordinates back for the entire object for (final GeoJsonObject object : objects) { final JsonArray subCoordinates = new JsonArray(); subCoordinates .add(object.jsonObject().getAsJsonObject(GEOMETRY).getAsJsonArray(COORDINATES)); coordinates.add(subCoordinates); } final JsonObject geometry = new JsonObject(); geometry.addProperty(TYPE, GeoJsonType.MULTI_POLYGON.getTypeString()); geometry.add(COORDINATES, coordinates); result.add(GEOMETRY, geometry); return new GeoJsonObject(result); } /** * Creates multipolygon from {@link Iterable} of {@link Polygon}s where first polygon is assumed * to be the outer ring and the rest are inner. * * @param polygons * an iterable of polygons where the first is assumed to be the outer polygon in a * multipolygon * @return a MultiPolygon geojson feature with one polygon that geometrically represents a * single outer Atlas Multipolygon */ public GeoJsonObject createOneOuterMultiPolygon(final Iterable polygons) { // Create the coordinates for each polygon final List objects = new ArrayList<>(); for (final Polygon polygon : polygons) { objects.add(this.create(polygon.closedLoop(), GeoJsonType.MULTI_POLYGON)); } final JsonObject result = new JsonObject(); result.addProperty(TYPE, FEATURE); final JsonArray coordinates = new JsonArray(); // Add the coordinates back for the entire object for (final GeoJsonObject object : objects) { coordinates .add(object.jsonObject().getAsJsonObject(GEOMETRY).getAsJsonArray(COORDINATES)); } final JsonArray newCoordinates = new JsonArray(); newCoordinates.add(coordinates); final JsonObject geometry = new JsonObject(); geometry.addProperty(TYPE, GeoJsonType.MULTI_POLYGON.getTypeString()); geometry.add(COORDINATES, newCoordinates); result.add(GEOMETRY, geometry); result.add(PROPERTIES, new JsonObject()); return new GeoJsonObject(result); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonCollection.java ================================================ package org.openstreetmap.atlas.geography.geojson; /** * Interface for all GeoJson collection types {@link GeojsonGeometryCollection} and * {@link GeoJsonFeatureCollection} * * @param * The type of GeoJsonGeometry in the collection * @author jklamer */ public interface GeoJsonCollection extends GeoJson { Iterable getGeoJsonObjects(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonConstants.java ================================================ package org.openstreetmap.atlas.geography.geojson; /** * Utility class for all Geojson Constants * * @author jklamer */ public final class GeoJsonConstants { public static final String TYPE = "type"; public static final String COORDINATES = "coordinates"; public static final String PROPERTIES = "properties"; public static final String GEOMETRY = "geometry"; public static final String GEOMETRIES = "geometries"; public static final String FEATURES = "features"; private GeoJsonConstants() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonFeature.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * From the spec https://tools.ietf.org/html/rfc7946#section-3.2 * * 
 *  A Feature object represents a spatially bounded thing.  Every Feature
 *    object is a GeoJSON object no matter where it occurs in a GeoJSON
 *    text.
 *
 *    o  A Feature object has a "type" member with the value "Feature".
 *
 *    o  A Feature object has a member with the name "geometry".  The value
 *       of the geometry member SHALL be either a Geometry object as
 *       defined above or, in the case that the Feature is unlocated, a
 *       JSON null value.
 *
 *    o  A Feature object has a member with the name "properties".  The
 *       value of the properties member is an object (any JSON object or a
 *       JSON null value).

 *    o  If a Feature has a commonly used identifier, that identifier
 *       SHOULD be included as a member of the Feature object with the name
 *       "id", and the value of this member is either a JSON string or
 *       number.
 *
* * This interface is for all classes with a geojson feature representation. Because a fully formed * geojson geometry is a part of a geojosn feature this interface extends from that. * * @author jklamer */ public interface GeoJsonFeature extends GeoJsonGeometry, GeoJsonProperties { @Override default JsonObject asGeoJson() { return GeoJsonUtils.feature(this); } @Override default GeoJsonType getGeoJsonType() { return GeoJsonType.FEATURE; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonFeatureCollection.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * From the spec https://tools.ietf.org/html/rfc7946#section-3.3 * * 
 *     A GeoJSON object with the type "FeatureCollection" is a
 *    FeatureCollection object.  A FeatureCollection object has a member
 *    with the name "features".  The value of "features" is a JSON array.
 *    Each element of the array is a Feature object as defined above.  It
 *    is possible for this array to be empty.
 *
* * @param * The Type of object that implements the {@link GeoJsonFeature} interface that is * returned by this implementation * @author jklamer */ public interface GeoJsonFeatureCollection extends GeoJsonCollection, GeoJsonProperties { @Override default JsonObject asGeoJson() { return GeoJsonUtils.featureCollection(this); } @Override default GeoJsonType getGeoJsonType() { return GeoJsonType.FEATURE_COLLECTION; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonGeometry.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * From the spec https://tools.ietf.org/html/rfc7946#section-3.1 * * 
 *      A Geometry object represents points, curves, and surfaces in
 *    coordinate space.  Every Geometry object is a GeoJSON object no
 *    matter where it occurs in a GeoJSON text.
 *
* * This interface is for all objects with a geojson Geometry object representation. This encompasses * all the Geojson Geometry types in {@link GeoJsonType#isGeometryType(GeoJsonType)}. * * @author jklamer */ public interface GeoJsonGeometry extends GeoJson { @Override default JsonObject asGeoJson() { return this.asGeoJsonGeometry(); } /** * This returns a Geojson object that is one of the Geometry types * (https://tools.ietf.org/html/rfc7946#section-3.1) * * @return Geojson geometry representation. */ JsonObject asGeoJsonGeometry(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonObject.java ================================================ package org.openstreetmap.atlas.geography.geojson; import java.io.BufferedWriter; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.WritableResource; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; /** * A Geo {@link JsonObject} with properties. * * @author matthieun */ public class GeoJsonObject { private JsonObject jsonObject; protected GeoJsonObject(final JsonObject jsonObject) { this.jsonObject = jsonObject; } public Map getProperties() { final Map result = new HashMap<>(); if (this.jsonObject.get("properties") != null) { final JsonObject propertiesObject = (JsonObject) this.jsonObject.get("properties"); for (final Map.Entry entry : propertiesObject.entrySet()) { result.put(entry.getKey(), entry.getValue().toString()); } } return result; } public JsonObject jsonObject() { return this.jsonObject; } public void makeFeatureCollection() { if (!"FeatureCollection".equals(this.jsonObject.get("type").getAsString())) { final JsonObject result = new JsonObject(); result.addProperty("type", "FeatureCollection"); final JsonArray features = new JsonArray(); features.add(this.jsonObject); result.add("features", features); this.jsonObject = result; } } public void save(final WritableResource output) { final BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(output.write(), StandardCharsets.UTF_8)); try { writer.write(this.jsonObject.toString()); Streams.close(writer); } catch (final Exception e) { Streams.close(writer); throw new CoreException("Could not save geojson object", e); } } public String toPrettyString() { return new GsonBuilder().setPrettyPrinting().create().toJson(this.jsonObject); } @Override public String toString() { return this.jsonObject.toString(); } /*** * Adds a parent member to a FeatureCollection object. This Member will be on the same level as * the "type" and "features" members. * * @param key * member key * @param value * member value * @return GeoJsonObject */ public GeoJsonObject withNewParentMember(final String key, final Object value) { // Check if jsonObject is a FeatureCollection final Map properties = new HashMap<>(); properties.put(key, value); return this.withNewParentMembers(properties); } /*** * Adds multiple members to the FeatureCollection object. * * @param properties * Map of member key and properties * @return GeoJsonObject */ public GeoJsonObject withNewParentMembers(final Map properties) { // Check if jsonObject is a FeatureCollection if (this.jsonObject.get(GeoJsonBuilder.TYPE).getAsString() .equals(GeoJsonBuilder.FEATURE_COLLECTION)) { final Gson gson = new Gson(); properties.forEach((key, value) -> this.jsonObject.add(key, gson.toJsonTree(value))); } return this; } public GeoJsonObject withNewProperties(final Map properties) { final JsonObject propertiesObject; if (this.jsonObject.get("properties") != null) { propertiesObject = (JsonObject) this.jsonObject.get("properties"); this.jsonObject.remove("properties"); } else { propertiesObject = new JsonObject(); } final Gson gson = new Gson(); properties.forEach((key, value) -> propertiesObject.add(key, gson.toJsonTree(value))); this.jsonObject.add("properties", propertiesObject); return this; } public GeoJsonObject withNewProperty(final String key, final Object value) { final Map properties = new HashMap<>(); properties.put(key, value); return this.withNewProperties(properties); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonProperties.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * Interface for all interfaces that have geojson properties. Namely {@link GeoJsonFeature} and * {@link GeoJsonFeatureCollection} From the spec : https://tools.ietf.org/html/rfc7946#section-3.2 * * 
 *      o  A Feature object has a member with the name "properties".  The
 *       value of the properties member is an object (any JSON object or a
 *       JSON null value).
 *
* * @author jklamer */ public interface GeoJsonProperties { /** * This returns the geojson properties associate with the * * @return a JsonObject for the "properties" field */ JsonObject getGeoJsonProperties(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonSaver.java ================================================ package org.openstreetmap.atlas.geography.geojson; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.MultiIterable; /** * Save some geometry items to a resource, as GeoJson * * @author matthieun */ public final class GeoJsonSaver { public static void save(final Iterable> geometries, final WritableResource destination) { final GeoJsonObject object = new GeoJsonBuilder().create(Iterables.translate(geometries, polyLine -> new GeoJsonBuilder.LocationIterableProperties(polyLine, Maps.hashMap()))); save(object, destination); } public static void saveMultipolygon(final Iterable geometries, final WritableResource destination) { final Iterable outers = Iterables.translateMulti(geometries, multiPolygon -> multiPolygon.outers()); final Iterable inners = Iterables.translateMulti(geometries, multiPolygon -> multiPolygon.inners()); final Iterable multi = new MultiIterable<>(outers, inners); save(multi, destination); } private static void save(final GeoJsonObject object, final WritableResource destination) { final JsonWriter writer = new JsonWriter(destination); writer.write(object.jsonObject()); writer.close(); } private GeoJsonSaver() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonType.java ================================================ package org.openstreetmap.atlas.geography.geojson; import java.util.EnumSet; import org.openstreetmap.atlas.exception.CoreException; import com.google.gson.JsonObject; /** * Geojson types case sensitive from the specification definition section 1.4: * https://tools.ietf.org/html/rfc7946#section-1.4 * * 
 *o  Inside this document, the term "geometry type" refers to seven
 *       case-sensitive strings: "Point", "MultiPoint", "LineString",
 *       "MultiLineString", "Polygon", "MultiPolygon", and
 *       "GeometryCollection".
 *o  As another shorthand notation, the term "GeoJSON types" refers to
 *       nine case-sensitive strings: "Feature", "FeatureCollection", and
 *       the geometry types listed above.
 *
* * The reasoning for this being an enum also comes from the spec section 7: * https://tools.ietf.org/html/rfc7946#section-7 * * 
 *      Implementations MUST NOT extend the fixed set of GeoJSON types:
 *    FeatureCollection, Feature, Point, LineString, MultiPoint, Polygon,
 *    MultiLineString, MultiPolygon, and GeometryCollection.
 *
* * @author matthieun * @author mgostintsev * @author jklamer */ public enum GeoJsonType { FEATURE("Feature"), FEATURE_COLLECTION("FeatureCollection"), POINT("Point"), MULTI_POINT("MultiPoint"), LINESTRING("LineString"), MULTI_LINESTRING("MultiLineString"), POLYGON("Polygon"), MULTI_POLYGON("MultiPolygon"), GEOMETRY_COLLECTION("GeometryCollection"); private static final EnumSet GEOMETRY_TYPES = EnumSet.of(POINT, MULTI_POINT, LINESTRING, MULTI_LINESTRING, POLYGON, MULTI_POLYGON, GEOMETRY_COLLECTION); private static final EnumSet FEATURE_TYPES = EnumSet.of(FEATURE, FEATURE_COLLECTION); private final String typeString; public static GeoJsonType forJson(final JsonObject object) { final String typeString; try { typeString = object.get(GeoJsonConstants.TYPE).getAsString(); } catch (final Exception exception) { throw new CoreException("Invalid geoJson type: {}", object.get(GeoJsonConstants.TYPE)); } return forString(typeString); } public static GeoJsonType forString(final String type) { for (final GeoJsonType value : values()) { if (value.getTypeString().equals(type)) { return value; } } throw new CoreException("Invalid geoJson type: {}", type); } public static boolean isFeatureType(final GeoJsonType type) { return FEATURE_TYPES.contains(type); } public static boolean isGeometryType(final GeoJsonType type) { return GEOMETRY_TYPES.contains(type); } GeoJsonType(final String typeString) { this.typeString = typeString; } public String getTypeString() { return this.typeString; } @Override public String toString() { return this.typeString; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonUtils.java ================================================ package org.openstreetmap.atlas.geography.geojson; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.COORDINATES; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.FEATURES; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.GEOMETRIES; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.GEOMETRY; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.PROPERTIES; import static org.openstreetmap.atlas.geography.geojson.GeoJsonConstants.TYPE; import static org.openstreetmap.atlas.geography.geojson.GeoJsonType.POLYGON; import java.util.Optional; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * These are utility functions that well help you create GeoJSON! * * @author hallahan */ public final class GeoJsonUtils { public static final String IDENTIFIER = "identifier"; public static final String OSM_IDENTIFIER = "osmIdentifier"; public static final String ITEM_TYPE = "itemType"; private static final Logger logger = LoggerFactory.getLogger(GeoJsonUtils.class); /** * Creates a GeoJSON Polygon geometry from a bounds. * * @param bounds * A bounds. * @return A GeoJSON Polygon geometry JsonObject. */ public static JsonObject boundsToPolygonGeometry(final Rectangle bounds) { final JsonArray outerRing = new JsonArray(); final Iterable locations = bounds.closedLoop(); for (final Location location : locations) { outerRing.add(coordinate(location)); } final JsonArray coordinates = new JsonArray(); coordinates.add(outerRing); return geometry(POLYGON, coordinates); } /** * From a location, we get a Latitude / Longitude Json Array [ latitude, longitude ] * * @param location * An atlas location * @return JsonArray [ longitude, latitude ] coordinate. */ public static JsonArray coordinate(final Location location) { return coordinate(location.getLongitude().asDegrees(), location.getLatitude().asDegrees()); } /** * Slightly more explicit, you provide a double longitude and latitude. * * @param longitude * The longitude (x). * @param latitude * The latitude (y). * @return JsonArray [ longitude, latitude ] coordinate. */ public static JsonArray coordinate(final double longitude, final double latitude) { final JsonArray coordinate = new JsonArray(); coordinate.add(new JsonPrimitive(longitude)); coordinate.add(new JsonPrimitive(latitude)); return coordinate; } public static JsonObject feature(final GeoJsonFeature geoJsonFeature) { if (!geoJsonFeature.getGeoJsonType().equals(GeoJsonType.FEATURE)) { logger.warn( "Constructing GeoJson Feature Json for something with incorrect Geojson type: object {} with type {}", geoJsonFeature, geoJsonFeature.getGeoJsonType()); } return GeoJsonUtils.feature(geoJsonFeature.asGeoJsonGeometry(), geoJsonFeature.getGeoJsonProperties()); } /** * Creates a GeoJSON Feature with a geometry and properties object. * * @param geometry * JsonObject that is the geometry. * @param properties * JsonObject that is the properties. * @return GeoJSON Feature as JsonObject. */ public static JsonObject feature(final JsonObject geometry, final JsonObject properties) { final JsonObject feature = new JsonObject(); feature.addProperty(TYPE, GeoJsonType.FEATURE.getTypeString()); feature.add(GEOMETRY, geometry); feature.add(PROPERTIES, properties); return feature; } public static JsonObject featureCollection( final GeoJsonFeatureCollection featureCollection) { if (!featureCollection.getGeoJsonType().equals(GeoJsonType.FEATURE_COLLECTION)) { logger.warn( "Constructing GeoJson Feature Json for something with incorrect Geojson type: object {} with type {}", featureCollection, featureCollection.getGeoJsonType()); } return GeoJsonUtils.featureCollection(featureCollection.getGeoJsonObjects(), featureCollection.getGeoJsonProperties()); } public static JsonObject featureCollection( final Iterable featureObjects, final JsonObject properties) { final JsonObject featureCollection = new JsonObject(); featureCollection.addProperty(TYPE, GeoJsonType.FEATURE_COLLECTION.getTypeString()); final JsonArray features = new JsonArray(); Iterables.stream(featureObjects).map(GeoJsonUtils::feature).forEach(features::add); featureCollection.add(FEATURES, features); featureCollection.add(PROPERTIES, properties); return featureCollection; } public static JsonObject geometry( final GeojsonGeometryCollection geojsonGeometryCollection) { if (!geojsonGeometryCollection.getGeoJsonType().equals(GeoJsonType.GEOMETRY_COLLECTION)) { logger.warn( "Constructing GeoJson Geometry Collection Json for something with incorrect Geojson type: object {} with type {}", geojsonGeometryCollection, geojsonGeometryCollection.getGeoJsonType()); } final JsonObject geometry = new JsonObject(); final JsonArray geometries = new JsonArray(); geojsonGeometryCollection.getGeoJsonObjects() .forEach(geoJsonGeometry -> geometries.add(geoJsonGeometry.asGeoJsonGeometry())); geometry.addProperty(TYPE, GeoJsonType.GEOMETRY_COLLECTION.getTypeString()); geometry.add(GEOMETRIES, geometries); return geometry; } public static JsonObject geometry(final GeoJsonType type, final JsonArray coordinates) { Validate.isTrue(GeoJsonType.isGeometryType(type), "Type is not geometry type. "); Validate.isTrue(!type.equals(GeoJsonType.GEOMETRY_COLLECTION), "Geometry Collection cannot be represented by coordinate array"); final JsonObject geometry = new JsonObject(); geometry.addProperty(TYPE, type.getTypeString()); geometry.add(COORDINATES, coordinates); return geometry; } /** * An iterable of locations will turn into a JsonArray of Longitude, Latitude coordinates. * * @param locations * An iterable of locations * @return A JsonArray of Longitude, Latitude coordinates. */ public static JsonArray locationsToCoordinates(final Iterable locations) { final JsonArray coordinates = new JsonArray(); for (final Location point : locations) { coordinates.add(coordinate(point)); } return coordinates; } /** * Convert an atlas {@link MultiPolygon} into it's geojson coordinate representation * * @param multiPolygon * the multiPolygon * @return the coordinate array */ public static JsonArray multiPolygonToCoordinates(final MultiPolygon multiPolygon) { final JsonArray polygons = new JsonArray(); multiPolygon.getOuterToInners().forEach((outer, inners) -> polygons .add(GeoJsonUtils.polygonToCoordinates(outer, Optional.of(inners)))); return polygons; } /** * Convert an atlas {@link Polygon} into it's geojson coordinate representation * * @param polygon * the polygon * @return the coordinate array */ public static JsonArray polygonToCoordinates(final Polygon polygon) { return GeoJsonUtils.polygonToCoordinates(polygon, Optional.empty()); } private static JsonArray polygonToCoordinates(final Polygon outer, final Optional> inners) { final JsonArray polygon = new JsonArray(); polygon.add(GeoJsonUtils.locationsToCoordinates(outer.closedLoop())); inners.ifPresent(innerPolygons -> innerPolygons.forEach(innerPolygon -> polygon .add(GeoJsonUtils.locationsToCoordinates(innerPolygon.closedLoop())))); return polygon; } private GeoJsonUtils() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/GeojsonGeometryCollection.java ================================================ package org.openstreetmap.atlas.geography.geojson; import com.google.gson.JsonObject; /** * From the spec https://tools.ietf.org/html/rfc7946#section-3.1.8 * * 
 *      A GeoJSON object with type "GeometryCollection" is a Geometry object.
 *    A GeometryCollection has a member with the name "geometries".  The
 *    value of "geometries" is an array.  Each element of this array is a
 *    GeoJSON Geometry object.  It is possible for this array to be empty.
 *
 *    Unlike the other geometry types described above, a GeometryCollection
 *    can be a heterogeneous composition of smaller Geometry objects.  For
 *    example, a Geometry object in the shape of a lowercase roman "i" can
 *    be composed of one point and one LineString.
 *
 *    GeometryCollections have a different syntax from single type Geometry
 *    objects (Point, LineString, and Polygon) and homogeneously typed
 *    multipart Geometry objects (MultiPoint, MultiLineString, and
 *    MultiPolygon) but have no different semantics.  Although a
 *    GeometryCollection object has no "coordinates" member, it does have
 *    coordinates: the coordinates of all its parts belong to the
 *    collection.  The "geometries" member of a GeometryCollection
 *    describes the parts of this composition.  Implementations SHOULD NOT
 *    apply any additional semantics to the "geometries" array.
 *
 *    To maximize interoperability, implementations SHOULD avoid nested
 *    GeometryCollections.  Furthermore, GeometryCollections composed of a
 *    single part or a number of parts of a single type SHOULD be avoided
 *    when that single part or a single object of multipart type
 *    (MultiPoint, MultiLineString, or MultiPolygon) could be used instead.
 *
* * This interface is for all classes with a GeoJsonGeometryCollection representation. * * @param * The Type of object that implements the {@link GeoJsonGeometry} interface that is * returned by this implementation * @author jklamer */ public interface GeojsonGeometryCollection extends GeoJsonCollection, GeoJsonGeometry { @Override default JsonObject asGeoJson() { return this.asGeoJsonGeometry(); } @Override default JsonObject asGeoJsonGeometry() { return GeoJsonUtils.geometry(this); } @Override default GeoJsonType getGeoJsonType() { return GeoJsonType.GEOMETRY_COLLECTION; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/GeoJsonParser.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser; import java.io.Serializable; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.GeoJsonItem; /** * @author Yazad Khambata */ public interface GeoJsonParser extends Serializable { GeoJsonItem deserialize(String geoJson); GeoJsonItem deserialize(Map map); T deserializeExtension(String json, Class targetClass); T deserializeExtension(Map map, Class targetClass); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/README.md ================================================ # GeoJSON Parsing ## Why do we need a `GeoJson` parser? Won't a regular `JSON` parser do? While the `GeoJSON` spec is well defined [RFC](https://tools.ietf.org/html/rfc7946), the specification is none the less very loose. This presents 2 challenges that are not common in `JSON` parsing where the schema is strictly defined. 1. Polymorphic nature of fields (`coordinates`), **A Point** ```json { "type": "Point", "coordinates": [40, 10] } ``` **A Polygon** ```json { "type": "Polygon", "coordinates": [ [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] ] } ``` Notice the structure in terms of fields is completely the same, but the value type is very different. For `Point` `coordinates` are an array of exactly 2 numbers of decimal type and for the `Polygon` it is an array-of-array of such point coordinates (itself represented as arrays). This presents a unique challenge for mapping Java objects in a way that preserves type safety while still has room to accommodate different types. 2. Polymorphic nature of fields (`geometry`, `geometries` and `features`). a. A `GeometryWithCoordinates` (all geometries except `GeometryCollection`) have coordinates. b. A `Feature` has one associated `geometry`, while a GeometryCollection has several assocated nested `geometries`. c. A FeatureCollection could have several associated `features`. d. `GeometryCollection`s can nest `GeometryCollection`s recursively. Example, **A Feature containing a LineString** ```json { "type": "Feature", "geometry": { "type": "LineString", "coordinates": [ [-122.009566, 37.33531],[-122.031007,37.390535],[-122.028932,37.332451] ] } } ``` **A recursively nested GeometryCollection** ```json { "type": "GeometryCollection", "geometries": [ ..., { "type": "GeometryCollection", "geometries": [ ..., { "type": "GeometryCollection", "geometries": [ ..., { "type": "GeometryCollection", "geometries": [ ... ] } ] } ] } ] } ``` Since the building block of a `Geometry` is itself highly dynamic, this dynamism "leaks" into all other major structures like `Feature`s, `FeatureCollection`s and `GeometryCollection`s. A challenge that is not suited for regular JSON parsing. 3. Very liberal specification The RFC supports the concept of [foreign fields](https://tools.ietf.org/html/rfc7946#section-6.1) and fields like `properties` are entirely foreign. This means that implementations that rely heavily in foreign fields would have to map the properties section to a map-of-map-of-map... or it's like. This makes coding against it difficult and also introduces bugs the the compiler cannot protect against due to loss of type-safety. Example, ```json { "type": "Feature", "bbox": [ ... ], "geometry": { ... }, "properties": { "featureChangeType": "ADD", "metadata": { "somekey1": "some value 1", "somekey2": "some value 2" }, "description": { "type": "UPDATE", "descriptors": [ { "name": "TAG", "type": "ADD", "key": "c", "value": "3" }, { "name": "TAG", "type": "UPDATE", "key": "b", "value": "2a", "originalValue": "2" }, { "name": "TAG", "type": "REMOVE", "key": "a", "value": "1" }, { "name": "GEOMETRY", "type": "ADD", "position": "5/5", "afterView": "LINESTRING (-122.028932 37.332451, -122.052138 37.317585, -122.0304871 37.3314171)" }, { "name": "GEOMETRY", "type": "REMOVE", "position": "0/5", "beforeView": "LINESTRING (-122.052138 37.317585, -122.0304871 37.3314171, -122.028932 37.332451)" }, { "name": "PARENT_RELATION", "type": "ADD", "afterView": "3" }, { "name": "PARENT_RELATION", "type": "REMOVE", "beforeView": "1" }, { "name": "START_NODE", "type": "UPDATE", "beforeView": "1", "afterView": "10" }, { "name": "END_NODE", "type": "UPDATE", "beforeView": "2", "afterView": "20" } ] }, "entityType": "EDGE", "completeEntityClass": "org.openstreetmap.atlas.geography.atlas.complete.CompleteEdge", "identifier": 123, "tags": { "b": "2a", "c": "3" }, " relations": [ 2, 3 ], "startNode": 10, "endNode": 20, "WKT": "LINESTRING (-122.009566 37.33531, -122.031007 37.390535, -122.028932 37.332451, -122.052138 37.317585, -122.0304871 37.3314171)", "bboxWKT": "POLYGON ((-122.052138 37.317585, -122.052138 37.390535, -122.009566 37.390535, -122.009566 37.317585, -122.052138 37.317585))" } } ``` Notice how difficult it would be to access the originalValue inside properties#decription#descriptor[index] without type information. In my exploration I didn't come across a framework that solves these problems to my satisfaction. In fact tools that deal with JsonSchema and JsonSchema-to-Java mapping seems to break due to the complexity of the geojson schema. ## What does this GeoJSON parser support? 1. Map GeoJSON to all standard types defined in the specification, including Feature, `FeatureCollection`, `Point`, `MultiPoint`, `LineString`, `MultiLineString`, `Polygon`, `MultiPolygon`. 2. Support complex `coordinates` structures. 3. Support for 2D and 3D `Bbox`es. 4. Full support for Foreign Fields. 5. Highly functional "auto"-`mapper` for properties, that automatically maps properties to your user-defined bean (subject to some restrictions). ## How do I use it? If you are just interested in the standard types and do not rely heavily on `Foreign Fields` or `properties` you just need one line of code. ```java final GeoJsonItem geoJsonItem = GeoJsonParserJacksonImpl.instance.deserialize(json); ``` If you do wish to map your properties to a deep nested `bean` or `pojo`, you will need an additional line while accessing properties (assume MyClass is your deep-nested POJO), ```java final MyClass myObj = geoJsonItem.getProperties().asType(MyClass.class); ``` ## What is (will) not (be) supported? Very detailed validations to ensure validation of the geometries. Example, [southwardly to northwardly](https://tools.ietf.org/html/rfc7946#section-5), [Anti-meridian](https://tools.ietf.org/html/rfc7946#section-3.1.9). These feature are available in `AtlasEntity`-ies. Only basic structural validations are added here. ## A note on Mutability GeoJSONs i.e all `Geometry`-ies, `Feature`s and `FeatureCollection`s, and their constituents like `Coordinates`, `Bbox`, `Position`, etc are immutable. However your custom class which will be auto-mapped from the `properties` requires setter methods. ## A note of `properties` auto-mapping `Properties` in the `GeoJSONItem` can be automatically mapped to a user-defined POJO or bean. This doesn't require the user to explicitly call setters/getters. Under the hood the auto-mapper is a custom recursive `BeanUtilsBean#copyProperties` which is collection and nexted structure aware. This means that for the auto-mapper to work you need to follow the `JavaBean` standards and some additional restrictions. ### `Properties` auto-mapping support and restrictions #### Data-Types supported * Scalar Values - String - [Java Wrapper Types](https://en.wikipedia.org/wiki/Primitive_wrapper_class) * 1D Arrays of Scalar Values. * A Java Bean that contains the above. * 1D arrays of Java Beans. * A Map of scalar values in it's `key` and `value`. #### Parsed Geo JSON Geometry to Atlas Geometry mapping | Geo Json Geometry | Atlas Geometry | | -------------------------|---------------------------------------------------------------------------------------| | Point | `Location` | | MultiPoint | `List` | | LineString | `PolyLine` | | MultiLineString | `List` | | Polygon | `Polygon` | | MultiPolygon | `List` | | GeometryCollection | *N/A* (Non-GeometryCollection children can be converted to the above.) | #### Restrictions | Restriction | Reason | Workaround | |------------------------------------------------|-------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------| | Public Constructor and setters | Needed for reflectively constructing and setting data in the bean | NA | | Arrays instead of `List`s or `Set`s | This makes auto mapping of Nested structures easier, since generic info is lost at runtime. * | Expose a getter method that converts the array to your `Collection` | | Maps of scalars only | Keys in JSON cannot cantain objects. Values may contain objects but the generic info is lost in Java. * | May be supported in the future, if the use case presents. In the mean while you can construct an array of `Pair`s as a workaround. | \* The loss of generic info at runtime in Java is due to [Type Erasure](https://docs.oracle.com/javase/tutorial/java/generics/erasure.html). While there are workarounds available to get that at runtime, they are awkward and would make the API complicated. ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/annotation/Foreign.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.annotation; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * @author Yazad Khambata */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE, ElementType.FIELD }) public @interface Foreign { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/AbstractGeoJsonItem.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.tuple.Pair; import org.openstreetmap.atlas.geography.geojson.parser.domain.bbox.Bbox; import org.openstreetmap.atlas.geography.geojson.parser.domain.bbox.Dimensions; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; import org.openstreetmap.atlas.geography.geojson.parser.domain.properties.Properties; /** * @author Yazad Khambata */ public abstract class AbstractGeoJsonItem implements GeoJsonItem { private Bbox bbox; private Properties properties; private ForeignFields foreignFields; public static Object extractBbox(final Map map) { final List list = (List) map.get("bbox"); if (CollectionUtils.isEmpty(list)) { return null; } final Object rawBbox = list.toArray(new Double[list.size()]); return rawBbox; } public static Map extractPropertiesMap(final Map map) { final Map properties = (Map) map.get("properties"); return properties; } protected static Map extractForeignFields(final Map map, final HashSet exclude) { return new HashMap<>(map).entrySet().stream() .map(entry -> Pair.of(entry.getKey(), entry.getValue())) .filter(pair -> !exclude.contains(pair.getKey())) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } private static Bbox toBbox(final Map map) { final Double[] coordinates = (Double[]) extractBbox(map); if (coordinates == null) { return null; } return Dimensions.toBbox(coordinates); } public AbstractGeoJsonItem(final Bbox bbox, final Properties properties, final ForeignFields foreignFields) { this.bbox = bbox; this.properties = properties; this.foreignFields = foreignFields; } public AbstractGeoJsonItem(final Map map, final ForeignFields foreignFields) { this(toBbox(map), new Properties(extractPropertiesMap(map)), foreignFields); Validate.notEmpty(map, "input map is empty."); } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } @Override public Bbox getBbox() { return this.bbox; } @Override public ForeignFields getForeignFields() { return this.foreignFields; } @Override public Properties getProperties() { return this.properties; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/GeoJsonItem.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base; import java.io.Serializable; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.Type; import org.openstreetmap.atlas.geography.geojson.parser.domain.bbox.Bbox; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.SupportsForeigners; import org.openstreetmap.atlas.geography.geojson.parser.domain.properties.Properties; /** * @author Yazad Khambata */ public interface GeoJsonItem extends SupportsForeigners, Serializable { Bbox getBbox(); Properties getProperties(); Type getType(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/type/FeatureType.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base.type; import java.util.Map; import org.apache.commons.lang3.reflect.ConstructorUtils; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.feature.AbstractFeature; import org.openstreetmap.atlas.geography.geojson.parser.domain.feature.Feature; import org.openstreetmap.atlas.geography.geojson.parser.domain.feature.FeatureCollection; /** * @author Yazad Khambata */ public enum FeatureType implements Type { FEATURE("Feature", Feature.class), FEATURE_COLLECTION("FeatureCollection", FeatureCollection.class, true); private String typeValue; private Class concreteClass; private boolean collection; public static AbstractFeature construct(final FeatureType geometryType, final GeoJsonParser goeJsonParser, final Map map) { try { final Class concreteClass = geometryType.getConcreteClass(); return ConstructorUtils.invokeConstructor(concreteClass, goeJsonParser, map); } catch (final Exception e) { throw new RuntimeException(e); } } FeatureType(final String typeValue, final Class concreteClass) { this(typeValue, concreteClass, false); } FeatureType(final String typeValue, final Class concreteClass, final boolean collection) { this.typeValue = typeValue; this.concreteClass = concreteClass; this.collection = collection; } @Override public AbstractFeature construct(final GeoJsonParser goeJsonParser, final Map map) { return FeatureType.construct(this, goeJsonParser, map); } @Override public Class getConcreteClass() { return this.concreteClass; } @Override public String getTypeValue() { return this.typeValue; } @Override public boolean isCollection() { return this.collection; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/type/GeometryType.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base.type; import java.util.Arrays; import java.util.Map; import java.util.function.Predicate; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.reflect.ConstructorUtils; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.Geometry; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.GeometryCollection; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.LineString; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.MultiLineString; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.MultiPoint; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.MultiPolygon; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.Point; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.Polygon; /** * @author Yazad Khambata */ public enum GeometryType implements Type { POINT("Point", Point.class), MULTI_POINT("MultiPoint", MultiPoint.class), LINE_STRING("LineString", LineString.class), MULTI_LINE_STRING("MultiLineString", MultiLineString.class), POLYGON("Polygon", Polygon.class), MULTI_POLYGON("MultiPolygon", MultiPolygon.class), GEOMETRY_COLLECTION("GeometryCollection", GeometryCollection.class, true); private String typeValue; private Class concreteClass; private boolean collection; public static Geometry construct(final GeometryType geometryType, final GeoJsonParser goeJsonParser, final Map map) { try { final Class concreteClass = geometryType.getConcreteClass(); if (geometryType.isCollection()) { return ConstructorUtils.invokeConstructor(concreteClass, goeJsonParser, map); } return ConstructorUtils.invokeConstructor(concreteClass, map); } catch (final Exception e) { throw new RuntimeException(e); } } public static GeometryType fromConcreteClass(final Class concreteClass) { Validate.notNull(concreteClass); final Predicate filterFunction = geometryType -> geometryType .getConcreteClass().equals(concreteClass); return associatedGeometryType(concreteClass.toString(), filterFunction); } public static GeometryType fromTypeValue(final String typeValue) { Validate.notEmpty(typeValue); final Predicate filterFunction = geometryType -> geometryType.getTypeValue() .equals(typeValue); return associatedGeometryType(typeValue, filterFunction); } private static GeometryType associatedGeometryType(final String typeValue, final Predicate filterFunction) { return Arrays.stream(GeometryType.values()).filter(filterFunction).findFirst() .orElseThrow(() -> new IllegalArgumentException(typeValue)); } GeometryType(final String typeValue, final Class concreteClass) { this(typeValue, concreteClass, false); } GeometryType(final String typeValue, final Class concreteClass, final boolean collection) { this.typeValue = typeValue; this.concreteClass = concreteClass; this.collection = collection; } @Override public Geometry construct(final GeoJsonParser goeJsonParser, final Map map) { return GeometryType.construct(this, goeJsonParser, map); } @Override public Class getConcreteClass() { return this.concreteClass; } @Override public String getTypeValue() { return this.typeValue; } @Override public boolean isCollection() { return this.collection; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/type/Type.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base.type; import java.util.Map; import org.apache.commons.lang3.EnumUtils; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.GeoJsonItem; /** * @author Yazad Khambata */ public interface Type { static > Type fromName(Class subTypeClass, String typeValue) { return EnumUtils.getEnumList((Class) subTypeClass).stream().map(item -> (Type) item) .filter(item -> item.getTypeValue().equals(typeValue)).findFirst() .orElseThrow(() -> new IllegalArgumentException(typeValue)); } GeoJsonItem construct(GeoJsonParser goeJsonParser, Map map); Class getConcreteClass(); String getTypeValue(); boolean isCollection(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/base/type/TypeUtil.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.base.type; import java.util.Arrays; import org.apache.commons.lang3.Validate; import com.google.common.collect.Streams; /** * @author Yazad Khambata */ public final class TypeUtil { public static Type identifyStandardType(final String typeAsStr) { Validate.notEmpty(typeAsStr, "typeAsStr is EMPTY!"); final Type identifiedType = Streams .concat(Arrays.stream(FeatureType.values()), Arrays.stream(GeometryType.values())) .map(type -> (Type) type).filter(type -> type.getTypeValue().equals(typeAsStr)) .findFirst().orElseThrow(() -> new IllegalArgumentException(typeAsStr)); return identifiedType; } private TypeUtil() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/bbox/AbstractBbox.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.bbox; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; /** * @author Yazad Khambata */ public abstract class AbstractBbox implements Bbox { private Dimensions dimensions; public AbstractBbox(final Dimensions dimensions) { this.dimensions = dimensions; } @Override public Dimensions applicableDimensions() { return this.dimensions; } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/bbox/Bbox.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.bbox; import java.io.Serializable; import java.util.List; /** * @author Yazad Khambata */ public interface Bbox extends Serializable { Dimensions applicableDimensions(); List toList(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/bbox/Bbox2D.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.bbox; import java.util.Arrays; import java.util.List; /** * @author Yazad Khambata */ public class Bbox2D extends AbstractBbox { private final Double coordinate1; private final Double coordinate2; private final Double coordinate3; private final Double coordinate4; private static final int ZERO = 0; private static final int ONE = 1; private static final int TWO = 2; private static final int THREE = 3; public Bbox2D(final Double... coordinates) { this(Dimensions.TWO_DIMENSIONAL, coordinates); } Bbox2D(final Dimensions dimensions, final Double... coordinates) { super(dimensions); dimensions.validate(coordinates); this.coordinate1 = coordinates[ZERO]; this.coordinate2 = coordinates[ONE]; this.coordinate3 = coordinates[TWO]; this.coordinate4 = coordinates[THREE]; } public Double getCoordinate1() { return this.coordinate1; } public Double getCoordinate2() { return this.coordinate2; } public Double getCoordinate3() { return this.coordinate3; } public Double getCoordinate4() { return this.coordinate4; } @Override public List toList() { return Arrays.asList(this.coordinate1, this.coordinate2, this.coordinate3, this.coordinate4); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/bbox/Bbox3D.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.bbox; import java.util.ArrayList; import java.util.Arrays; import java.util.List; /** * @author Yazad Khambata */ public class Bbox3D extends Bbox2D { private final Double coordinate5; private final Double coordinate6; private static final int FOUR = 4; private static final int FIVE = 4; public Bbox3D(final Double... coordinates) { super(Dimensions.THREE_DIMENSIONAL, coordinates); this.coordinate5 = coordinates[FOUR]; this.coordinate6 = coordinates[FIVE]; } public Double getCoordinate5() { return this.coordinate5; } public Double getCoordinate6() { return this.coordinate6; } @Override public List toList() { final List list = new ArrayList<>(super.toList()); list.addAll(Arrays.asList(this.coordinate5, this.coordinate6)); return list; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/bbox/Dimensions.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.bbox; import java.util.Arrays; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.reflect.ConstructorUtils; /** * @author Yazad Khambata */ public enum Dimensions { TWO_DIMENSIONAL(2, Bbox2D.class), THREE_DIMENSIONAL(3, Bbox3D.class); private int numberOfDimensions; private Class bboxClass; public static Bbox toBbox(final Double... coordinates) { Validate.notEmpty(coordinates); final int length = coordinates.length; final int minCoordinates = 4; Validate.isTrue(length >= minCoordinates, "length: %s.", length); final Dimensions dimensions = Arrays.stream(Dimensions.values()) .filter(theseDimensions -> theseDimensions.getNumberOfCoordinates() == length) .findFirst() .orElseThrow(() -> new IllegalArgumentException(Arrays.toString(coordinates))); try { return ConstructorUtils.invokeConstructor(dimensions.getBboxClass(), coordinates); } catch (final Exception e) { throw new IllegalStateException(e); } } Dimensions(final int numberOfDimensions, final Class bboxClass) { this.numberOfDimensions = numberOfDimensions; this.bboxClass = bboxClass; } public Class getBboxClass() { return this.bboxClass; } /** * Per https://tools.ietf.org/html/rfc7946#section-5 * * 
     *     The value of the bbox member MUST be an array of
     *    length 2*n where n is the number of dimensions represented in the
     *    contained geometries, with all axes of the most southwesterly point
     *    followed by all axes of the more northeasterly point.
     *
* * @return the number of coordinates. */ public int getNumberOfCoordinates() { final int multiply = 2; return this.getNumberOfDimensions() * multiply; } public int getNumberOfDimensions() { return this.numberOfDimensions; } public void validate(final Double... coordinates) { Validate.notEmpty(coordinates, "coordinates is EMPTY for %s.", this); final int actual = coordinates.length; final int expected = getNumberOfCoordinates(); Validate.isTrue(actual == expected, "coordinates.length actual {}; expected: {}.", actual, expected); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/feature/AbstractFeature.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.feature; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.AbstractGeoJsonItem; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; /** * @author Yazad Khambata */ public abstract class AbstractFeature extends AbstractGeoJsonItem { AbstractFeature(final Map map, final ForeignFields foreignFields) { super(map, foreignFields); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/feature/Feature.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.feature; import java.util.Arrays; import java.util.HashSet; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.FeatureType; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.Type; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.DefaultForeignFieldsImpl; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.Geometry; /** * @author Yazad Khambata */ public class Feature extends AbstractFeature { private Geometry geometry; public Feature(final GeoJsonParser goeJsonParser, final Map map) { super(map, new DefaultForeignFieldsImpl(extractForeignFields(map, new HashSet<>(Arrays.asList("type", "bbox", "geometry", "properties"))))); this.geometry = (Geometry) goeJsonParser .deserialize((Map) map.get("geometry")); } public Geometry getGeometry() { return this.geometry; } @Override public Type getType() { return FeatureType.FEATURE; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/feature/FeatureCollection.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.feature; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.FeatureType; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.Type; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.DefaultForeignFieldsImpl; /** * @author Yazad Khambata */ public class FeatureCollection extends AbstractFeature { private List features; public FeatureCollection(final GeoJsonParser goeJsonParser, final Map map) { super(map, new DefaultForeignFieldsImpl(extractForeignFields(map, new HashSet<>(Arrays.asList("type", "bbox", "features", "properties"))))); this.features = ((List>) map.get("features")).stream() .map(goeJsonParser::deserialize).map(item -> (Feature) item) .collect(Collectors.toList()); } public List getFeatures() { return this.features; } @Override public Type getType() { return FeatureType.FEATURE_COLLECTION; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/foreign/DefaultForeignFieldsImpl.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.foreign; import java.util.Collections; import java.util.Map; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; /** * @author Yazad Khambata */ public class DefaultForeignFieldsImpl implements ForeignFields { private Map valuesAsMap; public DefaultForeignFieldsImpl(final Map valuesAsMap) { this.valuesAsMap = valuesAsMap; } @Override public Map asMap() { if (this.valuesAsMap == null) { return Collections.EMPTY_MAP; } return Collections.unmodifiableMap(this.valuesAsMap); } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } @Override public Object get(final String key) { return this.valuesAsMap.get(key); } @Override public T get(final String key, final Class valueClass) { return (T) this.valuesAsMap.get(key); } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/foreign/ForeignFields.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.foreign; import java.io.Serializable; import java.util.Map; /** * @author Yazad Khambata */ public interface ForeignFields extends Serializable { Map asMap(); T get(String key, Class valueClass); Object get(String key); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/foreign/SupportsForeigners.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.foreign; /** * @author Yazad Khambata */ public interface SupportsForeigners { ForeignFields getForeignFields(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/AbstractGeometry.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.AbstractGeoJsonItem; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.GeometryType; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.Type; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; /** * @author Yazad Khambata */ public abstract class AbstractGeometry extends AbstractGeoJsonItem implements Geometry { AbstractGeometry(final Map map, final ForeignFields foreignFields) { super(map, foreignFields); } @Override public GeometryType getGeometryType() { return GeometryType.fromConcreteClass(this.getClass()); } @Override public Type getType() { return getGeometryType(); } @Override public String getTypeValue() { return getGeometryType().getTypeValue(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/AbstractGeometryWithCoordinateSupport.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.Arrays; import java.util.HashSet; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.DefaultForeignFieldsImpl; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; /** * An abstraction of geometries with coordinates. * * @param * - coordinate data type. * @param * - Atlas Geometry. * @author Yazad Khambata */ public abstract class AbstractGeometryWithCoordinateSupport extends AbstractGeometry implements GeometryWithCoordinates { public static Object extractRawCoordinates(final Map map) { return map.get("coordinates"); } public AbstractGeometryWithCoordinateSupport(final Map map) { super(map, new DefaultForeignFieldsImpl(extractForeignFields(map, new HashSet<>(Arrays.asList("type", "bbox", "coordinates", "properties"))))); } public AbstractGeometryWithCoordinateSupport(final Map map, final ForeignFields foreignFields) { super(map, foreignFields); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/Geometry.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.GeoJsonItem; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.GeometryType; /** * @author Yazad Khambata */ public interface Geometry extends GeoJsonItem { GeometryType getGeometryType(); String getTypeValue(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/GeometryCollection.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.DefaultForeignFieldsImpl; /** * {@link GeometryCollection} nesting inside other {@link GeometryCollection}(s) is NOT allowed. * * @author Yazad Khambata */ public class GeometryCollection extends AbstractGeometry { private List geometries; public GeometryCollection(final GeoJsonParser goeJsonParser, final Map map) { super(map, new DefaultForeignFieldsImpl(extractForeignFields(map, new HashSet<>(Arrays.asList("type", "bbox", "geometries", "properties"))))); this.geometries = ((List>) map.get("geometries")).stream() .map(goeJsonParser::deserialize).map(item -> (Geometry) item) .collect(Collectors.toList()); } public List getGeometries() { return this.geometries; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/GeometryWithCoordinates.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; /** * An abstraction of a geometry with coordinates. * * @param * - value of the coordinates. * @param * - The compatible Atlas Geometry that this geojson can be converted to. * @author Yazad Khambata */ public interface GeometryWithCoordinates extends Geometry { Coordinates getCoordinates(); G toAtlasGeometry(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/LineString.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.geojson.parser.domain.bbox.Bbox; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class LineString extends AbstractGeometryWithCoordinateSupport, PolyLine> { private MultiPoint value; public LineString(final Map map) { super(map, null); this.value = new MultiPoint(map); } @Override public Bbox getBbox() { return this.value.getBbox(); } @Override public Coordinates> getCoordinates() { return this.value.getCoordinates(); } @Override public ForeignFields getForeignFields() { return this.value.getForeignFields(); } @Override public PolyLine toAtlasGeometry() { final List locations = this.value.toAtlasGeometry(); return new PolyLine(locations); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/MultiLineString.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Positions; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class MultiLineString extends AbstractGeometryWithCoordinateSupport>, List> { private List> coordinates; public MultiLineString(final Map map) { super(map, null); this.coordinates = Coordinates .forMultiLineString((List>>) extractRawCoordinates(map)) .getValue(); } @Override public Coordinates>> getCoordinates() { return new Coordinates<>(this.coordinates); } @Override public List toAtlasGeometry() { final List> listsOfLocations = Positions .toCollectionsOfLocations(this.coordinates); return listsOfLocations.stream().map(PolyLine::new).collect(Collectors.toList()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/MultiPoint.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Positions; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class MultiPoint extends AbstractGeometryWithCoordinateSupport, List> { private List coordinates; public MultiPoint(final Map map) { super(map, null); this.coordinates = Coordinates .forMultiPoint((List>) extractRawCoordinates(map)).getValue(); } @Override public Coordinates> getCoordinates() { return new Coordinates<>(this.coordinates); } @Override public List toAtlasGeometry() { return Positions.toLocations(this.coordinates); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/MultiPolygon.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Positions; import org.openstreetmap.atlas.utilities.maps.MultiMap; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class MultiPolygon extends AbstractGeometryWithCoordinateSupport>>, org.openstreetmap.atlas.geography.MultiPolygon> { private List>> coordinates; public MultiPolygon(final Map map) { super(map, null); this.coordinates = Coordinates .forMultiPolygon((List>>>) extractRawCoordinates(map)) .getValue(); } @Override public Coordinates>>> getCoordinates() { return new Coordinates<>(this.coordinates); } @Override public org.openstreetmap.atlas.geography.MultiPolygon toAtlasGeometry() { final MultiMap outersToIneers = new MultiMap<>(); this.coordinates.stream() .map(geojsonPolygon -> geojsonPolygon.stream().map( geojsonLinearRing -> new Polygon(Positions.toLocations(geojsonLinearRing))) .collect(Collectors.toList())) .forEach(polygonList -> outersToIneers.put(polygonList.get(0), polygonList.size() > 1 ? polygonList.subList(1, polygonList.size()) : Collections.emptyList())); return new org.openstreetmap.atlas.geography.MultiPolygon(outersToIneers); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/Point.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Positions; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class Point extends AbstractGeometryWithCoordinateSupport { private Position coordinates; public Point(final Map map) { super(map); this.coordinates = Coordinates.forPoint((List) extractRawCoordinates(map)) .getValue(); } @Override public Coordinates getCoordinates() { return new Coordinates<>(this.coordinates); } @Override public Location toAtlasGeometry() { return Positions.toLocation(this.coordinates); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/Polygon.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.geography.geojson.parser.domain.bbox.Bbox; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Coordinates; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Position; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate.Positions; /** * @author Yazad Khambata */ @SuppressWarnings("squid:S2160") public class Polygon extends AbstractGeometryWithCoordinateSupport>, org.openstreetmap.atlas.geography.Polygon> { private MultiLineString value; public Polygon(final Map map) { super(map, null); this.value = new MultiLineString(map); } @Override public Bbox getBbox() { return this.value.getBbox(); } public Coordinates>> getCoordinates() { return this.value.getCoordinates(); } @Override public ForeignFields getForeignFields() { return this.value.getForeignFields(); } @Override public org.openstreetmap.atlas.geography.Polygon toAtlasGeometry() { return Positions.toAtlasPolygonFromMultiLineString(this.value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/coordinate/Coordinates.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; /** * The coordinates of the geometry. * * @param * - value of the coordinates. * @author Yazad Khambata */ public final class Coordinates { private V value; public static Coordinates> forLineString(final List> data) { return forMultiPoint(data); } public static Coordinates>> forMultiLineString( final List>> data) { return new Coordinates<>(toListOfPositionList(data)); } public static Coordinates> forMultiPoint(final List> data) { return new Coordinates<>(toPositionList(data)); } public static Coordinates>>> forMultiPolygon( final List>>> data) { return new Coordinates<>( data.stream().map(Coordinates::toListOfPositionList).collect(Collectors.toList())); } public static Coordinates forPoint(final List data) { return new Coordinates<>(toPosition(data)); } public static Coordinates>> forPolygon(final List>> data) { // Designed in the specification as a MultiPolygon of size 1. return forMultiLineString(data); } private static List> toListOfPositionList(final List>> data) { Validate.notEmpty(data, "list containing the lists of coordinates is EMPTY."); Validate.isTrue(data.size() >= 1, "multi point coordinates must be at least 1: %s.", data); return data.stream().map(listOfCoords -> toPositionList(listOfCoords)) .collect(Collectors.toList()); } private static Position toPosition(final List data) { Validate.notEmpty(data, "coordinates is EMPTY."); Validate.isTrue(data.size() == 2, "point coordinates is NOT 2: %s.", data); return new Position(data.get(0).doubleValue(), data.get(1).doubleValue()); } private static List toPositionList(final List> data) { Validate.notEmpty(data, "list of coordinates is EMPTY."); Validate.isTrue(data.size() >= 1, "multi point coordinates must be at least 1: %s.", data); return data.stream().map(coords -> toPosition(coords)).collect(Collectors.toList()); } public Coordinates(final V value) { this.value = value; } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } public V getValue() { return this.value; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/coordinate/Position.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate; import java.io.Serializable; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; /** * @author Yazad Khambata */ public class Position implements Serializable { private Double coordinate1; private Double coordinate2; public Position(final Double coordinate1, final Double coordinate2) { this.coordinate1 = coordinate1; this.coordinate2 = coordinate2; } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } public Double getCoordinate1() { return this.coordinate1; } public Double getCoordinate2() { return this.coordinate2; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/geometry/coordinate/Positions.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.coordinate; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.geojson.parser.domain.geometry.MultiLineString; /** * A utility class to help with conversions of geo json geometry instances with positions to atlas * geometry. * * @author Yazad Khambata */ public final class Positions { public static Polygon toAtlasPolygonFromMultiLineString(final MultiLineString multiLineString) { final List atlasPolygons = toListOfAtlasPolygonsFromMultiLineString( multiLineString, 1); Validate.isTrue(atlasPolygons.size() == 1); return atlasPolygons.get(0); } public static List> toCollectionsOfLocations( final List> collectionOfPositions) { return collectionOfPositions.stream().map(Positions::toLocations) .collect(Collectors.toList()); } public static List toListOfAtlasPolygonsFromMultiLineString( final MultiLineString multiLineString, final int expectedSize) { Validate.notNull(multiLineString); Validate.notNull(multiLineString.getCoordinates()); Validate.notEmpty(multiLineString.getCoordinates().getValue()); if (expectedSize > 0) { Validate.isTrue(expectedSize == multiLineString.getCoordinates().getValue().size()); } return multiLineString.getCoordinates().getValue().stream() .map(positions -> new Polygon(Positions.toLocations(positions))) .collect(Collectors.toList()); } public static List toListOfAtlasPolygonsFromMultiLineString( final MultiLineString multiLineString) { return toListOfAtlasPolygonsFromMultiLineString(multiLineString, -1); } /** * The order of longitude and latitude in GeoJson as per the RFC is [lon, lat, alt]. *

* However the order longitude and latitude is not shared in various atlas constructors, hence * the flip in the order. * * @param position * - the {@link Position} to convert to {@link Location}. * @return - the {@link Location} represented by the {@link Position}. */ public static Location toLocation(final Position position) { return new Location(Latitude.degrees(position.getCoordinate2()), Longitude.degrees(position.getCoordinate1())); } public static List toLocations(final List positions) { return positions.stream().map(Positions::toLocation).collect(Collectors.toList()); } private Positions() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/properties/Properties.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.properties; import java.util.Collections; import java.util.Map; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.DefaultForeignFieldsImpl; import org.openstreetmap.atlas.geography.geojson.parser.domain.foreign.ForeignFields; import org.openstreetmap.atlas.geography.geojson.parser.mapper.Mapper; import org.openstreetmap.atlas.geography.geojson.parser.mapper.impl.DefaultBeanUtilsBasedMapperImpl; /** * @author Yazad Khambata */ public class Properties implements ForeignFields { private ForeignFields values; public Properties(final Map valuesAsMap) { this.values = new DefaultForeignFieldsImpl(valuesAsMap); } @Override public Map asMap() { final Map foreignMap = this.values.asMap(); if (foreignMap == null) { return Collections.EMPTY_MAP; } return Collections.unmodifiableMap(foreignMap); } public T asType(final Class type, final Mapper mapper) { return mapper.map(this.asMap(), type); } public T asType(final Class type) { return asType(type, DefaultBeanUtilsBasedMapperImpl.instance); } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } @Override public Object get(final String key) { return this.values.get(key); } @Override public T get(final String key, final Class valueClass) { return this.values.get(key, valueClass); } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/properties/ext/change/Description.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.properties.ext.change; import java.io.Serializable; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.openstreetmap.atlas.geography.geojson.parser.domain.annotation.Foreign; /** * @author Yazad Khambata */ @Foreign public class Description implements Serializable { private String type; private Descriptor[] descriptors; public Description() { } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } public Descriptor[] getDescriptors() { return this.descriptors; } public String getType() { return this.type; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } public void setDescriptors(final Descriptor[] descriptors) { this.descriptors = descriptors; } public void setType(final String type) { this.type = type; } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/properties/ext/change/Descriptor.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.properties.ext.change; import java.io.Serializable; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.openstreetmap.atlas.geography.geojson.parser.domain.annotation.Foreign; /** * The Descriptor is a flattened version of subclasses of * {@link org.openstreetmap.atlas.geography.atlas.change.description.descriptors.ChangeDescriptor}. * * @author Yazad Khambata */ @Foreign public class Descriptor implements Serializable { private Long id; private String name; private String type; private String itemType; private String role; private String key; private String value; private String originalValue; private String position; private String beforeView; private String afterView; private Long beforeElement; private Long afterElement; public Descriptor() { } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } public Long getAfterElement() { return this.afterElement; } public String getAfterView() { return this.afterView; } public Long getBeforeElement() { return this.beforeElement; } public String getBeforeView() { return this.beforeView; } public Long getId() { return this.id; } public String getItemType() { return this.itemType; } public String getKey() { return this.key; } public String getName() { return this.name; } public String getOriginalValue() { return this.originalValue; } public String getPosition() { return this.position; } public String getRole() { return this.role; } public String getType() { return this.type; } public String getValue() { return this.value; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } public void setAfterElement(final Long afterElement) { this.afterElement = afterElement; } public void setAfterView(final String afterView) { this.afterView = afterView; } public void setBeforeElement(final Long beforeElement) { this.beforeElement = beforeElement; } public void setBeforeView(final String beforeView) { this.beforeView = beforeView; } public void setId(final Long id) { this.id = id; } public void setItemType(final String itemType) { this.itemType = itemType; } public void setKey(final String key) { this.key = key; } public void setName(final String name) { this.name = name; } public void setOriginalValue(final String originalValue) { this.originalValue = originalValue; } public void setPosition(final String position) { this.position = position; } public void setRole(final String role) { this.role = role; } public void setType(final String type) { this.type = type; } public void setValue(final String value) { this.value = value; } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/domain/properties/ext/change/FeatureChangeProperties.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.domain.properties.ext.change; import java.io.Serializable; import java.util.Map; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; import org.openstreetmap.atlas.geography.geojson.parser.domain.annotation.Foreign; /** * @author Yazad Khambata */ @Foreign public class FeatureChangeProperties implements Serializable { private String featureChangeType; private Map metadata; private Description description; private String entityType; private String completeEntityClass; private Long identifier; private Map tags; private Long[] relations; private Long startNode; private Long endNode; private String WKT; private String bboxWKT; public FeatureChangeProperties() { } @Override public boolean equals(final Object that) { return EqualsBuilder.reflectionEquals(this, that); } public String getBboxWKT() { return this.bboxWKT; } public String getCompleteEntityClass() { return this.completeEntityClass; } public Description getDescription() { return this.description; } public Long getEndNode() { return this.endNode; } public String getEntityType() { return this.entityType; } public String getFeatureChangeType() { return this.featureChangeType; } public Long getIdentifier() { return this.identifier; } public Map getMetadata() { return this.metadata; } public Long[] getRelations() { return this.relations; } public Long getStartNode() { return this.startNode; } public Map getTags() { return this.tags; } public String getWKT() { return this.WKT; } @Override public int hashCode() { return HashCodeBuilder.reflectionHashCode(this); } public void setBboxWKT(final String bboxWKT) { this.bboxWKT = bboxWKT; } public void setCompleteEntityClass(final String completeEntityClass) { this.completeEntityClass = completeEntityClass; } public void setDescription(final Description description) { this.description = description; } public void setEndNode(final Long endNode) { this.endNode = endNode; } public void setEntityType(final String entityType) { this.entityType = entityType; } public void setFeatureChangeType(final String featureChangeType) { this.featureChangeType = featureChangeType; } public void setIdentifier(final Long identifier) { this.identifier = identifier; } public void setMetadata(final Map metadata) { this.metadata = metadata; } public void setRelations(final Long[] relations) { this.relations = relations; } public void setStartNode(final Long startNode) { this.startNode = startNode; } public void setTags(final Map tags) { this.tags = tags; } public void setWKT(final String WKT) { this.WKT = WKT; } @Override public String toString() { return ToStringBuilder.reflectionToString(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/impl/jackson/GeoJsonParserJacksonImpl.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.impl.jackson; import java.util.Map; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.geojson.parser.GeoJsonParser; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.GeoJsonItem; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.Type; import org.openstreetmap.atlas.geography.geojson.parser.domain.base.type.TypeUtil; import org.openstreetmap.atlas.geography.geojson.parser.mapper.Mapper; import org.openstreetmap.atlas.geography.geojson.parser.mapper.impl.DefaultBeanUtilsBasedMapperImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; /** * @author Yazad Khambata * @author seancoulter */ public enum GeoJsonParserJacksonImpl implements GeoJsonParser { INSTANCE; private static final Logger log = LoggerFactory.getLogger(GeoJsonParserJacksonImpl.class); @Override public GeoJsonItem deserialize(final String geoJson) { log.trace("geoJson:: {}.", geoJson); final Map map = toMap(geoJson); return deserialize(map); } @Override public GeoJsonItem deserialize(final Map map) { log.trace("map:: {}.", map); final Type type = TypeUtil.identifyStandardType(getType(map)); return type.construct(GeoJsonParserJacksonImpl.INSTANCE, map); } @Override public T deserializeExtension(final String json, final Class targetClass) { final Map map = toMap(json); return deserializeExtension(map, targetClass); } @Override public T deserializeExtension(final Map map, final Class targetClass) { final Mapper mapper = DefaultBeanUtilsBasedMapperImpl.instance; return mapper.map(map, targetClass); } private String getType(final Map map) { final Object type = map.get("type"); Validate.isTrue(type instanceof String, "type: %s.", type); return (String) type; } private Map toMap(final String geoJson) { try { final ObjectMapper mapper = new ObjectMapper(); return (Map) mapper.readValue(geoJson, Object.class); } catch (final JsonProcessingException exception1) { throw new CoreException(exception1.getMessage()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/mapper/Mapper.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.mapper; import java.io.Serializable; import java.util.Map; /** * @author Yazad Khambata */ public interface Mapper extends Serializable { T map(Map map, Class targetClass); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/geojson/parser/mapper/impl/DefaultBeanUtilsBasedMapperImpl.java ================================================ package org.openstreetmap.atlas.geography.geojson.parser.mapper.impl; import java.beans.PropertyDescriptor; import java.lang.reflect.Array; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.beanutils.BeanUtilsBean; import org.apache.commons.lang3.Validate; import org.openstreetmap.atlas.geography.geojson.parser.mapper.Mapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Yazad Khambata */ public enum DefaultBeanUtilsBasedMapperImpl implements Mapper { instance; private static final Logger log = LoggerFactory .getLogger(DefaultBeanUtilsBasedMapperImpl.class); private static final Set> scalarTypes = new HashSet<>( Arrays.asList(String.class, Integer.class, Long.class, Float.class, Double.class, Short.class, Boolean.class, Byte.class)); @Override public T map(final Map map, final Class targetClass) { final T bean = create(targetClass); populate(map, bean); return bean; } private void copyProperty(final BeanUtilsBean beanUtilsBean, final T bean, final String name, final Object value) { try { beanUtilsBean.copyProperty(bean, name, value); } catch (final Exception e) { throw new IllegalStateException( "Failed to copy " + value + " in " + bean.getClass() + "#" + name + ".", e); } } private T create(final Class targetClass) { Validate.notNull(targetClass, "null class cannot be instantiated."); try { return targetClass.getConstructor().newInstance(); } catch (final ReflectiveOperationException reflectiveOperationException) { throw new IllegalStateException("Failed to construct instance of class: " + targetClass + "; isArray: " + targetClass.isArray(), reflectiveOperationException); } } private boolean isScalarType(final Class clazz) { return scalarTypes.contains(clazz) || clazz.isPrimitive(); } private void populate(final Map map, final T bean) { try { Validate.notNull(map, "input map is NULL."); Validate.notNull(bean, "bean is NULL"); final BeanUtilsBean beanUtilsBean = new BeanUtilsBean(); final PropertyDescriptor[] propertyDescriptors = beanUtilsBean.getPropertyUtils() .getPropertyDescriptors(bean); // Start with the concrete object for (final PropertyDescriptor propertyDescriptor : propertyDescriptors) { try { final String name = propertyDescriptor.getName(); final Class propertyType = propertyDescriptor.getPropertyType(); final Object value = map.get(name); if (value == null) { continue; } if (isScalarType(propertyType) || Map.class.isAssignableFrom(propertyType)) { // Scalar types or value is a Map in concrete class. // Map values can be scalar or nested maps of scalars. copyProperty(beanUtilsBean, bean, name, value); } else if (!propertyType.isArray()) { // User-defined concrete classes. final T child = (T) create(propertyType); populate((Map) value, child); copyProperty(beanUtilsBean, bean, name, child); } else { // Array case. final List values = (List) value; if (values == null || values.isEmpty() || values.get(0) == null) { continue; } if (isScalarType(values.get(0).getClass())) { copyProperty(beanUtilsBean, bean, name, values.toArray()); } else { log.info("values: {}.", values); final Class componentType = propertyType.getComponentType(); final Object valuesAsObjects = values.stream().map(item -> { Validate.notNull(item, "item is NULL, do you have a trailing comma in the JSON?"); final T child = (T) create(componentType); populate((Map) item, child); return child; }).toArray(Propersize -> (Object[]) Array.newInstance(componentType, values.size())); copyProperty(beanUtilsBean, bean, name, valuesAsObjects); } } } catch (final Exception e) { throw new IllegalStateException("Population failed. propertyDescriptor name: " + propertyDescriptor.getName() + "; map: " + map + "; bean: " + bean + ".", e); } } } catch (final Exception e) { throw new IllegalStateException( "Population failed. map: " + map + "; bean: " + bean + ".", e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/index/JtsSpatialIndex.java ================================================ package org.openstreetmap.atlas.geography.index; import java.io.Serializable; import java.util.List; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; /** * Define some common methods for spatial indices * * @param * The type to be indexed * @author tony */ public interface JtsSpatialIndex extends Located, Serializable { /** * Inserts an spatial item with an extent specified by the given {@link Rectangle} to the index * * @param bound * The bound the object belongs to * @param item * The item to insert */ void add(Rectangle bound, T item); /** * Queries the index for all items whose bounds intersect the given {@link Rectangle} * * @param bound * The bound to query * @return An {@link List} of features within or intersecting the bound. */ List get(Rectangle bound); /** * Queries the index for all items whose bounds intersect the given {@link Rectangle} and match * the given predicate * * @param bound * The bound to query * @param predicate * a predicate to apply to each item to determine if it should be included * @return An {@link List} of features within or intersecting the bound. */ List get(Rectangle bound, Predicate predicate); /** * Removes a single item from the tree. * * @param bound * The bound the object belongs to * @param item * The item to remove * @return true if the item was found */ boolean remove(Rectangle bound, T item); /** * @return the number of items in the index */ int size(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/index/PackedSpatialIndex.java ================================================ package org.openstreetmap.atlas.geography.index; import java.util.ArrayList; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.atlas.items.Edge; /** * This {@link PackedSpatialIndex} accepts a {@link Located} object (e.g. an {@link Edge}), but only * stores the packed information (identifier) into the real index. Spatial index can be an R-Tree or * Quad-Tree. * * @param * The type of {@link Located} item * @param * The type of packed item * @author tony */ public abstract class PackedSpatialIndex implements SpatialIndex { private static final long serialVersionUID = 1747435801359663115L; private final JtsSpatialIndex index; public PackedSpatialIndex(final JtsSpatialIndex index) { this.index = index; } @Override public void add(final L located) { final Rectangle bounds = located.bounds(); if (bounds != null) { this.index.add(bounds, compress(located)); } else { throw new CoreException( "Unable to get bounds for located item when building spatial index: {}", located); } } @Override public Rectangle bounds() { return this.index.bounds(); } @Override public Iterable get(final Rectangle bound) { return ((ArrayList) this.index.get(bound)).stream().map(this::restore) .collect(Collectors.toList()); } @Override public Iterable get(final Rectangle bound, final Predicate predicate) { return ((ArrayList) this.index.get(bound)).stream().map(this::restore) .filter(predicate).collect(Collectors.toList()); } /** * Extract a packed object from the given located object * * @param located * The {@link Located} item to extract * @return The packed object */ protected abstract Packed compress(L located); protected abstract boolean isValid(L located, Rectangle bounds); /** * Restore the located object from packed one * * @param packed * The packed object to restore * @return The restored object */ protected abstract L restore(Packed packed); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/index/QuadTree.java ================================================ package org.openstreetmap.atlas.geography.index; import java.util.List; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.locationtech.jts.index.quadtree.Quadtree; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; /** * A JTS Quadtree wrapper. *

* Quadtree is a spatial index structure for efficient range querying of items bounded by 2D * rectangles. This Quadtree index provides a primary filter for range rectangle queries. The * various query methods return a list of all items which may intersect the query rectangle. *

* Note that it may thus return items which do not in fact intersect the query rectangle. *

* A secondary filter is required to test for actual intersection between the query rectangle and * the envelope of each candidate item. The secondary filter may be performed explicitly, or it may * be provided implicitly by subsequent operations executed on the items (for instance, if the index * query is followed by computing a spatial predicate between the query geometry and tree items, the * envelope intersection check is performed automatically. * * @param * The type to be indexed * @author tony */ public class QuadTree implements JtsSpatialIndex { private static final long serialVersionUID = 7515245245282264428L; private final Quadtree tree = new Quadtree(); private Rectangle bound; public static QuadTree forCollection(final Iterable iterable, final Function transform) { final QuadTree toReturn = new QuadTree<>(); iterable.forEach(item -> toReturn.add(transform.apply(item), item)); return toReturn; } public static QuadTree forLocated(final Iterable locatedIterable) { final QuadTree toReturn = new QuadTree<>(); locatedIterable.forEach(located -> toReturn.add(located.bounds(), located)); return toReturn; } @Override public void add(final Rectangle bound, final T item) { this.tree.insert(bound.asEnvelope(), item); if (this.bound != null) { this.bound = this.bound.combine(bound); } else { this.bound = bound; } } @Override public Rectangle bounds() { return this.bound; } public int depth() { return this.tree.depth(); } @SuppressWarnings("unchecked") @Override public List get(final Rectangle bound) { return this.tree.query(bound.asEnvelope()); } @Override public List get(final Rectangle bound, final Predicate predicate) { return get(bound).stream().filter(predicate).collect(Collectors.toList()); } public boolean isEmpty() { return this.tree.isEmpty(); } @Override /** * Note that the remove operation won't adjust this.bound accordingly */ public boolean remove(final Rectangle bound, final T item) { return this.tree.remove(bound.asEnvelope(), item); } @Override public int size() { return this.tree.size(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/index/RTree.java ================================================ package org.openstreetmap.atlas.geography.index; import java.util.List; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.locationtech.jts.index.strtree.STRtree; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; /** * A wrapper of JTS STRtree. *

* A query-only R-tree created using the Sort-Tile-Recursive (STR) algorithm. For two-dimensional * spatial data. The STR packed R-tree is simple to implement and maximizes space utilization; that * is, as many leaves as possible are filled to capacity. Overlap between nodes is far less than in * a basic R-tree. However, once the tree has been built (explicitly or on the first call to * #query), items may not be added or removed. *

* * @param * The type to be indexed * @author tony */ public class RTree implements JtsSpatialIndex { private static final long serialVersionUID = -3672714932238885163L; private final STRtree tree; private Rectangle bound; public static RTree forCollection(final Iterable iterable, final Function transform) { final RTree toReturn = new RTree<>(); iterable.forEach(item -> toReturn.add(transform.apply(item), item)); return toReturn; } public static RTree forLocated(final Iterable locatedIterable) { final RTree toReturn = new RTree<>(); locatedIterable.forEach(located -> toReturn.add(located.bounds(), located)); return toReturn; } public RTree() { this.tree = new STRtree(); } public RTree(final int nodeCapacity) { this.tree = new STRtree(nodeCapacity); } @Override public void add(final Rectangle bound, final T item) { this.tree.insert(bound.asEnvelope(), item); if (this.bound != null) { this.bound = this.bound.combine(bound); } else { this.bound = bound; } } @Override public Rectangle bounds() { return this.bound; } public void build() { this.tree.build(); } public int depth() { return this.tree.depth(); } @SuppressWarnings("unchecked") @Override public List get(final Rectangle bound) { return this.tree.query(bound.asEnvelope()); } @Override public List get(final Rectangle bound, final Predicate predicate) { return get(bound).stream().filter(predicate).collect(Collectors.toList()); } public boolean isEmpty() { return this.tree.isEmpty(); } @SuppressWarnings("unchecked") public List itemsTree() { return this.tree.itemsTree(); } @Override /** * Note that the remove operation won't adjust this.bound accordingly */ public boolean remove(final Rectangle bound, final T item) { return this.tree.remove(bound.asEnvelope(), item); } @Override public int size() { return this.tree.size(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/index/SpatialIndex.java ================================================ package org.openstreetmap.atlas.geography.index; import java.io.Serializable; import java.util.function.Predicate; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; /** * @author matthieun * @param * The type to be indexed */ public interface SpatialIndex extends Located, Serializable { /** * Populate the spatial index * * @param feature * The feature to add */ void add(T feature); /** * Get an {@link Iterable} over the features that are within or intersecting some bounds. * * @param bounds * The bounds to query * @return An {@link Iterable} of features within or intersecting the bounds. */ Iterable get(Rectangle bounds); /** * Queries the index for all items whose bounds intersect the given {@link Rectangle} and match * the given predicate * * @param bound * The bound to query * @param predicate * a predicate to apply to each item to determine if it should be included * @return An {@link Iterable} of features within or intersecting the bound. */ Iterable get(Rectangle bound, Predicate predicate); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/matching/PolyLineMatch.java ================================================ package org.openstreetmap.atlas.geography.matching; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Segment; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Duration; import org.openstreetmap.atlas.utilities.threads.Pool; import org.openstreetmap.atlas.utilities.threads.Result; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Best match between a {@link PolyLine} and some other set of {@link PolyLine}s. * * @author matthieun */ public class PolyLineMatch { private static final Logger logger = LoggerFactory.getLogger(PolyLineMatch.class); private final PolyLine source; private final List candidates; /** * Constructor * * @param source * The source {@link PolyLine} to try to match. * @param candidates * The candidate {@link PolyLine}s that will provide the segments for the match */ public PolyLineMatch(final PolyLine source, final List candidates) { this.source = source; this.candidates = candidates; } /** * Find the best set of {@link Segment}s that form a {@link PolyLineRoute} that matches the * source {@link PolyLine} with a costDistance that is less than the threshold provided. * * @param threshold * The threshold provided * @return The best re-constructed route. Empty if none. */ public Optional match(final Distance threshold) { if (this.candidates.isEmpty()) { return Optional.empty(); } PolyLineRoute best = null; final SortedSet candidateRoutes = new TreeSet<>(); final Set visitedStitchingLocations = new HashSet<>(); int priorNumberOfCandidateRoutes = -1; boolean candidateRoutesEmpty = candidateRoutes.isEmpty(); boolean bestNonNull = best != null; boolean bestCostTooHigh = false; boolean candidateRoutesIncreased = false; while (candidateRoutesEmpty || bestNonNull && bestCostTooHigh && candidateRoutesIncreased) { priorNumberOfCandidateRoutes = candidateRoutes.size(); final Set toAdd = new HashSet<>(); for (int polyLineIndex = 0; polyLineIndex < this.candidates.size(); polyLineIndex++) { final PolyLine candidate = this.candidates.get(polyLineIndex); final List segments = candidate.segments(); for (int segmentIndex = 0; segmentIndex < segments.size(); segmentIndex++) { if (candidateRoutes.isEmpty()) { toAdd.add(PolyLineRoute.startFrom(this.source, this.candidates, polyLineIndex, segmentIndex)); } else { for (final PolyLineRoute existing : candidateRoutes) { final Optional stitchingLocation = existing .canAppend(polyLineIndex, segmentIndex); if (stitchingLocation.isPresent() && !visitedStitchingLocations.contains(stitchingLocation.get())) { visitedStitchingLocations.add(stitchingLocation.get()); final PolyLineRoute elected = existing.copyAndAppend(polyLineIndex, segmentIndex); if (!candidateRoutes.contains(elected)) { toAdd.add(elected); } } } } } } candidateRoutes.addAll(toAdd); best = candidateRoutes.first(); candidateRoutesEmpty = candidateRoutes.isEmpty(); bestNonNull = best != null; bestCostTooHigh = best.getCost().isGreaterThan(threshold); candidateRoutesIncreased = candidateRoutes.size() > priorNumberOfCandidateRoutes; } return Optional.ofNullable(best); } /** * Find the best set of {@link Segment}s that form a {@link PolyLineRoute} that matches the * source {@link PolyLine} with a costDistance that is less than the threshold provided. * * @param threshold * The threshold provided * @param maximum * The maximum {@link Duration} of the computation. * @return The best re-constructed route. Empty if none, or if the computation took longer than * the maximum {@link Duration} */ public Optional match(final Distance threshold, final Duration maximum) { try (Pool polyLineMatchPool = new Pool(1, "PolyLine Match")) { final Result> result = polyLineMatchPool .queue(() -> match(threshold)); return result.get(maximum); } catch (final TimeoutException e) { logger.warn("Was not able to compute PolyLineMatch in {} for {}", maximum, this.source); return Optional.empty(); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/matching/PolyLineRoute.java ================================================ package org.openstreetmap.atlas.geography.matching; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.Segment; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * A candidate {@link PolyLine} route matching the source {@link PolyLine}. *

* This is a re-constructed {@link PolyLine} from {@link Segment}s coming from a list of candidate * {@link PolyLine}s. The algorithm tries to find the best set of connected {@link Segment}s that * belong to many other {@link PolyLine}s and that matches the source {@link PolyLine} * * @author matthieun */ public final class PolyLineRoute implements Comparable { private final List candidateSegments; private final Distance cost; private final PolyLine source; private final List candidates; private PolyLine polyLine; protected static PolyLineRoute startFrom(final PolyLine source, final List candidates, final int polyLineIndex, final int segmentIndex) { final List candidateSegments = new ArrayList<>(); candidateSegments.add(candidateSegment(candidates, polyLineIndex, segmentIndex)); return new PolyLineRoute(source, candidates, candidateSegments); } private static Segment candidateSegment(final List candidates, final int polyLineIndex, final int segmentIndex) { return new Segment(candidates.get(polyLineIndex).get(segmentIndex), segmentIndex < candidates.get(polyLineIndex).size() - 1 ? candidates.get(polyLineIndex).get(segmentIndex + 1) : candidates.get(polyLineIndex).first()); } private PolyLineRoute(final PolyLine source, final List candidates, final List candidateSegments) { this.candidateSegments = candidateSegments; this.source = source; this.candidates = candidates; this.cost = this.source.averageOneWayDistanceTo(this.asPolyLine()) .add(this.asPolyLine().size() > 2 ? new PolyLine(this.asPolyLine().innerLocations()) .averageOneWayDistanceTo(this.source) : Distance.ZERO); } public PolyLine asPolyLine() { if (this.polyLine == null) { final List locations = new ArrayList<>(); this.candidateSegments.forEach(segment -> locations.add(segment.first())); if (this.source instanceof Polygon) { return new Polygon(locations); } else { locations.add(this.candidateSegments.get(this.candidateSegments.size() - 1).last()); return new PolyLine(locations); } } return this.polyLine; } @Override public int compareTo(final PolyLineRoute other) { return this.cost.isGreaterThan(other.getCost()) ? 1 : this.cost.isLessThan(other.getCost()) ? -1 : 0; } @Override public boolean equals(final Object other) { if (other instanceof PolyLineRoute) { return ((PolyLineRoute) other).asPolyLine().equals(this.asPolyLine()); } return false; } public Distance getCost() { return this.cost; } @Override public int hashCode() { return asPolyLine().hashCode(); } @Override public String toString() { return "[PolyLineRoute: " + asPolyLine() + "]"; } protected Optional canAppend(final int polyLineIndex, final int segmentIndex) { // The segment index and the index of the first point of the segment in the polyline are // the same! final Location end = this.candidateSegments.get(this.candidateSegments.size() - 1).last(); final Location proposed = this.candidates.get(polyLineIndex).get(segmentIndex); final Segment candidateSegment = candidateSegment(this.candidates, polyLineIndex, segmentIndex); final boolean matchingLocations = end.equals(proposed); final boolean candidateSegmentAlreadyContained = this.candidateSegments .contains(candidateSegment); if (matchingLocations && !candidateSegmentAlreadyContained) { return Optional.of(proposed); } else { return Optional.empty(); } } protected PolyLineRoute copyAndAppend(final int polyLineIndex, final int segmentIndex) { if (canAppend(polyLineIndex, segmentIndex).isPresent()) { final List polyLineIndexAndSegmentIndex = new ArrayList<>( this.candidateSegments); polyLineIndexAndSegmentIndex .add(candidateSegment(this.candidates, polyLineIndex, segmentIndex)); return new PolyLineRoute(this.source, this.candidates, polyLineIndexAndSegmentIndex); } else { throw new CoreException("Unable to append {} and {}", asPolyLine(), getSegment(polyLineIndex, segmentIndex)); } } private Segment getSegment(final int polyLineIndex, final int segmentIndex) { return this.candidates.get(polyLineIndex).segments().get(segmentIndex); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/CountryShard.java ================================================ package org.openstreetmap.atlas.geography.sharding; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.sharding.converters.StringToShardConverter; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonObject; /** * Shard and country pair * * @author matthieun */ public class CountryShard implements Shard { private static final long serialVersionUID = -4158215940506552768L; private final Shard shard; private final String country; public static CountryShard forName(final String name) { final StringList split = StringList.split(name, Shard.SHARD_DATA_SEPARATOR, 2); return new CountryShard(split.get(0), new StringToShardConverter().convert(split.get(1))); } public CountryShard(final String country, final Shard shard) { if (shard == null || country == null) { throw new CoreException("Cannot have null parameters: Country = {} and Shard = {}", country, shard); } this.shard = shard; this.country = country; } public CountryShard(final String country, final String shardString) { if (shardString == null || country == null) { throw new CoreException("Cannot have null parameters: Country = {} and Shard = {}", country, shardString); } this.country = country; this.shard = new StringToShardConverter().convert(shardString); } @Override public JsonObject asGeoJson() { return this.shard.asGeoJson(); } @Override public Rectangle bounds() { return this.shard.bounds(); } @Override public boolean equals(final Object other) { if (other instanceof CountryShard) { final CountryShard that = (CountryShard) other; return this.getCountry().equals(that.getCountry()) && this.getShard().equals(that.getShard()); } return false; } public String getCountry() { return this.country; } @Override public GeoJsonType getGeoJsonType() { return this.shard.getGeoJsonType(); } @Override public String getName() { return this.country + Shard.SHARD_DATA_SEPARATOR + this.shard.getName(); } public Shard getShard() { return this.shard; } @Override public int hashCode() { return new HashCodeBuilder().append(this.shard).append(this.country).hashCode(); } @Override public String toString() { return "[CountryShard: country = " + this.country + ", shard = " + this.shard.toString() + "]"; } @Override public byte[] toWkb() { return this.shard.toWkb(); } @Override public String toWkt() { return this.shard.toWkt(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/DynamicTileSharding.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.io.BufferedWriter; import java.io.Serializable; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder; import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties; import org.openstreetmap.atlas.geography.geojson.GeoJsonObject; import org.openstreetmap.atlas.geography.sharding.preparation.TilePrinter; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.streaming.writers.JsonWriter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Quad Tree sharding. The name does not reflect QuadTree to not confuse with the quad tree spatial * index. *

* The {@link Command} portion of this class is to read a csv file containing feature counts * (usually ways) for each of the maxZoom - 1 tiles. This is generated in the OSM database snapshot, * using the {@link TilePrinter} class to create the list of tiles needed. The csv file that is read * helps generate a tree, which is then serialized. * * @author matthieun * @author mgostintsev */ public class DynamicTileSharding extends Command implements Sharding { /** * Node of the quad tree. Implementation is recursive. * * @author matthieun */ private static class Node implements Located, Serializable { private static final int MAXIMUM_CHILDREN = 4; private static final long serialVersionUID = -7789058745501080439L; private final List children; private final SlippyTile tile; protected static Node read(final Resource resource) { return read(resource.lines().iterator()); } private static Node read(final Iterator lineIterator) { final String line = lineIterator.next(); final List children = new ArrayList<>(); String tileName = line; if (line.endsWith("+")) { for (int i = 0; i < MAXIMUM_CHILDREN; i++) { children.add(read(lineIterator)); } tileName = line.substring(0, line.length() - 1); } return new Node(SlippyTile.forName(tileName), children); } protected Node() { this(SlippyTile.ROOT); } private Node(final SlippyTile tile) { this.tile = tile; this.children = new ArrayList<>(); } private Node(final SlippyTile tile, final List children) { this.tile = tile; this.children = children; } @Override public Rectangle bounds() { return this.tile.bounds(); } @Override public boolean equals(final Object other) { if (other instanceof Node) { return ((Node) other).getTile().equals(this.tile); } return false; } public SlippyTile getTile() { return this.tile; } @Override public int hashCode() { return this.tile.hashCode(); } public Set leafNodes(final GeometricSurface surface) { final Set result = new HashSet<>(); final Rectangle polygonBounds = surface.bounds(); if (polygonBounds.overlaps(bounds())) { if (isFinal() && surface.overlaps(bounds())) { result.add(this); } else { for (final Node child : this.children) { result.addAll(child.leafNodes(surface)); } } } return result; } public Set leafNodesCovering(final Location location) { final Set result = new HashSet<>(); if (bounds().fullyGeometricallyEncloses(location)) { if (isFinal()) { result.add(this); } else { for (final Node child : this.children) { result.addAll(child.leafNodesCovering(location)); } } } return result; } public Set leafNodesIntersecting(final PolyLine polyLine) { final Rectangle polyLineBounds = polyLine.bounds(); return leafNodesIntersecting(polyLine, polyLineBounds); } public Set leafNodesIntersecting(final PolyLine polyLine, final Rectangle polyLineBounds) { final Set result = new HashSet<>(); if (polyLineBounds.overlaps(bounds())) { if (isFinal() && (polyLine.intersects(bounds()) || bounds().fullyGeometricallyEncloses(polyLine))) { result.add(this); } else { for (final Node child : this.children) { result.addAll(child.leafNodesIntersecting(polyLine, polyLineBounds)); } } } return result; } public LocationIterableProperties toGeoJsonBuildingBlock() { final Map tags = new HashMap<>(); tags.put("tile", this.name()); return new GeoJsonBuilder.LocationIterableProperties(bounds(), tags); } protected void build(final Predicate shouldSplit) { if (this.zoom() < SlippyTile.MAX_ZOOM && shouldSplit.test(this.tile)) { this.split(); for (final Node child : this.children) { child.build(shouldSplit); } } } protected boolean isFinal() { return this.children.isEmpty(); } protected String name() { return this.tile.getName(); } protected Set neighbors(final SlippyTile targetTile) { final Set neighboringNodes = new HashSet<>(); for (final Node leafNode : this.leafNodes(targetTile.bounds())) { final Rectangle expandedBoundary = leafNode.bounds() .expand(SlippyTile.calculateExpansionDistance(leafNode.bounds())); if (targetTile.bounds().overlaps(expandedBoundary) && !leafNode.bounds().equals(targetTile.bounds())) { neighboringNodes.add(leafNode); } } return neighboringNodes; } protected void save(final WritableResource resource) { final BufferedWriter writer = resource.writer(); try { this.save(writer); } catch (final Exception e) { Streams.close(writer); throw e; } Streams.close(writer); } protected void split() { this.children.addAll(this.tile.split(this.zoom() + 1).stream().map(Node::new) .collect(Collectors.toList())); } protected int zoom() { return this.tile.getZoom(); } /** * Does a deep equals with the other node * * @param other * other Node * @return true if entire structure is equal, false if not */ private boolean deepEquals(final Node other) { final Comparator nodeCompare = Comparator.comparing(Node::getTile); // BFS through both trees to get equality final Queue queue = new LinkedList<>(); queue.offer(this); queue.offer(other); while (!queue.isEmpty()) { // We always offer two at a time, so we can poll two at a time. final Node node1 = queue.poll(); final Node node2 = queue.poll(); if (node1.equals(node2) && node1.getChildren().size() == node2.getChildren().size()) { final List children1 = node1.getChildren(); final List children2 = node2.getChildren(); children1.sort(nodeCompare); children2.sort(nodeCompare); for (int index = 0; index < children1.size(); index++) { queue.offer(children1.get(index)); queue.offer(children2.get(index)); } } else { return false; } } return true; } private List getChildren() { return this.children; } private void save(final BufferedWriter writer) { try { writer.write(this.tile.getName()); if (!isFinal()) { writer.write("+"); } writer.write("\n"); } catch (final Exception e) { throw new CoreException("Unable to write slippy tile {}", this.tile, e); } for (final Node child : this.children) { child.save(writer); } } } public static final Switch DEFINITION = new Switch<>("definition", "Resource containing the maxZoom - 1 tile to feature count mapping.", File::new, Optionality.REQUIRED); public static final Switch GEOJSON = new Switch<>("geoJson", "The resource where to save the geojson tree for debugging", File::new, Optionality.OPTIONAL); public static final Switch MAXIMUM_COUNT = new Switch<>("maxCount", "The maximum feature count. Any cell with a larger feature count will be split, up to maxZoom", Integer::valueOf, Optionality.OPTIONAL, "200000"); public static final Switch MAXIMUM_ZOOM = new Switch<>("maxZoom", "The maximum zoom", Integer::valueOf, Optionality.OPTIONAL, "10"); public static final Switch MINIMUM_ZOOM = new Switch<>("minZoom", "The minimum zoom", Integer::valueOf, Optionality.OPTIONAL, "5"); public static final Switch OUTPUT = new Switch<>("output", "The resource where to save the serialized tree.", File::new, Optionality.REQUIRED); private static final int MINIMUM_TO_SPLIT = 1_000; private static final int READER_REPORT_FREQUENCY = 10_000_000; private static final Logger logger = LoggerFactory.getLogger(DynamicTileSharding.class); private static final long serialVersionUID = 229952569300405488L; // The root of the tree for this dynamic sharding private final Node root; private final String resourceName; public static void main(final String[] args) { new DynamicTileSharding().run(args); } /** * Construct. * * @param resource * The resource containing the serialized tree definition. */ public DynamicTileSharding(final Resource resource) { this.root = Node.read(resource); this.resourceName = resource.getName(); } /** * Construct, with a root that covers the whole world. */ private DynamicTileSharding() { this.root = new Node(); this.resourceName = "N/A"; } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || getClass() != other.getClass()) { return false; } final DynamicTileSharding that = (DynamicTileSharding) other; return this.root.deepEquals(that.root); } @Override public String getName() { return "dynamic@" + this.resourceName; } @Override public int hashCode() { return Objects.hash(this.root); } @Override public Iterable neighbors(final Shard shard) { return this.root.neighbors(SlippyTile.forName(shard.getName())).stream().map(Node::getTile) .collect(Collectors.toList()); } /** * Save the tree to a {@link WritableResource} * * @param resource * The {@link WritableResource} to serialize the tree definition to. */ public void save(final WritableResource resource) { this.root.save(resource); } public void saveAsGeoJson(final WritableResource resource) { final JsonWriter writer = new JsonWriter(resource); final GeoJsonObject geoJson = new GeoJsonBuilder().create(Iterables .translate(this.root.leafNodes(Rectangle.MAXIMUM), Node::toGeoJsonBuildingBlock)); writer.write(geoJson.jsonObject()); writer.close(); } @Override public Shard shardForName(final String name) { final SlippyTile result = SlippyTile.forName(name); if (!this.root.leafNodesCovering(result.bounds().center()).contains(new Node(result))) { throw new CoreException("This tree does not contain tile {}", name); } return result; } @Override public Iterable shards(final GeometricSurface surface) { return Iterables.stream(this.root.leafNodes(surface)).map(Node::getTile); } @Override public Iterable shardsCovering(final Location location) { return Iterables.stream(this.root.leafNodesCovering(location)).map(Node::getTile); } @Override public Iterable shardsIntersecting(final PolyLine polyLine) { return Iterables.stream(this.root.leafNodesIntersecting(polyLine)).map(Node::getTile); } /** * Calculates and saves the counts for each zoom layer lower than firstZoomLayerToGenerate. * * @param firstZoomLayerToGenerate * the first zoom layer for which to generate counts * @param counts * Map containing counts for all {@link SlippyTile}s in firstZoomLayerToGenerate+1 * @return a Map containing counts for all (@link SlippyTile}s in zoomLayerToGenerate and below */ protected Map calculateTileCountsForAllZoom( final int firstZoomLayerToGenerate, final Map counts) { for (int currentZoom = firstZoomLayerToGenerate; currentZoom >= 0; currentZoom--) { long count = 0; long tilesCalculated = 0; for (int x = 0; x < Math.pow(2, currentZoom + 1.0); x += 2) { for (int y = 0; y < Math.pow(2, currentZoom + 1.0); y += 2) { count = 0; // top left count += counts.getOrDefault(new SlippyTile(x, y, currentZoom + 1), (long) 0); // top right count += counts.getOrDefault(new SlippyTile(x + 1, y, currentZoom + 1), (long) 0); // bottom left count += counts.getOrDefault(new SlippyTile(x, y + 1, currentZoom + 1), (long) 0); // bottom right count += counts.getOrDefault(new SlippyTile(x + 1, y + 1, currentZoom + 1), (long) 0); if (count != 0) { counts.put(new SlippyTile(x / 2, y / 2, currentZoom), count); } if (++tilesCalculated % READER_REPORT_FREQUENCY == 0) { logger.info("Calculated {} zoom level {} tiles.", tilesCalculated, currentZoom); } } } } return counts; } @Override protected int onRun(final CommandMap command) { final Resource definition = (Resource) command.get(DEFINITION); final int numberLines = (int) Iterables.size(definition.lines()); logger.info("There are {} tiles.", numberLines); final Map counts = new HashMap<>(numberLines); final WritableResource output = (WritableResource) command.get(OUTPUT); final int maximum = (int) command.get(MAXIMUM_COUNT); final int minimumZoom = (int) command.get(MINIMUM_ZOOM); final int maximumZoom = (int) command.get(MAXIMUM_ZOOM); final WritableResource geoJson = (WritableResource) command.get(GEOJSON); int zoom = 0; int counter = 0; for (final String line : definition.lines()) { final StringList split = StringList.split(line, ","); final SlippyTile tile = SlippyTile.forName(split.get(0)); counts.put(tile, Long.valueOf(split.get(1))); zoom = tile.getZoom(); if (++counter % READER_REPORT_FREQUENCY == 0) { logger.info("Read counts for {} zoom level {} tiles.", counter, zoom); } } // maximumZoom is decremented by 2 because it represents the highest zoom that the sharding // tree will split to. The input CSV (definition) has count information at the // (maximumZoom-1) level, which is already put into the HashMap "counts" by the code above. // Therefore we want to start calculating counts one level below, which is (maximumZoom-2) final Map allCounts = calculateTileCountsForAllZoom(maximumZoom - 2, counts); if (zoom == 0) { throw new CoreException("No tiles in definition"); } final int finalZoom = zoom; if (maximumZoom > finalZoom + 1) { throw new CoreException( "Cannot go over the resolution of the counts definition. " + "MaxZoom = {} has to be at most equal to definition zoom + 1 = {}", maximumZoom, finalZoom); } this.root.build(tile -> { final long count = allCounts.getOrDefault(tile, (long) 0); if (count <= MINIMUM_TO_SPLIT) { return false; } if (tile.getZoom() < minimumZoom) { return true; } if (tile.getZoom() >= maximumZoom) { return false; } return count > maximum; }); this.save(output); final String outputLocation = lastRawCommand(OUTPUT); logger.info("Printed tree to {}. Loading for verification...", outputLocation); new DynamicTileSharding(new File(outputLocation)); logger.info("Successfully loaded tree from {}", outputLocation); if (geoJson != null) { if (logger.isInfoEnabled()) { logger.info("Saving geojson to {}...", lastRawCommand(GEOJSON)); } this.saveAsGeoJson(geoJson); } return 0; } @Override protected SwitchList switches() { return new SwitchList().with(DEFINITION, OUTPUT, MINIMUM_ZOOM, MAXIMUM_ZOOM, MAXIMUM_COUNT, GEOJSON); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/GeoHashSharding.java ================================================ package org.openstreetmap.atlas.geography.sharding; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * @author matthieun */ public class GeoHashSharding implements Sharding { private static final long serialVersionUID = -7355746343440111174L; private final int precision; public GeoHashSharding(final int precision) { GeoHashTile.validatePrecision(precision); this.precision = precision; } @Override public String getName() { return "geohash@" + this.precision; } public int getPrecision() { return this.precision; } @Override public Iterable neighbors(final Shard shard) { if (shard instanceof GeoHashTile) { return ((GeoHashTile) shard).neighbors(); } else { throw new CoreException("Shard parameter was of invalid type {}", shard.getClass().getName()); } } @Override public Shard shardForName(final String name) { if (name.length() != this.precision) { throw new CoreException( "This geohash sharding is of precision {}. \"{}\" is not the correct length.", this.precision, name); } return GeoHashTile.forName(name); } @Override public Iterable shards(final GeometricSurface surface) { return Iterables.stream(GeoHashTile.allTiles(this.precision, surface)) .map(tile -> (Shard) tile); } @Override public Iterable shardsCovering(final Location location) { return Iterables.stream(Iterables.from(GeoHashTile.covering(location, this.precision))) .map(tile -> (Shard) tile); } @Override public Iterable shardsIntersecting(final PolyLine polyLine) { return Iterables.stream(GeoHashTile.allTiles(this.precision, polyLine.bounds())) .filter(tile -> tile.bounds().intersects(polyLine)).map(tile -> (Shard) tile); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/GeoHashTile.java ================================================ package org.openstreetmap.atlas.geography.sharding; import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.io.GeohashUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.sharding.converters.RectangleToSpatial4JRectangleConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.gson.JsonObject; /** * @author matthieun */ public class GeoHashTile implements Shard { public static final int MAXIMUM_PRECISION = 12; public static final GeoHashTile ROOT = new GeoHashTile(""); static final char[] GEOHASH_CHARACTERS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' }; static final BiMap GEOHASH_CHARACTER_MAP = HashBiMap.create(); private static final long serialVersionUID = 525101912087621541L; private static final RectangleToSpatial4JRectangleConverter RECTANGLE_TO_SPATIAL_4_J_RECTANGLE_CONVERTER = new RectangleToSpatial4JRectangleConverter(); static { for (int index = 0; index < GEOHASH_CHARACTERS.length; index++) { GEOHASH_CHARACTER_MAP.put(index, GEOHASH_CHARACTERS[index]); } } private final String value; private Rectangle bounds; public static Iterable allTiles(final int precision) { validatePrecision(precision); if (precision == 0) { return Iterables.iterable(GeoHashTile.ROOT); } return new GeoHashTileIterable(precision); } public static Iterable allTiles(final int precision, final GeometricSurface surface) { validatePrecision(precision); if (precision == 0) { return Iterables.iterable(GeoHashTile.ROOT); } return new GeoHashTileIterable(precision, surface); } public static GeoHashTile covering(final Location location, final int precision) { validatePrecision(precision); return new GeoHashTile(GeohashUtils.encodeLatLon(location.getLatitude().asDegrees(), location.getLongitude().asDegrees(), precision)); } public static GeoHashTile forName(final String value) { return new GeoHashTile(value); } public static long numberTilesAtPrecision(final int precision) { if (precision == 0) { return 1L; } return (long) Math.pow((double) GEOHASH_CHARACTERS.length, (double) precision); } public static void validatePrecision(final int precision) { if (precision > MAXIMUM_PRECISION) { throw new CoreException("Cannot have precision {} > {}", precision, MAXIMUM_PRECISION); } if (precision < 0) { throw new CoreException("Cannot have precision {} < 0", precision); } } public GeoHashTile(final String value) { this.value = value; } @Override public JsonObject asGeoJson() { return bounds().asGeoJson(); } @Override public Rectangle bounds() { if (this.bounds == null) { if (this.value.isEmpty()) { this.bounds = Rectangle.MAXIMUM; } else { this.bounds = RECTANGLE_TO_SPATIAL_4_J_RECTANGLE_CONVERTER .convert(GeohashUtils.decodeBoundary(this.value, SpatialContext.GEO)); } } return this.bounds; } @Override public boolean equals(final Object other) { if (other instanceof GeoHashTile) { return this.value.equals(((GeoHashTile) other).getName()); } return false; } @Override public GeoJsonType getGeoJsonType() { return bounds().getGeoJsonType(); } @Override public String getName() { return this.value; } public int getPrecision() { return this.value.length(); } @Override public int hashCode() { return this.value.hashCode(); } public Iterable neighbors() { return Iterables.stream(GeoHashTile.allTiles(this.getPrecision(), bounds())) .filter(tile -> !this.equals(tile)).map(tile -> (Shard) tile).collect(); } public char[] toCharArray() { return this.value.toCharArray(); } @Override public String toString() { return "[GeoHashTile: value = " + this.value + "]"; } @Override public byte[] toWkb() { return bounds().toWkb(); } @Override public String toWkt() { return bounds().toWkt(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/GeoHashTileIterable.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Rectangle; /** * @author matthieun */ class GeoHashTileIterable implements Iterable { /** * @author matthieun */ private class GeoHashTileIterator implements Iterator { private final char[] current = Arrays.copyOf(GeoHashTileIterable.this.starting, GeoHashTileIterable.this.precision); private boolean done = false; private boolean firstCall = true; private int backIndex = GeoHashTileIterable.this.precision - 1; @Override public boolean hasNext() { return !this.done; } @Override public GeoHashTile next() { if (!hasNext()) { throw new NoSuchElementException(); } GeoHashTile result = new GeoHashTile(String.valueOf(this.current)); if (!isMaximum() && this.firstCall) { while (!this.done && !overlaps(result)) { upgrade(); result = new GeoHashTile(String.valueOf(this.current)); } } upgrade(); while (!isMaximum() && !this.done && !overlaps(new GeoHashTile(String.valueOf(this.current)))) { upgrade(); } this.firstCall = false; return result; } private boolean overlaps(final GeoHashTile tile) { return GeoHashTileIterable.this.surface.overlaps(tile.bounds()); } private void tick(final int index) { final char currentChar = this.current[index]; final int currentCharIndex = GeoHashTile.GEOHASH_CHARACTER_MAP.inverse() .get(currentChar); this.current[index] = GeoHashTile.GEOHASH_CHARACTER_MAP.get(currentCharIndex + 1); } private void upgrade() { final char lastCharacter = GeoHashTile.GEOHASH_CHARACTERS[GeoHashTile.GEOHASH_CHARACTERS.length - 1]; while (this.backIndex >= GeoHashTileIterable.this.prefix.length && this.current[this.backIndex] == lastCharacter) { this.backIndex--; } if (this.backIndex < GeoHashTileIterable.this.prefix.length) { this.done = true; return; } tick(this.backIndex); if (this.backIndex < GeoHashTileIterable.this.precision - 1) { zero(this.backIndex + 1); this.backIndex = GeoHashTileIterable.this.precision - 1; } } private void zero(final int index) { for (int subIndex = index; subIndex < GeoHashTileIterable.this.precision; subIndex++) { this.current[subIndex] = GeoHashTile.GEOHASH_CHARACTERS[0]; } } } private final char[] starting; private final char[] prefix; private final int precision; private final GeometricSurface surface; private final boolean isMaximum; GeoHashTileIterable(final int precision) { this(precision, Rectangle.MAXIMUM); } GeoHashTileIterable(final int precision, final GeometricSurface surface) { this.precision = precision; this.surface = surface; this.starting = new char[precision]; this.prefix = prefix(); this.isMaximum = surface instanceof Rectangle && Rectangle.MAXIMUM.equals(surface); for (int index = 0; index < this.prefix.length; index++) { this.starting[index] = this.prefix[index]; } for (int index = this.prefix.length; index < precision; index++) { this.starting[index] = GeoHashTile.GEOHASH_CHARACTERS[0]; } } @Override public Iterator iterator() { return new GeoHashTileIterator(); } private boolean isMaximum() { return this.isMaximum; } /** * @return The prefix of geohash that totally covers the bounds of the surface */ private char[] prefix() // NOSONAR { if (isMaximum()) { return new char[0]; } final List geoHashes = new ArrayList<>(); for (final Location corner : this.surface.bounds()) { geoHashes.add(GeoHashTile.covering(corner, this.precision).toCharArray()); } final List prefixAsList = new ArrayList<>(); for (int index = 0; index < this.precision; index++) { Character candidate = null; boolean valid = true; for (int cornerIndex = 0; cornerIndex < geoHashes.size(); cornerIndex++) { final char cornerCharacter = geoHashes.get(cornerIndex)[index]; if (candidate == null) { candidate = cornerCharacter; } else if (!candidate.equals(cornerCharacter)) { valid = false; break; } } if (valid) { prefixAsList.add(candidate); } else { break; } } final char[] result = new char[prefixAsList.size()]; for (int index = 0; index < prefixAsList.size(); index++) { result[index] = prefixAsList.get(index); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/LocationToShardCommand.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.util.Set; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * @author matthieun */ public class LocationToShardCommand extends Command { private static final Switch SHARDING = new Switch<>("sharding", "The sharding tree", Sharding::forString, Optionality.REQUIRED); private static final Switch LOCATION = new Switch<>("location", "The location to check as \"latitude,longitude\"", Location::forString, Optionality.OPTIONAL); private static final Switch WKT_POINT = new Switch<>("wktPoint", "The location to check as a WKT point", Location::forWkt, Optionality.OPTIONAL); public static void main(final String[] args) { new LocationToShardCommand().run(args); } @Override protected int onRun(final CommandMap command) { final Sharding sharding = (Sharding) command.get(SHARDING); Location location = (Location) command.get(LOCATION); if (location == null) { location = (Location) command.get(WKT_POINT); } if (location == null) { System.err.println("No location found! Make sure to use either the -" + LOCATION.getName() + " or -" + WKT_POINT.getName() + " switch."); return 0; } final Set shards = Iterables.asSet(sharding.shardsCovering(location)); if (shards.size() > 0) { System.out.println("Shard(s) covering " + location.toCompactString() + ":"); shards.forEach( shard -> System.out.println(shard.getName() + "; " + shard.bounds().toWkt())); } else { System.err.println("No shard found!"); return 1; } return 0; } @Override protected SwitchList switches() { return new SwitchList().with(SHARDING, LOCATION, WKT_POINT); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/README.md ================================================ # Sharding Sharding is a way to split the world in small units of work, with the goal of having each unit be roughly the same effort. ## SlippyTileSharding Each shard is a bounding box referenced with a zoom-level, x and y coordinates. All shards have the same zoom-level and the same size. Example: Invoke with `Sharding.forString("slippy@10");` ## DynamicTreeSharding Each shard is a bounding box referenced with a zoom-level, x and y coordinates. All shards are leaf nodes in a quad tree, the root of which is `Rectangle.MAXIMUM`. The quad tree can be serialized to a simple text file. Example: Invoke with `Sharding.forString("dynamic@file:///path/to/tree.txt");` ## GeohashSharding Each shard is a bounding box referenced with a geohash encoded string. All shards have the same precision and the same size. Example: Invoke with `Sharding.forString("geohash@4");` ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/Shard.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.io.Serializable; import org.openstreetmap.atlas.geography.GeometryPrintable; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.sharding.converters.StringToShardConverter; /** * A {@link Sharding} shard. * * @author matthieun */ public interface Shard extends Located, Serializable, GeometryPrintable // NOSONAR { /** * The separator character for data within a shard string. For example, this character should be * used to separate the country code from the shard name: USA_1-2-3. It should also be used to * separate metadata from the shard name: USA_4-5-6_zz/xx/yy */ String SHARD_DATA_SEPARATOR = "_"; /** * Get the name of this {@link Shard}. The result of this method should be parsable by * {@link StringToShardConverter}. * * @return the parsable name of this {@link Shard} */ String getName(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/Sharding.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.io.Serializable; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.geojson.GeoJson; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.gson.JsonArray; import com.google.gson.JsonObject; /** * Sharding strategy * * @author matthieun */ public interface Sharding extends Serializable, GeoJson { int SHARDING_STRING_SPLIT = 2; int SLIPPY_ZOOM_MAXIMUM = 18; /** * Parse a sharding definition string and use the given {@link FileSystem} if the {@link String} * references a {@link DynamicTileSharding}. * * @param sharding * The definition string * @param fileSystem * the {@link FileSystem} to use for a {@link DynamicTileSharding} * @return The corresponding {@link Sharding} instance. */ static Sharding forString(final String sharding, final FileSystem fileSystem) { final StringList split; split = StringList.split(sharding, "@"); if (split.size() != SHARDING_STRING_SPLIT) { throw new CoreException( "Invalid sharding string: {} (correct e.g. dynamic@/path/to/tree, slippy@9, etc.)", sharding); } if ("slippy".equals(split.get(0))) { final int zoom; zoom = Integer.valueOf(split.get(1)); if (zoom > SLIPPY_ZOOM_MAXIMUM) { throw new CoreException("Slippy Sharding zoom too high : {}, max is {}", zoom, SLIPPY_ZOOM_MAXIMUM); } return new SlippyTileSharding(zoom); } if ("geohash".equals(split.get(0))) { final int precision; precision = Integer.valueOf(split.get(1)); return new GeoHashSharding(precision); } if ("dynamic".equals(split.get(0))) { final String definition = split.get(1); return new DynamicTileSharding(new File(definition, fileSystem)); } throw new CoreException("Sharding type {} is not recognized.", split.get(0)); } /** * Parse a sharding definition string and use the default {@link FileSystem} if the * {@link String} references a {@link DynamicTileSharding}. * * @param sharding * The definition string * @return The corresponding {@link Sharding} instance. */ static Sharding forString(final String sharding) { return Sharding.forString(sharding, FileSystems.getDefault()); } @Override default JsonObject asGeoJson() { final JsonObject featureCollectionObject = new JsonObject(); featureCollectionObject.addProperty("type", "FeatureCollection"); final JsonArray features = new JsonArray(); for (final Shard shard : this.shards(Rectangle.MAXIMUM)) { final JsonObject featureObject = new JsonObject(); featureObject.addProperty("type", "Feature"); featureObject.add("geometry", new PolyLine(shard.bounds().closedLoop()).asGeoJsonGeometry()); final JsonObject propertiesObject = new JsonObject(); propertiesObject.addProperty("shard", shard.getName()); featureObject.add("properties", propertiesObject); features.add(featureObject); } featureCollectionObject.add("features", features); return featureCollectionObject; } @Override default GeoJsonType getGeoJsonType() { return GeoJsonType.FEATURE_COLLECTION; } /** * Get the name of this {@link Sharding}. The name should succinctly describe this * {@link Sharding}'s parameters. * * @return the name of this {@link Sharding} */ String getName(); /** * Get the neighboring shards for a given shard. * * @param shard * The shard for which to get neighbors * @return The shards {@link Iterable}, neighboring the supplied shard */ Iterable neighbors(Shard shard); /** * Get a shard given its name * * @param name * The name of the shard * @return The corresponding shard */ Shard shardForName(String name); /** * Generate shards. This needs to be deterministic! * * @param surface * The bounds to limit the shards. * @return The shards {@link Iterable}. */ Iterable shards(GeometricSurface surface); /** * Generate shards for the whole planet. This needs to be deterministic! * * @return The shards {@link Iterable}, covering the whole planet. */ default Iterable shards() { return shards(Rectangle.MAXIMUM); } /** * Generate shards. This needs to be deterministic! * * @param location * The location to find * @return The shards {@link Iterable} (In case the location falls right at the boundary between * shards) */ Iterable shardsCovering(Location location); /** * Generate shards. This needs to be deterministic! * * @param polyLine * The line intersecting the shards * @return The shards {@link Iterable}. */ Iterable shardsIntersecting(PolyLine polyLine); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/SlippyTile.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Queue; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import org.openstreetmap.atlas.geography.sharding.converters.SlippyTileConverter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Distance; import com.google.gson.JsonObject; /** * OSM Slippy tile * * @see "http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames" * @author matthieun * @author mgostintsev */ public class SlippyTile implements Shard, Comparable { public static final SlippyTile ROOT = new SlippyTile(0, 0, 0); public static final int MAX_ZOOM = 30; public static final String COORDINATE_SEPARATOR = "-"; private static final long serialVersionUID = -3752920878013084039L; private static final SlippyTileConverter CONVERTER = new SlippyTileConverter(); private static final double CIRCULAR_MULTIPLIER = 2.0; private static final double ZOOM_LEVEL_POWER = 2.0; private static final int BIT_SHIFT = 2; private static final double FULL_ROTATION_DEGREES = 360.0; private static final double HALF_ROTATION_DEGREES = 180.0; private static final double LONGITUDE_BOUNDARY = 180; private static final double NEIGHBOR_EXPANSION_SCALE = 0.10; private Rectangle bounds; private final int xAxis; private final int yAxis; private final int zoom; /** * All tiles at some zoom level * * @param zoom * The zoom to consider * @return All tiles at some zoom level */ public static Iterable allTiles(final int zoom) { return allTiles(zoom, Rectangle.MAXIMUM); } /** * All tiles within some bounds * * @param zoom * The zoom to consider * @param bounds * The bounds to consider * @return All tiles within some bounds */ public static Iterable allTiles(final int zoom, final Rectangle bounds) { if (zoom > MAX_ZOOM) { throw new CoreException("Zoom too large."); } final Iterable result = () -> allTilesIterator(zoom, bounds); final List list = Iterables.asList(result); if (list.isEmpty()) { throw new CoreException("List cannot be empty"); } return list; } /** * Iterator for all tiles within some bounds * * @param zoom * The zoom to consider * @param bounds * The bounds to consider * @return Iterator for all tiles within some bounds */ public static Iterator allTilesIterator(final int zoom, final Rectangle bounds) { if (zoom > MAX_ZOOM) { throw new CoreException("Zoom too large."); } final SlippyTile lowerLeft = new SlippyTile(bounds.lowerLeft(), zoom); final SlippyTile upperRight = new SlippyTile(bounds.upperRight(), zoom); final int minX = lowerLeft.getX(); final int maxX = upperRight.getX(); final int minY = upperRight.getY(); final int maxY = lowerLeft.getY(); return new Iterator() { private int xAxis = minX; private int yAxis = minY; @Override public boolean hasNext() { return this.yAxis <= maxY && this.xAxis <= maxX; } @Override public SlippyTile next() { if (!hasNext()) { throw new NoSuchElementException(); } final SlippyTile result = new SlippyTile(this.xAxis, this.yAxis, zoom); this.xAxis++; if (this.xAxis > maxX) { this.yAxis++; this.xAxis = minX; } return result; } }; } /** * Expansion distance will be the smaller of the height/width scaled by 1/4. * * @param bounds * The bounds * @return The smaller of the height/width scaled by 1/4 */ public static Distance calculateExpansionDistance(final Rectangle bounds) { final Distance shorterSide = bounds.width().onEarth().isLessThanOrEqualTo( bounds.height().onEarth()) ? bounds.width().onEarth() : bounds.height().onEarth(); return shorterSide.scaleBy(NEIGHBOR_EXPANSION_SCALE); } public static SlippyTile forName(final String name) { return CONVERTER.backwardConvert(name); } /** * Construct * * @param xAxis * The x index (along north-south) * @param yAxis * The y index (along west-east) * @param zoom * The zoom level */ public SlippyTile(final int xAxis, final int yAxis, final int zoom) { if (zoom > MAX_ZOOM) { throw new CoreException("Zoom {} is too large.", zoom); } this.zoom = zoom; this.xAxis = xAxis; this.yAxis = yAxis; } /** * Construct * * @param location * The location to get the overlapping {@link SlippyTile} * @param zoom * The zoom level */ public SlippyTile(final Location location, final int zoom) { if (zoom > MAX_ZOOM) { throw new CoreException("Zoom {} is too large.", zoom); } final List tileNumbers = this.getTileNumbers(location, zoom); this.zoom = zoom; this.xAxis = tileNumbers.get(0); this.yAxis = tileNumbers.get(1); } @Override public JsonObject asGeoJson() { return bounds().asGeoJsonGeometry(); } @Override public Rectangle bounds() { if (this.bounds == null) { this.bounds = tile2boundingBox(this.xAxis, this.yAxis, this.zoom); } return this.bounds; } @Override public int compareTo(final SlippyTile other) { // Order by z-level, x-value and then y-value final int zoomLevelDelta = this.getZoom() - other.getZoom(); if (zoomLevelDelta > 0) { return 1; } else if (zoomLevelDelta < 0) { return -1; } else { final int xDelta = this.getX() - other.getX(); if (xDelta > 0) { return 1; } else if (xDelta < 0) { return -1; } else { final int yDelta = this.getY() - other.getY(); if (yDelta > 0) { return 1; } else if (yDelta < 0) { return -1; } else { return 0; } } } } @Override public boolean equals(final Object other) { if (other instanceof SlippyTile) { final SlippyTile that = (SlippyTile) other; return this.getZoom() == that.getZoom() && this.getX() == that.getX() && this.getY() == that.getY(); } return false; } @Override public GeoJsonType getGeoJsonType() { return GeoJsonType.POLYGON; } @Override public String getName() { return CONVERTER.convert(this); } /** * @return The tile's X index. */ public int getX() { return this.xAxis; } /** * @return The tile's Y index. */ public int getY() { return this.yAxis; } /** * @return The tile's zoom. */ public int getZoom() { return this.zoom; } @Override public int hashCode() { final long result = (long) Math.pow(getMaxXorY(this.zoom - 1), 2) + (long) Math.pow(this.xAxis, 2) + this.yAxis; return (int) result % Integer.MAX_VALUE; } /** * @return All neighbors for the current tile, including diagonal tiles that may share a single * vertex on the corner they meet. */ public Set neighbors() { return Iterables .stream(new SlippyTileSharding(this.zoom) .shards(bounds().expand(calculateExpansionDistance(bounds())))) .map(shard -> (SlippyTile) shard).filter(tile -> !this.equals(tile)).collectToSet(); } /** * @return The {@link SlippyTile} that has one less zoom level, and that covers that * {@link SlippyTile}. If the zoom is already 0, return itself. */ public SlippyTile parent() { if (this.zoom == 0) { return this; } final int parentZoom = this.zoom - 1; final int parentX = this.xAxis / 2; final int parentY = this.yAxis / 2; return new SlippyTile(parentX, parentY, parentZoom); } /** * @return The 4 {@link SlippyTile} that represent the current {@link SlippyTile} at one more * zoom level. */ public List split() { if (this.zoom == MAX_ZOOM) { throw new CoreException("Cannot split further than zoom {}", MAX_ZOOM); } final List result = new ArrayList<>(); for (int i = 2 * this.xAxis; i <= 2 * this.xAxis + 1; i++) { for (int j = 2 * this.yAxis; j <= 2 * this.yAxis + 1; j++) { result.add(new SlippyTile(i, j, this.zoom + 1)); } } return result; } /** * Split a {@link SlippyTile} up to the given zoom level. If the zoom is smaller than the * current tile's, then the method will return the only parent tile at the specified zoom in the * result list. * * @param newZoom * The zoom to split to * @return the split tiles */ public List split(final int newZoom) { if (newZoom < 0 || newZoom > MAX_ZOOM) { throw new CoreException( "Cannot split to a zoom {} which is not between 0 and {} included.", newZoom, MAX_ZOOM); } List result = new ArrayList<>(); if (newZoom == this.zoom) { result.add(this); } if (newZoom > this.zoom) { List temporary = new ArrayList<>(); temporary.add(this); int temporaryZoom = this.zoom + 1; while (temporaryZoom <= newZoom) { final List newTemporary = new ArrayList<>(); for (final SlippyTile temporaryTile : temporary) { newTemporary.addAll(temporaryTile.split()); } temporary = newTemporary; temporaryZoom++; } result = temporary; } if (newZoom < this.zoom) { SlippyTile temporary = this; int temporaryZoom = temporary.getZoom() - 1; while (temporaryZoom >= newZoom) { temporary = temporary.parent(); temporaryZoom--; } result.add(temporary); } return result; } @Override public String toString() { return "[SlippyTile: zoom = " + this.zoom + ", x = " + this.xAxis + ", y = " + this.yAxis + "]"; } @Override public byte[] toWkb() { return bounds().toWkb(); } @Override public String toWkt() { return bounds().toWkt(); } /** * Add the siblings and parent. * * @param candidates * The candidate tiles * @param visitedTiles * The tiles already visited * @param targetTile * The target tile */ protected void getNeighborsForAllZoomLevels(final Queue candidates, final Set visitedTiles, final SlippyTile targetTile) { final SlippyTile parent = targetTile.parent(); for (final SlippyTile child : parent.split()) { if (!visitedTiles.contains(child.getName()) && !child.equals(targetTile)) { candidates.add(child); } } candidates.add(parent); } /** * @param zoom * The provided zoom level * @return The maximum slippy tile index (x or y) for that zoom level */ private int getMaxXorY(final int zoom) { if (zoom <= 0) { return 1; } final int result = getTileNumbers(new Location(Latitude.MINIMUM, Longitude.ZERO), zoom) .get(0); return result; } /** * @param location * The location to consider * @param zoom * The zoom level * @return The list made of the corresponding x and y indices. */ private List getTileNumbers(final Location location, final int zoom) { final double latitude = location.getLatitude().asDegrees(); final double longitude = location.getLongitude().asDegrees(); int xAxis = (int) Math .floor((longitude + HALF_ROTATION_DEGREES) / FULL_ROTATION_DEGREES * (1 << zoom)); int yAxis = (int) Math.floor((1 - Math .log(Math.tan(Math.toRadians(latitude)) + 1 / Math.cos(Math.toRadians(latitude))) / Math.PI) / BIT_SHIFT * (1 << zoom)); if (xAxis < 0) { xAxis = 0; } if (xAxis >= 1 << zoom) { xAxis = (1 << zoom) - 1; } if (yAxis < 0) { yAxis = 0; } if (yAxis >= 1 << zoom) { yAxis = (1 << zoom) - 1; } final List result = new ArrayList<>(); result.add(xAxis); result.add(yAxis); return result; } /** * @param xAxis * The x index * @param yAxis * The y index * @param zoom * The zoom level * @return The corresponding bounding box {@link Rectangle} */ private Rectangle tile2boundingBox(final int xAxis, final int yAxis, final int zoom) { final Latitude minLat = tile2lat(yAxis + 1, zoom); final Latitude maxLat = tile2lat(yAxis, zoom); final Longitude minLon = tile2lon(xAxis, zoom); final Longitude maxLon = tile2lon(xAxis + 1, zoom); return Rectangle.forCorners(new Location(minLat, minLon), new Location(maxLat, maxLon)); } /** * @param yAxis * The tile's y index * @param zoom * The zoom level * @return The corresponding latitude */ private Latitude tile2lat(final int yAxis, final int zoom) { final double pivot = Math.PI - CIRCULAR_MULTIPLIER * Math.PI * yAxis / Math.pow(ZOOM_LEVEL_POWER, zoom); return Latitude.degrees(Math.toDegrees(Math.atan(Math.sinh(pivot)))); } /** * @param xAxis * The tile's x index * @param zoom * The zoom level * @return The corresponding longitude */ private Longitude tile2lon(final int xAxis, final int zoom) { final double longitude = xAxis / Math.pow(ZOOM_LEVEL_POWER, zoom) * FULL_ROTATION_DEGREES - HALF_ROTATION_DEGREES; return longitude >= LONGITUDE_BOUNDARY ? Longitude.MAXIMUM : longitude < -LONGITUDE_BOUNDARY ? Longitude.MINIMUM : Longitude.degrees(longitude); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/SlippyTileSharding.java ================================================ package org.openstreetmap.atlas.geography.sharding; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.GeometricSurface; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Atlas Sharding with {@link SlippyTile}s. * * @author matthieun */ public class SlippyTileSharding implements Sharding { private static final long serialVersionUID = -6727830583309410676L; private final int zoom; public SlippyTileSharding(final int zoom) { this.zoom = zoom; } @Override public String getName() { return "slippy@" + this.zoom; } @Override public Iterable neighbors(final Shard shard) { if (shard instanceof SlippyTile) { return ((SlippyTile) shard).neighbors().stream().map(neighbor -> (Shard) neighbor) .collect(Collectors.toSet()); } else { throw new CoreException("Cannot have neighbors from another type of shard."); } } @Override public Shard shardForName(final String name) { final SlippyTile result = SlippyTile.forName(name); if (result.getZoom() != this.zoom) { throw new CoreException("This sharding is of zoom {}, and \"{}\" is not.", this.zoom, name); } return result; } @Override public Iterable shards(final GeometricSurface surface) { return Iterables.stream(SlippyTile.allTiles(this.zoom, surface.bounds())) .filter(slippyTile -> surface.overlaps(slippyTile.bounds())) .map(shard -> (Shard) shard); } @Override public Iterable shardsCovering(final Location location) { return Iterables.stream(SlippyTile.allTiles(this.zoom, location.bounds())) .filter(slippyTile -> slippyTile.bounds().fullyGeometricallyEncloses(location)) .map(shard -> (Shard) shard); } @Override public Iterable shardsIntersecting(final PolyLine polyLine) { return Iterables.stream(SlippyTile.allTiles(this.zoom, polyLine.bounds())) .filter(slippyTile -> polyLine.intersects(slippyTile.bounds()) || slippyTile.bounds().fullyGeometricallyEncloses(polyLine)) .map(shard -> (Shard) shard); } @Override public String toString() { return "slippy:" + this.zoom; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/converters/DynamicTileShardingGeoJsonConverter.java ================================================ package org.openstreetmap.atlas.geography.sharding.converters; import org.openstreetmap.atlas.geography.sharding.DynamicTileSharding; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Save an existing serialized tree into a Geojson file. * * @author matthieun */ public class DynamicTileShardingGeoJsonConverter extends Command { public static final Switch GEOJSON = new Switch<>("geojson", "The resource where to save the geojson tree for debugging", File::new, Optionality.REQUIRED); public static final Switch INPUT = new Switch<>("input", "The resource where to read the serialized tree", File::new, Optionality.REQUIRED); public static void main(final String[] args) { new DynamicTileShardingGeoJsonConverter().run(args); } @Override protected int onRun(final CommandMap command) { final Resource input = (Resource) command.get(INPUT); final WritableResource geojson = (WritableResource) command.get(GEOJSON); new DynamicTileSharding(input).saveAsGeoJson(geojson); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(GEOJSON, INPUT); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/converters/RectangleToSpatial4JRectangleConverter.java ================================================ package org.openstreetmap.atlas.geography.sharding.converters; import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * @author matthieun */ public class RectangleToSpatial4JRectangleConverter implements TwoWayConverter { @Override public org.locationtech.spatial4j.shape.Rectangle backwardConvert(final Rectangle other) { final Location lowerLeft = other.lowerLeft(); final Location upperRight = other.upperRight(); return new RectangleImpl(lowerLeft.getLongitude().asDegrees(), upperRight.getLongitude().asDegrees(), lowerLeft.getLatitude().asDegrees(), upperRight.getLatitude().asDegrees(), SpatialContext.GEO); } @Override public Rectangle convert(final org.locationtech.spatial4j.shape.Rectangle other) { return Rectangle.forCorners( new Location(Latitude.degrees(other.getMinY()), Longitude.degrees(other.getMinX())), new Location(Latitude.degrees(other.getMaxY()), Longitude.degrees(other.getMaxX()))); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/converters/SlippyTileConverter.java ================================================ package org.openstreetmap.atlas.geography.sharding.converters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Convert a SlippyTile to its string value * * @author tony */ public class SlippyTileConverter implements TwoWayConverter { private static final int TILE_DIMENSIONS = 3; @Override public SlippyTile backwardConvert(final String slippyTileParameters) { final StringList splits = StringList.split(slippyTileParameters, SlippyTile.COORDINATE_SEPARATOR); if (splits.size() != TILE_DIMENSIONS) { throw new CoreException("Wrong format of input string {}", slippyTileParameters); } final int zoom = Integer.parseInt(splits.get(0)); final int xAxis = Integer.parseInt(splits.get(1)); final int yAxis = Integer.parseInt(splits.get(2)); return new SlippyTile(xAxis, yAxis, zoom); } @Override public String convert(final SlippyTile slippyTile) { return slippyTile.getZoom() + SlippyTile.COORDINATE_SEPARATOR + slippyTile.getX() + SlippyTile.COORDINATE_SEPARATOR + slippyTile.getY(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/converters/StringToShardConverter.java ================================================ package org.openstreetmap.atlas.geography.sharding.converters; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.sharding.CountryShard; import org.openstreetmap.atlas.geography.sharding.GeoHashTile; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.Converter; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Convert a string representation of a {@link Shard} to a concrete {@link Shard} object. Any time a * new {@link Shard} implementation is added, this class needs to be updated. See the Javadoc for * the {@link StringToShardConverter#convert(String)} method for more information. * * @author lcram */ public class StringToShardConverter implements Converter { private static final String COUNTRY_CODE_REGEX = "^[A-Z][A-Z0-9][A-Z0-9]$"; private static final String SLIPPY_TILE_REGEX = "^[0-9]+\\-[0-9]+\\-[0-9]+$"; private static final String GEOHASH_TILE_REGEX = "^(?:(?![ailo])[0-9a-z])+$"; /** * Convert a string into a concrete {@link Shard} object. This method attempts to handle all * possible shard strings that exist in the wild. Note that this method will correctly handle * country shards in the form "ABC_1-2-3", where "_" is the declared country-shard separator. To * extract metadata from the shard string (e.g. ABC_1-2-3_xx/yy/zz), see * {@link StringToShardConverter#convertWithMetadata}. * * @param shardString * the shard in string format * @return the constructed {@link Shard} */ @Override public Shard convert(final String shardString) { return convertWithMetadata(shardString).getFirst(); } /** * Convert the shard string, and also get any metadata appended to the end of the string. * * @param shardString * the shard in string format * @return a {@link Tuple} containing the constructed {@link Shard} as well as the metadata. */ public Tuple> convertWithMetadata(final String shardString) { final StringList shardSplit = StringList.split(shardString, Shard.SHARD_DATA_SEPARATOR, 2); try { return convertHelper(shardSplit); } catch (final Exception exception) { throw new CoreException("Could not parse shard string: {}", shardString, exception); } } private Tuple> convertHelper(final StringList shardSplit) { /* * In this case, the shardSplit should contain just the shard string. For e.g. "1-2-3" for * SlippyTile or "bb2r5" for GeoHashTile. */ if (shardSplit.size() == 1) { return shardFromSplitExcludingCountryShard(shardSplit); } /* * In this case, the shardSplit should contain a country code, a shard string, and * optionally some additional metadata introduced by downstream client code. For e.g. we may * have "ABC_1-2-2" or "DEF_bb2r5" or "GHI_1-2-3_additionalmetadata". For the purposes of * shard parsing, we ignore the additional metadata. Note that we use a recursive call to * recover the subshard, in the case of a nested CountryShard (e.g. * ABC_DEF_1-2-3_somemetadata). It is also possible the the shardSplit is just a shard * string and some metadata (e.g. 1-2-3_somemetadata). In that case, we ignore country code * parsing. */ else if (shardSplit.size() > 1) { if (shardSplit.get(0).matches(COUNTRY_CODE_REGEX)) { final StringList newListWithoutLeadingCountryCode = StringList .split(shardSplit.get(1), Shard.SHARD_DATA_SEPARATOR, 2); final Tuple> conversionResult = convertHelper( newListWithoutLeadingCountryCode); return new Tuple<>(new CountryShard(shardSplit.get(0), conversionResult.getFirst()), conversionResult.getSecond()); } return shardFromSplitExcludingCountryShard(shardSplit); } /* * Otherwise we have to fail. */ else { throw new CoreException("Split list {} had invalid size {}, must be at least 1", shardSplit, shardSplit.size()); } } /** * A helper to convert a {@link StringList} into a {@link Shard} object, but excluding the * {@link CountryShard} implementation. This method will ignore any trailing metadata. * * @param shardSplit * the {@link StringList} containing the shard info * @return the constructed {@link Shard} */ private Tuple> shardFromSplitExcludingCountryShard( final StringList shardSplit) { if (shardSplit.size() < 1) { throw new CoreException("Split list {} had invalid size {}, must be at least 1", shardSplit, shardSplit.size()); } final Shard shard; final String shardString = shardSplit.get(0); if (shardString.matches(SLIPPY_TILE_REGEX)) { shard = SlippyTile.forName(shardString); } else if (shardString.matches(GEOHASH_TILE_REGEX)) { shard = GeoHashTile.forName(shardString); } else { throw new CoreException("Unrecognized shard component: {}", shardString); } if (shardSplit.size() > 1) { return new Tuple<>(shard, Optional.of(shardSplit.get(1))); } return new Tuple<>(shard, Optional.empty()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/preparation/TilePrinter.java ================================================ package org.openstreetmap.atlas.geography.sharding.preparation; import java.util.Iterator; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.writers.SafeBufferedWriter; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Print all the rectangles of all {@link SlippyTile}s at a specified zoom level in a {@link File}. * This class creates a SQL file that can be executed towards an OSM database, and will return the * way counts for each slippy tile. * * @author matthieun */ public class TilePrinter extends Command { private static final Logger logger = LoggerFactory.getLogger(TilePrinter.class); private static final int MAX_SHARDS_PER_FILE = 4_000_000; private static final Switch OUTPUT_FOLDER = new Switch<>("output", "The output folder", value -> new File(value)); private static final Switch ZOOM_SWITCH = new Switch<>("zoom", "The zoom", value -> Integer.valueOf(value)); private static final Switch USER = new Switch<>("user", "The user for the db", value -> new String(value)); private int index; private File folder; private int zoom; public static void main(final String[] args) { new TilePrinter().run(args); } @Override protected int onRun(final CommandMap command) { this.index = 0; this.zoom = (int) command.get(ZOOM_SWITCH); this.folder = (File) command.get(OUTPUT_FOLDER); final String user = (String) command.get(USER); this.folder.mkdirs(); SafeBufferedWriter writer = getNextFile().writer(); writer.writeLine("CREATE SCHEMA sharding AUTHORIZATION " + user + ";"); writer.writeLine("DROP TABLE sharding.tiles;"); writer.writeLine( "CREATE TABLE sharding.tiles(tile text, bounds geometry) WITH ( OIDS=FALSE );"); writer.writeLine("ALTER TABLE sharding.tiles OWNER TO " + user + ";"); writer.writeLine("DROP TABLE sharding.counts;"); writer.writeLine( "CREATE TABLE sharding.counts(tile text, count integer) WITH ( OIDS=FALSE );"); writer.writeLine("ALTER TABLE sharding.counts OWNER TO " + user + ";"); final Iterator tileIterator = SlippyTile.allTilesIterator(this.zoom, Rectangle.MAXIMUM); while (tileIterator.hasNext()) { Streams.close(writer); writer = getNextFile().writer(); writer.writeLine("INSERT INTO sharding.tiles(tile, bounds) VALUES "); SlippyTile tile = null; int counter = 0; while (tileIterator.hasNext() && counter < MAX_SHARDS_PER_FILE) { tile = tileIterator.next(); final Rectangle bounds = tile.bounds(); writer.write("(\'" + tile.getName() + "\', ST_MakeEnvelope(" + bounds.lowerLeft().getLongitude().asDegrees() + "," + bounds.lowerLeft().getLatitude().asDegrees() + "," + bounds.upperRight().getLongitude().asDegrees() + "," + bounds.upperRight().getLatitude().asDegrees() + ",4326))"); counter++; if (tileIterator.hasNext() && counter < MAX_SHARDS_PER_FILE) { writer.write(","); } else { writer.write(";"); } writer.write("\n"); } } Streams.close(writer); writer = getNextFile().writer(); writer.writeLine("CREATE OR REPLACE FUNCTION countForTiles() RETURNS void AS $$"); writer.writeLine("DECLARE"); writer.writeLine("s_tile text;"); writer.writeLine("s_bounds geometry;"); writer.writeLine("s_count integer;"); writer.writeLine("n_count integer;"); writer.writeLine("BEGIN"); writer.writeLine(" FOR s_tile, s_bounds IN SELECT tile, bounds FROM sharding.tiles"); writer.writeLine(" LOOP"); writer.writeLine( " SELECT count(*) INTO s_count FROM public.ways WHERE ST_Intersects(s_bounds, linestring);"); writer.writeLine( " SELECT count(*) INTO n_count FROM public.nodes WHERE ST_Intersects(s_bounds, geom);"); writer.writeLine( " INSERT INTO sharding.counts(tile,count) VALUES (s_tile, s_count + n_count);"); writer.writeLine(" END LOOP;"); writer.writeLine(" RETURN;"); writer.writeLine("END"); writer.writeLine("$$ LANGUAGE 'plpgsql' ;"); Streams.close(writer); return 0; } @Override protected SwitchList switches() { return new SwitchList().with(ZOOM_SWITCH, OUTPUT_FOLDER, USER); } private File getNextFile() { final File result = this.folder.child("tiles-" + this.zoom + "_" + this.index++ + ".sql"); logger.info("Generating file {}", result); return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/preparation/tileDownload.sh ================================================ #!/bin/bash # Once the tiles have been counted in the database, use this to download them. : ${1:?"Make sure to pass the database user name to the script"} psql -U $1 -d osm_planet -c "\copy sharding.counts TO ./counts.csv WITH DELIMITER ',' CSV" ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/preparation/tileExecute.sh ================================================ #!/bin/bash # Run the function that will count the tiles members in the database. : ${1:?"Make sure to pass the database user name to the script"} psql -U $1 -d osm_planet -c "SELECT * FROM countForTiles()" ================================================ FILE: src/main/java/org/openstreetmap/atlas/geography/sharding/preparation/tilePrinter.sh ================================================ #!/bin/bash # Once the tiles sql files are generated, run this script to have the database do the counting. : ${1:?"Make sure to pass the database user name to the script"} for sqlscript in ./*.sql do echo "processing $sqlscript" psql -U $1 -d osm_planet -f $sqlscript done ================================================ FILE: src/main/java/org/openstreetmap/atlas/locale/IsoCountry.java ================================================ package org.openstreetmap.atlas.locale; import java.io.Serializable; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Java Locale based countries, including ISO2, ISO3, and descriptive country name representations. * * @author robert_stack * @author lcram */ public final class IsoCountry implements Serializable { // Package private fields used by other classes in the locale package static final Set ALL_COUNTRY_CODES; static final Set ALL_DISPLAY_COUNTRIES; static final Map ISO2_TO_DISPLAY_COUNTRY; static final Map DISPLAY_COUNTRY_TO_ISO2; static final Map ISO2_TO_ISO3; static final Map ISO3_TO_ISO2; // private static final BiMap ISO2_ISO3_MAP; static final Map ISO_COUNTRIES; private static final long serialVersionUID = 8686298246454085812L; private static final Logger logger = LoggerFactory.getLogger(IsoCountry.class); // Use United States fixed Locale for display use cases private static final String LOCALE_LANGUAGE = Locale.ENGLISH.getLanguage(); private static final int ISO2_LENGTH = 2; private static final int ISO3_LENGTH = 3; static { // All Locale based 2 letter country codes final String[] countries = Locale.getISOCountries(); // Set of language codes--exposed publically through allLanguageCodes() ALL_COUNTRY_CODES = Collections.unmodifiableSet( Arrays.stream(countries).map(String::intern).collect(Collectors.toSet())); // Map from ISO2 to full country name ISO2_TO_DISPLAY_COUNTRY = Collections.unmodifiableMap( Arrays.stream(countries).collect(Collectors.toMap(iso2 -> iso2.intern(), iso2 -> new Locale(LOCALE_LANGUAGE, iso2).getDisplayCountry().intern()))); /* * Check that country names are actually unique, and log an error if not. NOTE that this * relies on English country names. If you are updating this code to handle * internationalization, these assumptions may not hold. */ final Map countriesSeen = new HashMap<>(); for (final String iso2Country : countries) { final String countryName = new Locale(LOCALE_LANGUAGE, iso2Country).getDisplayCountry() .intern(); if (countriesSeen.containsKey(countryName)) { logger.error("Detected duplicate country name {} -> {} AND {}", countryName, iso2Country, countriesSeen.get(countryName)); } countriesSeen.put(countryName, iso2Country); } // Map from full country name to ISO2 DISPLAY_COUNTRY_TO_ISO2 = Collections.unmodifiableMap(Arrays.stream(countries) .collect(Collectors.toMap( iso2 -> new Locale(LOCALE_LANGUAGE, iso2).getDisplayCountry().intern(), iso2 -> iso2.intern()))); // Map from ISO2 to ISO3 ISO2_TO_ISO3 = Collections.unmodifiableMap( Arrays.stream(countries).collect(Collectors.toMap(iso2 -> iso2.intern(), iso2 -> new Locale(LOCALE_LANGUAGE, iso2).getISO3Country().intern()))); // Map from ISO3 to ISO2 // Have verified a 1:1 between ISO2 to ISO3, otherwise there would be overwrites of ISO3 // keys ISO3_TO_ISO2 = Collections .unmodifiableMap(ISO2_TO_ISO3.entrySet().stream().collect(Collectors .toMap(iso3 -> iso3.getValue().intern(), iso3 -> iso3.getKey().intern()))); // TODO Use Guava BiMap // ISO2_ISO3_MAP = Collections.unmodifiableMap(Arrays.stream(countries).collect( // Collectors.toMap(x -> x.get(0), x -> x.get(1), (a, b) -> b, HashBiMap::create))); // Map from ISO2 to IsoCountry ISO_COUNTRIES = Collections.unmodifiableMap(Arrays.stream(countries) .collect(Collectors.toMap(iso2 -> iso2.intern(), iso2 -> new IsoCountry(iso2)))); // Set of display country names, do this one last since it relies on the other maps ALL_DISPLAY_COUNTRIES = Collections.unmodifiableSet(ALL_COUNTRY_CODES.stream() .map(IsoCountry::displayCountry).map(Optional::get).collect(Collectors.toSet())); } // This validated country code private final String iso2CountryCode; private final String iso3CountryCode; private final String displayCountry; /** * Provides a set of all Locale based country codes; supports convenience methods that use all * country codes * * @return Set of country codes */ public static Set allCountryCodes() { return ALL_COUNTRY_CODES; } /** * Provides a set of all Locale based country long names. * * @return Set of country long names */ public static Set allDisplayCountries() { return ALL_DISPLAY_COUNTRIES; } /** * Provides long name country for country code * * @param countryCode * 2 or 3 character country code, case sensitive (examples "US", "USA") * @return Optional of display country string (example "United States") */ public static Optional displayCountry(final String countryCode) { if (countryCode != null) { if (countryCode.length() == ISO2_LENGTH && ISO2_TO_ISO3.keySet().contains(countryCode)) { final String displayCountry = ISO2_TO_DISPLAY_COUNTRY.get(countryCode); return Optional.ofNullable(displayCountry); } if (countryCode.length() == ISO3_LENGTH && ISO3_TO_ISO2.keySet().contains(countryCode)) { final String iso2 = ISO3_TO_ISO2.get(countryCode); if (iso2 != null) { final String displayCountry = ISO2_TO_DISPLAY_COUNTRY.get(iso2); return Optional.ofNullable(displayCountry); } } } return Optional.empty(); } /** * Provides IsoCountry for valid country code * * @param countryCode * 2 or 3 character country code, case sensitive (examples "US", "USA") * @return Optional of valid country code, or Optional.empty if not recognized */ public static Optional forCountryCode(final String countryCode) { if (countryCode != null) { if (countryCode.length() == ISO2_LENGTH && ISO2_TO_ISO3.keySet().contains(countryCode)) { return Optional.ofNullable(ISO_COUNTRIES.get(countryCode)); } else if (countryCode.length() == ISO3_LENGTH && ISO3_TO_ISO2.keySet().contains(countryCode)) { return Optional.ofNullable(ISO_COUNTRIES.get(ISO3_TO_ISO2.get(countryCode))); } } return Optional.empty(); } /** * Provides IsoCountry for a valid country display name. Ignores capitalization (e.g. "united * stAtes" and "United States" are the same) * * @param displayCountry * the display country name, e.g. "United States" * @return an Optional containing the IsoCountry if present */ public static Optional forDisplayCountry(final String displayCountry) { if (displayCountry != null) { /* * We want to allow the displayCountry parameter to have inconsistent case. E.g. * displayCountry="united States" should match IsoCountry<"United States"> */ String foundKey = null; for (final String key : DISPLAY_COUNTRY_TO_ISO2.keySet()) { if (displayCountry.equalsIgnoreCase(key)) { foundKey = key; break; } } return Optional.ofNullable(foundKey).map(DISPLAY_COUNTRY_TO_ISO2::get) .map(ISO_COUNTRIES::get); } return Optional.empty(); } /** * Indicates whether the ISO2 or ISO3 country code is valid * * @param isoCountry * 2 or 3 character country code, case sensitive (examples "US", "USA") * @return Whether this is a valid ISO2 or ISO3 country code */ public static boolean isValidCountryCode(final String isoCountry) { if (isoCountry != null) { if (isoCountry.length() == ISO2_LENGTH && ISO2_TO_ISO3.keySet().contains(isoCountry)) { return true; } else if (isoCountry.length() == ISO3_LENGTH && ISO3_TO_ISO2.keySet().contains(isoCountry)) { return true; } } return false; } /** * Provides ISO2 string for ISO3 * * @param iso3 * 3 character country code, case sensitive (example "USA") * @return Optional of ISO2 country code, or Optional.empty if not recognized */ public static Optional iso2ForIso3(final String iso3) { String iso2 = null; if (iso3 != null) { iso2 = ISO3_TO_ISO2.get(iso3); } return Optional.ofNullable(iso2); } /** * Provides ISO3 string for ISO2 * * @param iso2 * 2 character country code, case sensitive (example "US") * @return Optional of ISO3 country code, or Optional.empty if not recognized */ public static Optional iso3ForIso2(final String iso2) { String iso3 = null; if (iso2 != null) { iso3 = ISO2_TO_ISO3.get(iso2); } return Optional.ofNullable(iso3); } private IsoCountry(final String iso2) { this.iso2CountryCode = iso2; this.iso3CountryCode = ISO2_TO_ISO3.get(this.iso2CountryCode); this.displayCountry = ISO2_TO_DISPLAY_COUNTRY.get(this.iso2CountryCode); } /** * Provides the ISO2 country code for this IsoCountry * * @return 2 character (ISO2) language code */ public String getCountryCode() { return this.iso2CountryCode; } /** * Provides the display name for this IsoCountry * * @return Display country string */ public String getDisplayCountry() { return this.displayCountry; } /** * Provides the ISO3 country code for this IsoCountry * * @return 3 character (ISO3) language code */ public String getIso3CountryCode() { return this.iso3CountryCode; } @Override public String toString() { return this.getDisplayCountry(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/locale/IsoCountryFuzzyMatcher.java ================================================ package org.openstreetmap.atlas.locale; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; import org.apache.commons.text.similarity.LevenshteinDistance; import org.openstreetmap.atlas.exception.CoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Extends {@link IsoCountry} functionality with fuzzy matching for display names. Note that this * functionality is very simple, and assumes dependence on English country name representations. * * @author lcram */ public final class IsoCountryFuzzyMatcher { private static final Logger logger = LoggerFactory.getLogger(IsoCountryFuzzyMatcher.class); /** * Provides closest IsoCountries for a country display name. If the given display name does not * perfectly match a valid IsoCountry, this will return the closest string match up to number of * matches. * * @param number * the number of matches to show * @param displayCountry * the display country name, e.g. "united stats" (which does not match any display * country) * @return an Optional containing the IsoCountry if present */ public static List forDisplayCountryTopMatches(final int number, final String displayCountry) { if (displayCountry != null) { final List results = new ArrayList<>(); if (IsoCountry.DISPLAY_COUNTRY_TO_ISO2.containsKey(displayCountry)) { results.add(IsoCountry.ISO_COUNTRIES .get(IsoCountry.DISPLAY_COUNTRY_TO_ISO2.get(displayCountry))); } else { final List closestCountries = closestIsoCountries(number, displayCountry); if (!closestCountries.isEmpty()) { logger.debug( "Exact match for {} was not found, returning closest {} matches {}", displayCountry, number, closestCountries); results.addAll(closestCountries.stream() .map(countryString -> IsoCountry.ISO_COUNTRIES .get(IsoCountry.DISPLAY_COUNTRY_TO_ISO2.get(countryString))) .collect(Collectors.toList())); } } return results; } return new ArrayList<>(); } /* * This algorithm could definitely be improved for cases with multi-word country matching. */ private static List closestIsoCountries(final int number, final String displayCountry) { if (number <= 0 || number > IsoCountry.ALL_DISPLAY_COUNTRIES.size()) { throw new CoreException("number " + number + " out of range (0, " + IsoCountry.ALL_DISPLAY_COUNTRIES.size() + "]"); } final Map countryRankings = new HashMap<>(); for (final String countryName : IsoCountry.ALL_DISPLAY_COUNTRIES) { final String countryNameLowerCase = countryName.toLowerCase(); final String[] nameComponents = countryNameLowerCase.split("\\s+"); if (nameComponents.length > 1) { int bestInterCountryDistance = Integer.MAX_VALUE; for (final String nameComponent : nameComponents) { final int distance = LevenshteinDistance.getDefaultInstance() .apply(displayCountry, nameComponent); if (distance < bestInterCountryDistance) { bestInterCountryDistance = distance; } countryRankings.put(countryName, bestInterCountryDistance); } } else { final int distance = LevenshteinDistance.getDefaultInstance().apply(displayCountry, countryNameLowerCase); countryRankings.put(countryName, distance); } } final List> entries = new ArrayList<>(countryRankings.entrySet()); entries.sort(Entry.comparingByValue()); return entries.subList(0, number).stream().map(Entry::getKey).collect(Collectors.toList()); } private IsoCountryFuzzyMatcher() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/locale/IsoLanguage.java ================================================ package org.openstreetmap.atlas.locale; import java.io.Serializable; import java.util.Arrays; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; import org.openstreetmap.atlas.utilities.collections.EnhancedCollectors; /** * Languages derived from Locale * * @author robert_stack */ public final class IsoLanguage implements Comparable, Serializable { // Use United States fixed Locale for display use cases private static final Locale LANGUAGE_LOCALE = Locale.US; private static Set ALL_LANGUAGE_CODES; private static Set DISPLAY_LANGUAGES_SET; private static Map ISO_LANGUAGES; static { // All 2 letter language codes final String[] languages = Locale.getISOLanguages(); // Set of language codes--exposed publically through allLanguageCodes() ALL_LANGUAGE_CODES = Arrays.stream(languages) .collect(EnhancedCollectors.toUnmodifiableSortedSet()); // Set of display languages; here only to support performant validation of display language. // Using a specific fixed Locale rather than system dependent Locale DISPLAY_LANGUAGES_SET = Arrays.stream(languages) .map(language -> new Locale(language).getDisplayLanguage(LANGUAGE_LOCALE)) .collect(EnhancedCollectors.toUnmodifiableSortedSet()); // Map from language codes to IsoLanguages ISO_LANGUAGES = Arrays.stream(languages).collect(EnhancedCollectors.toUnmodifiableSortedMap( languageCode -> languageCode, languageCode -> new IsoLanguage(languageCode))); } private final String languageCode; private final String displayLanguage; /** * Convenience method for getting all of the IsoLanguage objects * * @return a stream of IsoLanguage objects */ public static Stream all() { return ISO_LANGUAGES.values().stream(); } /** * Provides a set of all Locale based language codes; supports convenience methods that use all * language codes * * @return Set of language codes */ public static Set allLanguageCodes() { return ALL_LANGUAGE_CODES; } /** * Provide the display language per Locale * * @param languageCode * Locale based language code * @return The display language */ public static Optional displayLanguageForLanguageCode(final String languageCode) { String displayLanguage = null; if (languageCode != null) { final IsoLanguage isoLanguage = ISO_LANGUAGES.get(languageCode); if (isoLanguage != null) { displayLanguage = isoLanguage.getDisplayLanguage(); } } return Optional.ofNullable(displayLanguage); } /** * Provides IsoLanguage for valid language code * * @param languageCode * 2 character language code, case sensitive (examples "en", "es") * @return Optional of valid language code, or empty if not recognized */ public static Optional forLanguageCode(final String languageCode) { return Optional.ofNullable(ISO_LANGUAGES.get(languageCode)); } /** * Check if display language is valid * * @param displayLanguage * Display language (example "United States") * @return whether this display language is valid */ public static boolean isValidDisplayLanguage(final String displayLanguage) { return displayLanguage != null && DISPLAY_LANGUAGES_SET.contains(displayLanguage); } /** * Check if language code is valid * * @param languageCode * 2 letter language code * @return whether this language code is valid */ public static boolean isValidLanguageCode(final String languageCode) { return languageCode != null && ISO_LANGUAGES.containsKey(languageCode); } private IsoLanguage(final String languageCode) { this.languageCode = languageCode; // Using a specific fixed Locale rather than system dependent Locale this.displayLanguage = new Locale(languageCode).getDisplayLanguage(LANGUAGE_LOCALE); } @Override public int compareTo(final IsoLanguage other) { if (this == other) { return 0; } return this.languageCode.compareTo(other.getLanguageCode()); } @Override public boolean equals(final Object other) { return other instanceof IsoLanguage && this.compareTo((IsoLanguage) other) == 0; } /** * Provides the display language for this IsoLanguage * * @return Display language string */ public String getDisplayLanguage() { return this.displayLanguage; } /** * Provides the language code for this IsoLanguage * * @return 2 character language code */ public String getLanguageCode() { return this.languageCode; } @Override public int hashCode() { return this.getLanguageCode().hashCode(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/ProtoSerializable.java ================================================ package org.openstreetmap.atlas.proto; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; /** * {@link ProtoSerializable} is a contract for types that would like to be serialized in protocol * buffer format. A type that implements this interface must be able to provide a valid adapter to * its owner. * * @author lcram */ public interface ProtoSerializable { /** * @return The adapter associated with this {@link ProtoSerializable} */ ProtoAdapter getProtoAdapter(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.proto.ProtoSerializable; /** * Any protobuf adapter class must conform to this interface. * * @author lcram */ public interface ProtoAdapter { /** * @param byteArray * The raw byte representation of the {@link ProtoSerializable} in protocol buffer * format * @return The object represented by the byte stream. */ ProtoSerializable deserialize(byte[] byteArray); /** * @param serializable * The object to serialize * @return The raw byte representation of the {@link ProtoSerializable} in protocol buffer * format. */ byte[] serialize(ProtoSerializable serializable); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoAtlasMetaDataAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.proto.ProtoAtlasMetaData; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.converters.ProtoTagListConverter; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link AtlasMetaData} and * {@link ProtoAtlasMetaData}. * * @author lcram */ public class ProtoAtlasMetaDataAdapter implements ProtoAdapter { private static final ProtoTagListConverter PROTOTAG_LIST_CONVERTER = new ProtoTagListConverter(); @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoAtlasMetaData protoAtlasMetaData = null; try { protoAtlasMetaData = ProtoAtlasMetaData.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } AtlasSize atlasSize = null; final boolean hasAllAtlasSizeFeatures = protoAtlasMetaData.hasEdgeNumber() && protoAtlasMetaData.hasNodeNumber() && protoAtlasMetaData.hasAreaNumber() && protoAtlasMetaData.hasLineNumber() && protoAtlasMetaData.hasPointNumber() && protoAtlasMetaData.hasRelationNumber(); if (hasAllAtlasSizeFeatures) { atlasSize = new AtlasSize(protoAtlasMetaData.getEdgeNumber(), protoAtlasMetaData.getNodeNumber(), protoAtlasMetaData.getAreaNumber(), protoAtlasMetaData.getLineNumber(), protoAtlasMetaData.getPointNumber(), protoAtlasMetaData.getRelationNumber()); } final boolean original = protoAtlasMetaData.getOriginal(); String codeVersion = null; if (protoAtlasMetaData.hasCodeVersion()) { codeVersion = protoAtlasMetaData.getCodeVersion(); } String dataVersion = null; if (protoAtlasMetaData.hasDataVersion()) { dataVersion = protoAtlasMetaData.getDataVersion(); } String country = null; if (protoAtlasMetaData.hasCountry()) { country = protoAtlasMetaData.getCountry(); } String shardName = null; if (protoAtlasMetaData.hasShardName()) { shardName = protoAtlasMetaData.getShardName(); } final Map tags = PROTOTAG_LIST_CONVERTER .convert(protoAtlasMetaData.getTagsList()); final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, original, codeVersion, dataVersion, country, shardName, tags); return atlasMetaData; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof AtlasMetaData)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final AtlasMetaData atlasMetaData = (AtlasMetaData) serializable; final ProtoAtlasMetaData.Builder protoMetaDataBuilder = ProtoAtlasMetaData.newBuilder(); if (atlasMetaData.getSize() != null) { protoMetaDataBuilder.setEdgeNumber(atlasMetaData.getSize().getEdgeNumber()); protoMetaDataBuilder.setNodeNumber(atlasMetaData.getSize().getNodeNumber()); protoMetaDataBuilder.setAreaNumber(atlasMetaData.getSize().getAreaNumber()); protoMetaDataBuilder.setLineNumber(atlasMetaData.getSize().getLineNumber()); protoMetaDataBuilder.setPointNumber(atlasMetaData.getSize().getPointNumber()); protoMetaDataBuilder.setRelationNumber(atlasMetaData.getSize().getRelationNumber()); } protoMetaDataBuilder.setOriginal(atlasMetaData.isOriginal()); atlasMetaData.getCodeVersion().ifPresent(value -> { protoMetaDataBuilder.setCodeVersion(value); }); atlasMetaData.getDataVersion().ifPresent(value -> { protoMetaDataBuilder.setDataVersion(value); }); atlasMetaData.getCountry().ifPresent(value -> { protoMetaDataBuilder.setCountry(value); }); atlasMetaData.getShardName().ifPresent(value -> { protoMetaDataBuilder.setShardName(value); }); if (atlasMetaData.getTags() != null) { protoMetaDataBuilder.addAllTags(ProtoAtlasMetaDataAdapter.PROTOTAG_LIST_CONVERTER .backwardConvert(atlasMetaData.getTags())); } return protoMetaDataBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoByteArrayOfArraysAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoByteArray; import org.openstreetmap.atlas.proto.ProtoByteArrayOfArrays; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.arrays.ByteArrayOfArrays; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link ByteArrayOfArrays} and * {@link ProtoByteArrayOfArrays}. * * @author lcram */ public class ProtoByteArrayOfArraysAdapter implements ProtoAdapter { @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoByteArrayOfArrays protoByteArrayOfArrays = null; try { protoByteArrayOfArrays = ProtoByteArrayOfArrays.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final ByteArrayOfArrays byteArrayOfArrays = new ByteArrayOfArrays( protoByteArrayOfArrays.getArraysCount(), protoByteArrayOfArrays.getArraysCount(), protoByteArrayOfArrays.getArraysCount()); if (protoByteArrayOfArrays.hasName()) { byteArrayOfArrays.setName(protoByteArrayOfArrays.getName()); } protoByteArrayOfArrays.getArraysList().stream().forEach(array -> { final byte[] items = array.getElements().toByteArray(); byteArrayOfArrays.add(items); }); return byteArrayOfArrays; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof ByteArrayOfArrays)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final ByteArrayOfArrays byteArrayOfArrays = (ByteArrayOfArrays) serializable; if (byteArrayOfArrays.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", byteArrayOfArrays.getClass().getName(), byteArrayOfArrays.size()); } final ProtoByteArrayOfArrays.Builder protoArraysBuilder = ProtoByteArrayOfArrays .newBuilder(); for (final byte[] elementArray : byteArrayOfArrays) { final ProtoByteArray.Builder elementArrayBuilder = ProtoByteArray.newBuilder(); if (elementArray == null) { throw new CoreException("{} cannot serialize arrays with null elements", this.getClass().getName()); } elementArrayBuilder.setElements(ByteString.copyFrom(elementArray)); protoArraysBuilder.addArrays(elementArrayBuilder); } if (byteArrayOfArrays.getName() != null) { protoArraysBuilder.setName(byteArrayOfArrays.getName()); } return protoArraysBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoIntegerArrayOfArraysAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoIntegerArrayOfArrays; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.converters.ProtoIntegerArrayOfArraysConverter; import org.openstreetmap.atlas.utilities.arrays.IntegerArrayOfArrays; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link IntegerArrayOfArrays} and * {@link ProtoIntegerArrayOfArrays}. * * @author lcram */ public class ProtoIntegerArrayOfArraysAdapter implements ProtoAdapter { private static final ProtoIntegerArrayOfArraysConverter CONVERTER = new ProtoIntegerArrayOfArraysConverter(); @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoIntegerArrayOfArrays protoIntegerArrayOfArrays = null; try { protoIntegerArrayOfArrays = ProtoIntegerArrayOfArrays.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final IntegerArrayOfArrays integerArrayOfArrays = CONVERTER .convert(protoIntegerArrayOfArrays); return integerArrayOfArrays; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof IntegerArrayOfArrays)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final IntegerArrayOfArrays integerArrayOfArrays = (IntegerArrayOfArrays) serializable; ProtoIntegerArrayOfArrays protoArrays = null; try { protoArrays = CONVERTER.backwardConvert(integerArrayOfArrays); } catch (final Exception exception) { throw new CoreException("Failed to serialize {}", integerArrayOfArrays.getClass().getName(), exception); } return protoArrays.toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoIntegerStringDictionaryAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import java.lang.reflect.Field; import java.util.HashMap; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoIntegerStringDictionary; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.compression.IntegerDictionary; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link IntegerDictionary} and * {@link ProtoIntegerStringDictionary}. This adapter will fail when attempting to serialize an * {@link IntegerDictionary} that is not type parameterized using {@link String}. Also note that * {@link ProtoIntegerStringDictionaryAdapter#deserialize(byte[])} will give back an * {@link IntegerDictionary} parameterized with {@link String}, no matter what type parameterization * was used by the parent {@link IntegerDictionary} the adapter belongs to. * * @author lcram */ public class ProtoIntegerStringDictionaryAdapter implements ProtoAdapter { /* * IntegerDictionary does not provide an interface for setting its subfields directly. This * class's implementation uses reflection to side-step the issue. */ /* * IntegerDictionary relies on null entries. Since protobuf cannot serialize Java's 'null' * value, we must represent 'null' in a non-null way. Note that if there are any tags that have * this sentinel as an actual key or value, the adapter will drop them when deserializing. */ private static final String NULL_SENTINEL_VALUE = "_+_NuLl{681FCC7E5213&E39443D7A0DE607A557|385D422B6092F_727517603F69880B5648}_||__"; @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoIntegerStringDictionary protoIntegerStringDictionary = null; try { protoIntegerStringDictionary = ProtoIntegerStringDictionary.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final IntegerDictionary dictionary = new IntegerDictionary<>(); final Integer currentIndex = protoIntegerStringDictionary.getCurrentIndex(); final List indexes = protoIntegerStringDictionary.getIndexesList(); final List words = protoIntegerStringDictionary.getWordsList(); final Map wordToIndex = new HashMap<>(); final Map indexToWord = new HashMap<>(); for (int index = 0; index < words.size(); index++) { String word = words.get(index); if (word.equals(NULL_SENTINEL_VALUE)) { word = null; } final Integer theIndex = indexes.get(index); wordToIndex.put(word, theIndex); indexToWord.put(theIndex, word); } Field wordToIndexField = null; Field indexToWordField = null; Field indexField = null; try { wordToIndexField = dictionary.getClass() .getDeclaredField(IntegerDictionary.FIELD_WORD_TO_INDEX); wordToIndexField.setAccessible(true); wordToIndexField.set(dictionary, wordToIndex); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", IntegerDictionary.FIELD_WORD_TO_INDEX, dictionary.getClass().getName(), exception); } try { indexToWordField = dictionary.getClass() .getDeclaredField(IntegerDictionary.FIELD_INDEX_TO_WORD); indexToWordField.setAccessible(true); indexToWordField.set(dictionary, indexToWord); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", IntegerDictionary.FIELD_INDEX_TO_WORD, dictionary.getClass().getName(), exception); } try { indexField = dictionary.getClass().getDeclaredField(IntegerDictionary.FIELD_INDEX); indexField.setAccessible(true); indexField.set(dictionary, currentIndex); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", IntegerDictionary.FIELD_INDEX, dictionary.getClass().getName(), exception); } return dictionary; } @Override @SuppressWarnings("unchecked") public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof IntegerDictionary)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final IntegerDictionary integerDictionary = (IntegerDictionary) serializable; if (integerDictionary.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", integerDictionary.getClass().getName(), integerDictionary.size()); } final ProtoIntegerStringDictionary.Builder protoDictionaryBuilder = ProtoIntegerStringDictionary .newBuilder(); Field indexToWordField = null; Map indexToWord = null; Field indexField = null; Integer index = -1; try { indexToWordField = integerDictionary.getClass() .getDeclaredField(IntegerDictionary.FIELD_INDEX_TO_WORD); indexToWordField.setAccessible(true); indexToWord = (Map) indexToWordField.get(integerDictionary); } catch (final Exception exception) { throw new CoreException("Unable to read field \"{}\" from {}", IntegerDictionary.FIELD_INDEX_TO_WORD, integerDictionary.getClass().getName(), exception); } /* * Wondering why we don't read the field wordToIndex? We don't need to, since it is * symmetric with the indexToWord field! We can populate the underlying proto arrays by * simply grabbing the keys and values from the indexToWord map. */ try { indexField = integerDictionary.getClass() .getDeclaredField(IntegerDictionary.FIELD_INDEX); indexField.setAccessible(true); index = (Integer) indexField.get(integerDictionary); } catch (final Exception exception) { throw new CoreException("Unable to read field \"{}\" from {}", IntegerDictionary.FIELD_INDEX, integerDictionary.getClass().getName(), exception); } try { for (final Integer key : indexToWord.keySet()) { String word = indexToWord.get(key); if (word == null) { word = NULL_SENTINEL_VALUE; } protoDictionaryBuilder.addIndexes(key); protoDictionaryBuilder.addWords(word); } } catch (final ClassCastException exception) { throw new CoreException( "This adapter is incompatible with type parametrization of the owning {}. Must be java.lang.String", serializable.getClass().getName(), exception); } protoDictionaryBuilder.setCurrentIndex(index); return protoDictionaryBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoLongArrayAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoLongArray; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.arrays.LongArray; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link LongArray} and * {@link ProtoLongArray}. * * @author lcram */ public class ProtoLongArrayAdapter implements ProtoAdapter { @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoLongArray protoLongArray = null; try { protoLongArray = ProtoLongArray.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final int elementCount = protoLongArray.getElementsCount(); final LongArray longArray = new LongArray(elementCount, elementCount, elementCount); protoLongArray.getElementsList().stream().forEach(element -> { longArray.add(element); }); if (protoLongArray.hasName()) { longArray.setName(protoLongArray.getName()); } return longArray; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof LongArray)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final LongArray longArray = (LongArray) serializable; if (longArray.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", longArray.getClass().getName(), longArray.size()); } final ProtoLongArray.Builder protoLongArrayBuilder = ProtoLongArray.newBuilder(); for (final long element : longArray) { protoLongArrayBuilder.addElements(element); } if (longArray.getName() != null) { protoLongArrayBuilder.setName(longArray.getName()); } return protoLongArrayBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoLongArrayOfArraysAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoLongArray; import org.openstreetmap.atlas.proto.ProtoLongArrayOfArrays; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.arrays.LongArrayOfArrays; import com.google.common.primitives.Longs; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link LongArrayOfArrays} and * {@link ProtoLongArrayOfArrays}. * * @author lcram */ public class ProtoLongArrayOfArraysAdapter implements ProtoAdapter { @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoLongArrayOfArrays protoLongArrayOfArrays = null; try { protoLongArrayOfArrays = ProtoLongArrayOfArrays.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final LongArrayOfArrays longArrayOfArrays = new LongArrayOfArrays( protoLongArrayOfArrays.getArraysCount(), protoLongArrayOfArrays.getArraysCount(), protoLongArrayOfArrays.getArraysCount()); if (protoLongArrayOfArrays.hasName()) { longArrayOfArrays.setName(protoLongArrayOfArrays.getName()); } protoLongArrayOfArrays.getArraysList().stream().forEach(array -> { final long[] items = Longs.toArray(array.getElementsList()); longArrayOfArrays.add(items); }); return longArrayOfArrays; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof LongArrayOfArrays)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final LongArrayOfArrays longArrayOfArrays = (LongArrayOfArrays) serializable; if (longArrayOfArrays.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", longArrayOfArrays.getClass().getName(), longArrayOfArrays.size()); } final ProtoLongArrayOfArrays.Builder protoArraysBuilder = ProtoLongArrayOfArrays .newBuilder(); for (final long[] elementArray : longArrayOfArrays) { final ProtoLongArray.Builder elementArrayBuilder = ProtoLongArray.newBuilder(); if (elementArray == null) { throw new CoreException("{} cannot serialize arrays with null elements", this.getClass().getName()); } for (int subIndex = 0; subIndex < elementArray.length; subIndex++) { elementArrayBuilder.addElements(elementArray[subIndex]); } protoArraysBuilder.addArrays(elementArrayBuilder); } if (longArrayOfArrays.getName() != null) { protoArraysBuilder.setName(longArrayOfArrays.getName()); } return protoArraysBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoLongToLongMapAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoLongArray; import org.openstreetmap.atlas.proto.ProtoLongToLongMap; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.maps.LongToLongMap; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link LongToLongMap} and * {@link ProtoLongToLongMap}. * * @author lcram */ public class ProtoLongToLongMapAdapter implements ProtoAdapter { /* * If the maximum size of the LongToLongMap we are reading is less than this value, just use * this value instead of the actual max size. This ensures that calculations in LargeMap using * DEFAULT_HASH_MODULO_RATIO do not fail do to integer division truncation. */ private static final int DEFAULT_MAX_SIZE = 1024; @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoLongToLongMap protoLongToLongMap = null; try { protoLongToLongMap = ProtoLongToLongMap.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } String deserializedName = null; if (protoLongToLongMap.hasName()) { deserializedName = protoLongToLongMap.getName(); } final int size = protoLongToLongMap.getKeys().getElementsCount() <= DEFAULT_MAX_SIZE ? DEFAULT_MAX_SIZE : protoLongToLongMap.getKeys().getElementsCount(); final LongToLongMap longToLongMap = new LongToLongMap(deserializedName, size, size, size, size, size, size); for (int index = 0; index < protoLongToLongMap.getKeys().getElementsCount(); index++) { longToLongMap.put(protoLongToLongMap.getKeys().getElements(index), protoLongToLongMap.getValues().getElements(index)); } return longToLongMap; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof LongToLongMap)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final LongToLongMap longMap = (LongToLongMap) serializable; if (longMap.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", longMap.getClass().getName(), longMap.size()); } final ProtoLongToLongMap.Builder protoMapBuilder = ProtoLongToLongMap.newBuilder(); final ProtoLongArray.Builder keysBuilder = ProtoLongArray.newBuilder(); final ProtoLongArray.Builder valuesBuilder = ProtoLongArray.newBuilder(); final Iterable iterable = () -> longMap.iterator(); for (final Long key : iterable) { final Long value = longMap.get(key); keysBuilder.addElements(key); valuesBuilder.addElements(value); } protoMapBuilder.setKeys(keysBuilder); protoMapBuilder.setValues(valuesBuilder); if (longMap.getName() != null) { protoMapBuilder.setName(longMap.getName()); } return protoMapBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoLongToLongMultiMapAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoLongArray; import org.openstreetmap.atlas.proto.ProtoLongToLongMultiMap; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.maps.LongToLongMultiMap; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link LongToLongMultiMap} and * {@link ProtoLongToLongMultiMap}. * * @author lcram */ public class ProtoLongToLongMultiMapAdapter implements ProtoAdapter { /* * If the maximum size of the LongToLongMap we are reading is less than this value, just use * this value instead of the actual max size. This ensures that calculations in LargeMap using * DEFAULT_HASH_MODULO_RATIO do not fail do to integer division truncation. */ private static final int DEFAULT_MAX_SIZE = 1024; @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoLongToLongMultiMap protoLongToLongMultiMap = null; try { protoLongToLongMultiMap = ProtoLongToLongMultiMap.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } String deserializedName = null; if (protoLongToLongMultiMap.hasName()) { deserializedName = protoLongToLongMultiMap.getName(); } final int size = protoLongToLongMultiMap.getKeys().getElementsCount() <= DEFAULT_MAX_SIZE ? DEFAULT_MAX_SIZE : protoLongToLongMultiMap.getKeys().getElementsCount(); final LongToLongMultiMap longToLongMultiMap = new LongToLongMultiMap(deserializedName, size, size, size, size, size, size); for (int index = 0; index < protoLongToLongMultiMap.getKeys().getElementsCount(); index++) { // First we get the proto format array associated with this key final ProtoLongArray protoLongArray = protoLongToLongMultiMap.getValuesList() .get(index); // Now we convert this proto format array to a List and then to a primitive long[] final long[] values = protoLongArray.getElementsList().stream() .mapToLong(Long::longValue).toArray(); longToLongMultiMap.put(protoLongToLongMultiMap.getKeys().getElements(index), values); } return longToLongMultiMap; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof LongToLongMultiMap)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final LongToLongMultiMap longMultiMap = (LongToLongMultiMap) serializable; if (longMultiMap.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", longMultiMap.getClass().getName(), longMultiMap.size()); } if (longMultiMap.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize provided {}, size {} too large", serializable.getClass().getName(), longMultiMap.size()); } final ProtoLongToLongMultiMap.Builder protoMapBuilder = ProtoLongToLongMultiMap .newBuilder(); final ProtoLongArray.Builder keysBuilder = ProtoLongArray.newBuilder(); final Iterable iterable = () -> longMultiMap.iterator(); for (final Long key : iterable) { final ProtoLongArray.Builder valuesBuilder = ProtoLongArray.newBuilder(); final long[] value = longMultiMap.get(key); if (value == null) { throw new CoreException("{} cannot serialize arrays with null elements", this.getClass().getName()); } keysBuilder.addElements(key); for (int index = 0; index < value.length; index++) { valuesBuilder.addElements(value[index]); } protoMapBuilder.addValues(valuesBuilder); } protoMapBuilder.setKeys(keysBuilder); if (longMultiMap.getName() != null) { protoMapBuilder.setName(longMultiMap.getName()); } return protoMapBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoPackedTagStoreAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import java.lang.reflect.Field; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.packed.PackedTagStore; import org.openstreetmap.atlas.proto.ProtoIntegerArrayOfArrays; import org.openstreetmap.atlas.proto.ProtoPackedTagStore; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.converters.ProtoIntegerArrayOfArraysConverter; import org.openstreetmap.atlas.utilities.arrays.IntegerArrayOfArrays; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link PackedTagStore} and * {@link ProtoPackedTagStore}. * * @author lcram */ public class ProtoPackedTagStoreAdapter implements ProtoAdapter { /* * PackedTagStore does not provide an interface for setting its arrays directly. This class's * implementation uses reflection to side-step the issue. Unlike the other PackedAtlas field * classes, there is no nice converter for the ProtoPackedTagStore (since it cannot be rebuilt * through its public API). */ private static final ProtoIntegerArrayOfArraysConverter CONVERTER = new ProtoIntegerArrayOfArraysConverter(); @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoPackedTagStore protoStore = null; try { protoStore = ProtoPackedTagStore.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final PackedTagStore store = new PackedTagStore(); final ProtoIntegerArrayOfArrays protoKeyArray = protoStore.getKeys(); final ProtoIntegerArrayOfArrays protoValueArray = protoStore.getValues(); final IntegerArrayOfArrays keyArray = CONVERTER.convert(protoKeyArray); final IntegerArrayOfArrays valueArray = CONVERTER.convert(protoValueArray); final Long index = protoStore.getIndex(); Field keyArrayField = null; Field valueArrayField = null; Field indexField = null; try { keyArrayField = store.getClass().getDeclaredField(PackedTagStore.FIELD_KEYS); keyArrayField.setAccessible(true); keyArrayField.set(store, keyArray); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", PackedTagStore.FIELD_KEYS, store.getClass().getName(), exception); } try { valueArrayField = store.getClass().getDeclaredField(PackedTagStore.FIELD_VALUES); valueArrayField.setAccessible(true); valueArrayField.set(store, valueArray); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", PackedTagStore.FIELD_VALUES, store.getClass().getName(), exception); } try { indexField = store.getClass().getDeclaredField(PackedTagStore.FIELD_INDEX); indexField.setAccessible(true); indexField.set(store, index); } catch (final Exception exception) { throw new CoreException("Unable to set field \"{}\" in {}", PackedTagStore.FIELD_INDEX, store.getClass().getName(), exception); } return store; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof PackedTagStore)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final PackedTagStore store = (PackedTagStore) serializable; final ProtoPackedTagStore.Builder protoTagStoreBuilder = ProtoPackedTagStore.newBuilder(); Field keyArrayField = null; IntegerArrayOfArrays keyArray = null; Field valueArrayField = null; IntegerArrayOfArrays valueArray = null; Field indexField = null; Long index = -1L; try { keyArrayField = store.getClass().getDeclaredField(PackedTagStore.FIELD_KEYS); keyArrayField.setAccessible(true); keyArray = (IntegerArrayOfArrays) keyArrayField.get(store); } catch (final Exception exception) { throw new CoreException("Unable to read field \"{}\" from {}", PackedTagStore.FIELD_KEYS, store.getClass().getName(), exception); } try { valueArrayField = store.getClass().getDeclaredField(PackedTagStore.FIELD_VALUES); valueArrayField.setAccessible(true); valueArray = (IntegerArrayOfArrays) valueArrayField.get(store); } catch (final Exception exception) { throw new CoreException("Unable to read field \"{}\" from {}", PackedTagStore.FIELD_VALUES, store.getClass().getName(), exception); } try { indexField = store.getClass().getDeclaredField(PackedTagStore.FIELD_INDEX); indexField.setAccessible(true); index = (Long) indexField.get(store); } catch (final Exception exception) { throw new CoreException("Unable to read field \"{}\" from {}", PackedTagStore.FIELD_INDEX, store.getClass().getName(), exception); } try { protoTagStoreBuilder.setKeys(CONVERTER.backwardConvert(keyArray)); } catch (final Exception exception) { throw new CoreException("Failed to serialize {}", keyArray.getClass().getName(), exception); } try { protoTagStoreBuilder.setValues(CONVERTER.backwardConvert(valueArray)); } catch (final Exception exception) { throw new CoreException("Failed to serialize {}", valueArray.getClass().getName(), exception); } protoTagStoreBuilder.setIndex(index); return protoTagStoreBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoPolyLineArrayAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.StringCompressedPolyLine; import org.openstreetmap.atlas.proto.ProtoPolyLineArray; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.arrays.PolyLineArray; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link PolyLineArray} and * {@link ProtoPolyLineArray}. * * @author lcram */ public class ProtoPolyLineArrayAdapter implements ProtoAdapter { @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoPolyLineArray protoPolyLineArray = null; try { protoPolyLineArray = ProtoPolyLineArray.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final int encodingsCount = protoPolyLineArray.getEncodingsCount(); final PolyLineArray polyLineArray = new PolyLineArray(encodingsCount, encodingsCount, encodingsCount); for (final ByteString encoding : protoPolyLineArray.getEncodingsList()) { polyLineArray.add(new StringCompressedPolyLine(encoding.toByteArray()).asPolyLine()); } if (protoPolyLineArray.hasName()) { polyLineArray.setName(protoPolyLineArray.getName()); } return polyLineArray; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof PolyLineArray)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final PolyLineArray polyLineArray = (PolyLineArray) serializable; if (polyLineArray.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", polyLineArray.getClass().getName(), polyLineArray.size()); } final ProtoPolyLineArray.Builder protoPolyLineArrayBuilder = ProtoPolyLineArray .newBuilder(); for (final PolyLine polyLine : polyLineArray) { if (polyLine == null) { throw new CoreException("{} cannot serialize arrays with null elements", this.getClass().getName()); } protoPolyLineArrayBuilder.addEncodings( ByteString.copyFrom(new StringCompressedPolyLine(polyLine).getEncoding())); } if (polyLineArray.getName() != null) { protoPolyLineArrayBuilder.setName(polyLineArray.getName()); } return protoPolyLineArrayBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/adapters/ProtoPolygonArrayAdapter.java ================================================ package org.openstreetmap.atlas.proto.adapters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.StringCompressedPolyLine; import org.openstreetmap.atlas.proto.ProtoPolygonArray; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.utilities.arrays.PolygonArray; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; /** * Implements the {@link ProtoAdapter} interface to connect {@link PolygonArray} and * {@link ProtoPolygonArray}. * * @author lcram */ public class ProtoPolygonArrayAdapter implements ProtoAdapter { @Override public ProtoSerializable deserialize(final byte[] byteArray) { ProtoPolygonArray protoPolygonArray = null; try { protoPolygonArray = ProtoPolygonArray.parseFrom(byteArray); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error encountered while parsing protobuf bytestream", exception); } final int encodingsCount = protoPolygonArray.getEncodingsCount(); final PolygonArray polygonArray = new PolygonArray(encodingsCount, encodingsCount, encodingsCount); for (final ByteString encoding : protoPolygonArray.getEncodingsList()) { polygonArray.add( new Polygon(new StringCompressedPolyLine(encoding.toByteArray()).asPolyLine())); } if (protoPolygonArray.hasName()) { polygonArray.setName(protoPolygonArray.getName()); } return polygonArray; } @Override public byte[] serialize(final ProtoSerializable serializable) { if (!(serializable instanceof PolygonArray)) { throw new CoreException( "Invalid ProtoSerializable type was provided to {}: cannot serialize {}", this.getClass().getName(), serializable.getClass().getName()); } final PolygonArray polygonArray = (PolygonArray) serializable; if (polygonArray.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot serialize {}, size too large ({})", polygonArray.getClass().getName(), polygonArray.size()); } final ProtoPolygonArray.Builder protoPolygonArrayBuilder = ProtoPolygonArray.newBuilder(); for (final Polygon polygon : polygonArray) { if (polygon == null) { throw new CoreException("{} cannot serialize arrays with null elements", this.getClass().getName()); } protoPolygonArrayBuilder.addEncodings( ByteString.copyFrom(new StringCompressedPolyLine(polygon).getEncoding())); } if (polygonArray.getName() != null) { protoPolygonArrayBuilder.setName(polygonArray.getName()); } return protoPolygonArrayBuilder.build().toByteArray(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/builder/ProtoAtlasBuilder.java ================================================ package org.openstreetmap.atlas.proto.builder; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.ReverseIdentifierFactory; import org.openstreetmap.atlas.proto.ProtoArea; import org.openstreetmap.atlas.proto.ProtoAtlas; import org.openstreetmap.atlas.proto.ProtoAtlasMetaData; import org.openstreetmap.atlas.proto.ProtoEdge; import org.openstreetmap.atlas.proto.ProtoLine; import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.proto.ProtoNode; import org.openstreetmap.atlas.proto.ProtoPoint; import org.openstreetmap.atlas.proto.ProtoRelation; import org.openstreetmap.atlas.proto.converters.ProtoLocationConverter; import org.openstreetmap.atlas.proto.converters.ProtoTagListConverter; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.protobuf.InvalidProtocolBufferException; /** * Build an {@link Atlas} from a ProtoAtlas formatted file, or write an {@link Atlas} to a * ProtoAtlas formatted file. ProtoAtlas is a naive encoding for {@link Atlas}es using protocol * buffers. A more compact and performant encoding can be obtained by using * {@link PackedAtlasSerializer}. * * @author lcram */ public class ProtoAtlasBuilder { private static final ProtoLocationConverter PROTOLOCATION_CONVERTER = new ProtoLocationConverter(); private static final ProtoTagListConverter PROTOTAG_LIST_CONVERTER = new ProtoTagListConverter(); private static final ReverseIdentifierFactory REVERSE_IDENTIFIER_FACTORY = new ReverseIdentifierFactory(); private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); /* * When performing serialization, metadata fields with 'null' values will be serialized as * "unknown". When deserializing, fields not present in the proto object will be interpreted * with this value. */ private static final String NULL_SENTINEL = "unknown"; /* * String that describes the data format of the atlas. This is used by the AtlasMetaData class * to record this version. */ public static final String PROTOATLAS_DATA_VERSION = "ProtoAtlas"; /** * Read a resource in naive ProtoAtlas format into a PackedAtlas. * * @param resource * the resource in naive ProtoAtlas format * @return the constructed PackedAtlas */ public PackedAtlas read(final Resource resource) { ProtoAtlas protoAtlas = null; // First, we need to construct the container object from the proto binary try { protoAtlas = ProtoAtlas.parseFrom(resource.readBytesAndClose()); } catch (final InvalidProtocolBufferException exception) { throw new CoreException("Error deserializing the ProtoAtlasContainer from {}", resource.getName(), exception); } // TODO make sure metadata read is consistent with what is written final ProtoAtlasMetaData protoAtlasMetaData = protoAtlas.getMetaData(); AtlasSize atlasSize = null; final boolean hasAllAtlasSizeFeatures = protoAtlasMetaData.hasEdgeNumber() && protoAtlasMetaData.hasNodeNumber() && protoAtlasMetaData.hasAreaNumber() && protoAtlasMetaData.hasLineNumber() && protoAtlasMetaData.hasPointNumber() && protoAtlasMetaData.hasRelationNumber(); if (hasAllAtlasSizeFeatures) { atlasSize = new AtlasSize(protoAtlasMetaData.getEdgeNumber(), protoAtlasMetaData.getNodeNumber(), protoAtlasMetaData.getAreaNumber(), protoAtlasMetaData.getLineNumber(), protoAtlasMetaData.getPointNumber(), protoAtlasMetaData.getRelationNumber()); } else { logger.warn("Could not deserialize AtlasSize, using defaults"); atlasSize = AtlasSize.DEFAULT; } final String codeVersion = protoAtlasMetaData.hasCodeVersion() ? protoAtlasMetaData.getCodeVersion() : NULL_SENTINEL; final String dataVersion = protoAtlasMetaData.hasDataVersion() ? protoAtlasMetaData.getDataVersion() : NULL_SENTINEL; final String country = protoAtlasMetaData.hasCountry() ? protoAtlasMetaData.getCountry() : NULL_SENTINEL; final String shardName = protoAtlasMetaData.hasShardName() ? protoAtlasMetaData.getShardName() : NULL_SENTINEL; final Map tags = PROTOTAG_LIST_CONVERTER .convert(protoAtlasMetaData.getTagsList()); final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, protoAtlasMetaData.getOriginal(), codeVersion, dataVersion, country, shardName, tags); final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(atlasSize) .withMetaData(atlasMetaData).withName(resource.getName()); // build the atlas features parsePoints(builder, protoAtlas.getPointsList()); parseLines(builder, protoAtlas.getLinesList()); parseAreas(builder, protoAtlas.getAreasList()); parseNodes(builder, protoAtlas.getNodesList()); parseEdges(builder, protoAtlas.getEdgesList()); parseRelations(builder, protoAtlas.getRelationsList()); return (PackedAtlas) builder.get(); } /** * Write an Atlas to a resource in the naive ProtoAtlas format. * * @param atlas * the Atlas to be written * @param resource * the resource to write into */ public void write(final Atlas atlas, final WritableResource resource) { final ProtoAtlas.Builder protoAtlasBuilder = ProtoAtlas.newBuilder(); // put the Atlas features into the ProtoAtlasBuilder writePointsToBuilder(atlas, protoAtlasBuilder); writeLinesToBuilder(atlas, protoAtlasBuilder); writeAreasToBuilder(atlas, protoAtlasBuilder); writeNodesToBuilder(atlas, protoAtlasBuilder); writeEdgesToBuilder(atlas, protoAtlasBuilder); writeRelationsToBuilder(atlas, protoAtlasBuilder); final AtlasMetaData atlasMetaData = atlas.metaData(); final ProtoAtlasMetaData.Builder protoMetaDataBuilder = ProtoAtlasMetaData.newBuilder(); if (atlasMetaData.getSize() != null) { protoMetaDataBuilder.setEdgeNumber(atlasMetaData.getSize().getEdgeNumber()); protoMetaDataBuilder.setNodeNumber(atlasMetaData.getSize().getNodeNumber()); protoMetaDataBuilder.setAreaNumber(atlasMetaData.getSize().getAreaNumber()); protoMetaDataBuilder.setLineNumber(atlasMetaData.getSize().getLineNumber()); protoMetaDataBuilder.setPointNumber(atlasMetaData.getSize().getPointNumber()); protoMetaDataBuilder.setRelationNumber(atlasMetaData.getSize().getRelationNumber()); } protoMetaDataBuilder.setOriginal(atlasMetaData.isOriginal()); atlasMetaData.getCodeVersion().ifPresent(protoMetaDataBuilder::setCodeVersion); atlasMetaData.getDataVersion().ifPresent(protoMetaDataBuilder::setDataVersion); atlasMetaData.getCountry().ifPresent(protoMetaDataBuilder::setCountry); atlasMetaData.getShardName().ifPresent(protoMetaDataBuilder::setShardName); if (atlasMetaData.getTags() != null) { protoMetaDataBuilder .addAllTags(PROTOTAG_LIST_CONVERTER.backwardConvert(atlasMetaData.getTags())); } protoAtlasBuilder.setMetaData(protoMetaDataBuilder); final ProtoAtlas protoAtlas = protoAtlasBuilder.build(); resource.writeAndClose(protoAtlas.toByteArray()); } private void parseAreas(final PackedAtlasBuilder builder, final List areas) { areas.forEach(protoArea -> { final long identifier = protoArea.getId(); final List shapePoints = protoArea.getShapePointsList().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) .collect(Collectors.toList()); final Polygon geometry = new Polygon(shapePoints); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoArea.getTagsList()); builder.addArea(identifier, geometry, tags); }); } private void parseEdges(final PackedAtlasBuilder builder, final List edges) { edges.forEach(protoEdge -> { final long identifier = protoEdge.getId(); final List shapePoints = protoEdge.getShapePointsList().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) .collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoEdge.getTagsList()); builder.addEdge(identifier, geometry, tags); }); } private void parseLines(final PackedAtlasBuilder builder, final List lines) { lines.forEach(protoLine -> { final long identifier = protoLine.getId(); final List shapePoints = protoLine.getShapePointsList().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) .collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoLine.getTagsList()); builder.addLine(identifier, geometry, tags); }); } private void parseNodes(final PackedAtlasBuilder builder, final List nodes) { nodes.forEach(protoNode -> { final long identifier = protoNode.getId(); final Longitude longitude = Longitude.dm7(protoNode.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoNode.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoNode.getTagsList()); builder.addNode(identifier, geometry, tags); }); } private void parsePoints(final PackedAtlasBuilder builder, final List points) { points.forEach(protoPoint -> { final long identifier = protoPoint.getId(); final Longitude longitude = Longitude.dm7(protoPoint.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoPoint.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoPoint.getTagsList()); builder.addPoint(identifier, geometry, tags); }); } private RelationBean parseRelationBean(final ProtoRelation protoRelation) { final RelationBean bean = new RelationBean(); protoRelation.getBeansList().forEach(protoRelationBean -> { final long memberId = protoRelationBean.getMemberId(); final String memberRole = protoRelationBean.getMemberRole(); final ItemType memberType = ItemType .forValue(protoRelationBean.getMemberType().getNumber()); bean.addItem(memberId, memberRole, memberType); }); return bean; } private void parseRelations(final PackedAtlasBuilder builder, final List relations) { relations.forEach(protoRelation -> { final long identifier = protoRelation.getId(); final RelationBean bean = parseRelationBean(protoRelation); final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoRelation.getTagsList()); builder.addRelation(identifier, ProtoAtlasBuilder.REVERSE_IDENTIFIER_FACTORY.getOsmIdentifier(identifier), bean, tags); }); } private void writeAreasToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfAreas = 0; for (final Area area : atlas.areas()) { final ProtoArea.Builder protoAreaBuilder = ProtoArea.newBuilder(); protoAreaBuilder.setId(area.getIdentifier()); final List protoLocations = area.asPolygon().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) .collect(Collectors.toList()); protoAreaBuilder.addAllShapePoints(protoLocations); final Map tags = area.getTags(); protoAreaBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfAreas++; protoAtlasBuilder.addAreas(protoAreaBuilder.build()); } protoAtlasBuilder.setNumberOfAreas(numberOfAreas); } private void writeEdgesToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfEdges = 0; for (final Edge edge : atlas.edges()) { final ProtoEdge.Builder protoEdgeBuilder = ProtoEdge.newBuilder(); protoEdgeBuilder.setId(edge.getIdentifier()); final List protoLocations = edge.asPolyLine().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) .collect(Collectors.toList()); protoEdgeBuilder.addAllShapePoints(protoLocations); final Map tags = edge.getTags(); protoEdgeBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfEdges++; protoAtlasBuilder.addEdges(protoEdgeBuilder.build()); } protoAtlasBuilder.setNumberOfEdges(numberOfEdges); } private void writeLinesToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfLines = 0; for (final Line line : atlas.lines()) { final ProtoLine.Builder protoLineBuilder = ProtoLine.newBuilder(); protoLineBuilder.setId(line.getIdentifier()); final List protoLocations = line.asPolyLine().stream() .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) .collect(Collectors.toList()); protoLineBuilder.addAllShapePoints(protoLocations); final Map tags = line.getTags(); protoLineBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfLines++; protoAtlasBuilder.addLines(protoLineBuilder.build()); } protoAtlasBuilder.setNumberOfLines(numberOfLines); } private void writeNodesToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfNodes = 0; for (final Node node : atlas.nodes()) { final ProtoNode.Builder protoNodeBuilder = ProtoNode.newBuilder(); protoNodeBuilder.setId(node.getIdentifier()); protoNodeBuilder.setLocation( ProtoAtlasBuilder.PROTOLOCATION_CONVERTER.backwardConvert(node.getLocation())); final Map tags = node.getTags(); protoNodeBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfNodes++; protoAtlasBuilder.addNodes(protoNodeBuilder.build()); } protoAtlasBuilder.setNumberOfNodes(numberOfNodes); } private void writePointsToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfPoints = 0; for (final Point point : atlas.points()) { final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); protoPointBuilder.setId(point.getIdentifier()); protoPointBuilder.setLocation( ProtoAtlasBuilder.PROTOLOCATION_CONVERTER.backwardConvert(point.getLocation())); final Map tags = point.getTags(); protoPointBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfPoints++; protoAtlasBuilder.addPoints(protoPointBuilder.build()); } protoAtlasBuilder.setNumberOfPoints(numberOfPoints); } private void writeRelationsToBuilder(final Atlas atlas, final ProtoAtlas.Builder protoAtlasBuilder) { long numberOfRelations = 0; for (final Relation relation : atlas.relations()) { final ProtoRelation.Builder protoRelationBuilder = ProtoRelation.newBuilder(); protoRelationBuilder.setId(relation.getIdentifier()); for (final RelationMember member : relation.members()) { final ProtoRelation.RelationBean.Builder beanBuilder = ProtoRelation.RelationBean .newBuilder(); beanBuilder.setMemberId(member.getEntity().getIdentifier()); beanBuilder.setMemberRole(member.getRole()); final ItemType type = ItemType.forEntity(member.getEntity()); beanBuilder.setMemberType(ProtoRelation.ProtoItemType.valueOf(type.getValue())); protoRelationBuilder.addBeans(beanBuilder.build()); } final Map tags = relation.getTags(); protoRelationBuilder .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfRelations++; protoAtlasBuilder.addRelations(protoRelationBuilder.build()); } protoAtlasBuilder.setNumberOfRelations(numberOfRelations); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/command/PackedToProtoAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.proto.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * Command for converting a serialized {@link PackedAtlas} to a serialized ProtoAtlas * * @author lcram */ public class PackedToProtoAtlasSubCommand implements FlexibleSubCommand { private static final String NAME = "packed-to-proto"; private static final String DESCRIPTION = "converts a packed atlas to a naive proto-based atlas"; private static final String PACKED_SWITCH_TEXT = "packed-atlas"; private static final String PROTO_SWITCH_TEXT = "proto-atlas"; private static final Switch INPUT_PARAMETER = new Switch<>(PACKED_SWITCH_TEXT, "Input atlas data in text atlas format", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_PARAMETER = new Switch<>(PROTO_SWITCH_TEXT, "Output atlas data path", Paths::get, Optionality.REQUIRED); private Path inputPath; private Path outputPath; @Override public int execute(final CommandMap map) { this.inputPath = (Path) map.get(INPUT_PARAMETER); this.outputPath = (Path) map.get(OUTPUT_PARAMETER); verifyArguments(); PackedAtlas.load(new File(this.inputPath.toFile())) .saveAsProto(new File(this.outputPath.toFile())); return 0; } @Override public String getDescription() { return DESCRIPTION; } @Override public String getName() { return NAME; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.println("-" + PACKED_SWITCH_TEXT + "=/input/path/to/packed/atlas"); writer.println("-" + PROTO_SWITCH_TEXT + "=/output/path/to/proto/atlas"); } private void verifyArguments() { if (!Files.isRegularFile(this.inputPath)) { throw new CoreException("{} is not a readable file", this.inputPath); } try { if (Files.isDirectory(this.outputPath)) { throw new CoreException("{} is a directory. Aborting", this.outputPath); } Files.createDirectories(this.outputPath.getParent()); } catch (final IOException exception) { throw new CoreException("Error when creating directories {}", this.outputPath.getParent(), exception); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/command/ProtoToPackedAtlasSubCommand.java ================================================ package org.openstreetmap.atlas.proto.command; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.proto.builder.ProtoAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; import org.openstreetmap.atlas.utilities.runtime.Command.Switch; import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; /** * Command for converting a serialized ProtoAtlas to a serialized {@link PackedAtlas} * * @author lcram */ public class ProtoToPackedAtlasSubCommand implements FlexibleSubCommand { private static final String NAME = "proto-to-packed"; private static final String DESCRIPTION = "converts a naive proto-based atlas to a packed atlas"; private static final String PROTO_SWITCH_TEXT = "proto-atlas"; private static final String PACKED_SWITCH_TEXT = "packed-atlas"; private static final Switch INPUT_PARAMETER = new Switch<>(PROTO_SWITCH_TEXT, "Input atlas data in text atlas format", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_PARAMETER = new Switch<>(PACKED_SWITCH_TEXT, "Output atlas data path", Paths::get, Optionality.REQUIRED); private Path inputPath; private Path outputPath; @Override public int execute(final CommandMap map) { this.inputPath = (Path) map.get(INPUT_PARAMETER); this.outputPath = (Path) map.get(OUTPUT_PARAMETER); verifyArguments(); new ProtoAtlasBuilder().read(new File(this.inputPath.toFile())) .save(new File(this.outputPath.toFile())); return 0; } @Override public String getDescription() { return DESCRIPTION; } @Override public String getName() { return NAME; } @Override public SwitchList switches() { return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); } @Override public void usage(final PrintStream writer) { writer.println("-" + PROTO_SWITCH_TEXT + "=/input/path/to/proto/atlas"); writer.println("-" + PACKED_SWITCH_TEXT + "=/output/path/to/packed/atlas"); } private void verifyArguments() { if (!Files.isRegularFile(this.inputPath)) { throw new CoreException("{} is not a readable file", this.inputPath); } try { if (Files.isDirectory(this.outputPath)) { throw new CoreException("{} is a directory. Aborting", this.outputPath); } Files.createDirectories(this.outputPath.getParent()); } catch (final IOException exception) { throw new CoreException("Error when creating directories {}", this.outputPath.getParent(), exception); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/converters/ProtoIntegerArrayOfArraysConverter.java ================================================ package org.openstreetmap.atlas.proto.converters; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoIntegerArray; import org.openstreetmap.atlas.proto.ProtoIntegerArrayOfArrays; import org.openstreetmap.atlas.utilities.arrays.IntegerArrayOfArrays; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import com.google.common.primitives.Ints; /** * Converts between the {@link IntegerArrayOfArrays} and its autogenerated protobuf counterpart. * * @author lcram */ public class ProtoIntegerArrayOfArraysConverter implements TwoWayConverter { @Override public ProtoIntegerArrayOfArrays backwardConvert(final IntegerArrayOfArrays array) { if (array.size() > Integer.MAX_VALUE) { throw new CoreException("Cannot convert {}, size too large ({})", array.getClass().getName(), array.size()); } final ProtoIntegerArrayOfArrays.Builder arraysBuilder = ProtoIntegerArrayOfArrays .newBuilder(); for (final int[] elementArray : array) { final ProtoIntegerArray.Builder subArrayBuilder = ProtoIntegerArray.newBuilder(); if (elementArray == null) { throw new CoreException("{} cannot convert arrays with null elements", this.getClass().getName()); } for (final int element : elementArray) { subArrayBuilder.addElements(element); } arraysBuilder.addArrays(subArrayBuilder); } if (array.getName() != null) { arraysBuilder.setName(array.getName()); } return arraysBuilder.build(); } @Override public IntegerArrayOfArrays convert(final ProtoIntegerArrayOfArrays protoArray) { final IntegerArrayOfArrays integerArrayOfArrays = new IntegerArrayOfArrays( protoArray.getArraysCount(), protoArray.getArraysCount(), protoArray.getArraysCount()); protoArray.getArraysList().stream().forEach(array -> { final int[] items = Ints.toArray(array.getElementsList()); integerArrayOfArrays.add(items); }); if (protoArray.hasName()) { integerArrayOfArrays.setName(protoArray.getName()); } return integerArrayOfArrays; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/converters/ProtoLocationConverter.java ================================================ package org.openstreetmap.atlas.proto.converters; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Converts back and forth between ProtoLocation and Location * * @author lcram */ public class ProtoLocationConverter implements TwoWayConverter { @Override public ProtoLocation backwardConvert(final Location location) { final ProtoLocation.Builder protoLocationBuilder = ProtoLocation.newBuilder(); protoLocationBuilder.setLatitude(Math.toIntExact(location.getLatitude().asDm7())); protoLocationBuilder.setLongitude(Math.toIntExact(location.getLongitude().asDm7())); return protoLocationBuilder.build(); } @Override public Location convert(final ProtoLocation protoLocation) { final Longitude longitude = Longitude.dm7(protoLocation.getLongitude()); final Latitude latitude = Latitude.dm7(protoLocation.getLatitude()); return new Location(latitude, longitude); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/proto/converters/ProtoTagListConverter.java ================================================ package org.openstreetmap.atlas.proto.converters; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoTag; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Converts back and forth between a List of ProtoTags and an OSM tag Map. * * @author lcram */ public class ProtoTagListConverter implements TwoWayConverter, Map> { private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverter.class); @Override public List backwardConvert(final Map osmTagMap) { final List protoTags = new ArrayList<>(); for (final Map.Entry entry : osmTagMap.entrySet()) { final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); final String keyText; final String valueText; if (entry.getKey() == null) { logger.warn("Conversion from OSM tagmap found null key, skipping..."); continue; } else { keyText = entry.getKey(); } if (entry.getValue() == null) { logger.warn("Conversion from OSM tagmap found null value for key {}", keyText); valueText = ""; } else { valueText = entry.getValue(); } tagBuilder.setKey(keyText); tagBuilder.setValue(valueText); protoTags.add(tagBuilder.build()); } return protoTags; } @Override public Map convert(final List protoTagList) { try { final Map result = Maps.hashMap(); for (final ProtoTag tag : protoTagList) { result.put(tag.getKey(), tag.getValue()); } return result; } catch (final Throwable error) { throw new CoreException("Unable to parse proto tags", error); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/CounterOutputStream.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.IOException; import java.io.OutputStream; import org.openstreetmap.atlas.exception.CoreException; /** * @author matthieun */ public class CounterOutputStream extends OutputStream { private long count = 0; private boolean closed = false; @Override public void close() { this.closed = true; } public long getCount() { if (!this.closed) { throw new CoreException("Cannot get the counts when the stream has not been closed."); } return this.count; } @Override public void write(final byte[] bite, final int offset, final int length) throws IOException { this.count += length - offset; } @Override public void write(final int value) throws IOException { this.count++; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/NotifyingIOUtils.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.EventListener; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; /** * Helpful class for notifying a caller on status of copying data from one stream to anonther * * @author cstaylor */ public class NotifyingIOUtils { /** * Notified on IO lifecycle events: *

    *
  • [1] started
    • *
  • [N] statusUpdate
    • *
  • [1] completed | failed
    • *
* * @author cstaylor */ public interface IOProgressListener extends EventListener { void completed(); void failed(IOException oops); void started(); void statusUpdate(long read); } private static final int EOF = -1; private static final int DEFAULT_BUFFER_SIZE = 4096; private Optional progessListener; private int bufferSize; public static long copy(final InputStream input, final OutputStream output, final int bufferSize, final IOProgressListener listener) throws IOException { return new NotifyingIOUtils().withListener(listener).withBufferSize(bufferSize).copy(input, output); } public static long copy(final InputStream input, final OutputStream output, final IOProgressListener listener) throws IOException { return new NotifyingIOUtils().withListener(listener).copy(input, output); } public NotifyingIOUtils() { this.progessListener = Optional.empty(); this.bufferSize = DEFAULT_BUFFER_SIZE; } public long copy(final InputStream input, final OutputStream output) throws IOException { final byte[] buffer = new byte[this.bufferSize]; long count = 0; int bufferReadCount = 0; try { this.progessListener.ifPresent(IOProgressListener::started); while (EOF != (bufferReadCount = input.read(buffer))) { output.write(buffer, 0, bufferReadCount); count += bufferReadCount; final long temporaryCount = count; this.progessListener.ifPresent(listener -> listener.statusUpdate(temporaryCount)); } this.progessListener.ifPresent(IOProgressListener::completed); } catch (final IOException oops) { this.progessListener.ifPresent(listener -> listener.failed(oops)); } return count; } public NotifyingIOUtils withBufferSize(final int bufferSize) { if (bufferSize <= 0) { throw new CoreException("Buffer size must be larger than zero: {}", bufferSize); } this.bufferSize = bufferSize; return this; } public NotifyingIOUtils withListener(final IOProgressListener listener) { this.progessListener = Optional.ofNullable(listener); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/SplittableInputStream.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; /** * From *

* IMPORTANT! Make sure to read from the original stream as well, and not just the split ones, * otherwise the buffer will blow up. *

* Additionally, this class has been made thread safe. * * @author matthieun */ public class SplittableInputStream extends InputStream { /** * Almost an input stream: The read-method takes an id. * * @author matthieun */ public static class MultiplexedSource { public static final int MINIMUM_BUFFER = 512; public static final int MAXIMUM_BUFFER = 10 * 512; // Underlying source private final InputStream source; // Read positions of each SplittableInputStream private final List readPositions = new ArrayList<>(); // Data to be read by the SplittableInputStreams private int[] buffer = new int[MINIMUM_BUFFER]; // Last valid position in buffer private int writePosition = 0; public MultiplexedSource(final InputStream source) { this.source = source; } /** * Read and advance position for given reader * * @param readerIdentifier * The reader identifier * @return The byte read * @throws IOException * In case the source read failed. */ public synchronized int read(final int readerIdentifier) throws IOException { // Enough data in buffer? if (this.readPositions.get(readerIdentifier) >= this.writePosition) { readJustBuffer(); this.buffer[this.writePosition++] = this.source.read(); } final int position = this.readPositions.get(readerIdentifier); final int byteValue = this.buffer[position]; if (byteValue != -1) { this.readPositions.set(readerIdentifier, position + 1); } return byteValue; } /** * Add a multiplexed reader * * @param splitIdentifier * The split identifier * @return The new reader identifier. */ protected int addSource(final int splitIdentifier) { this.readPositions .add(splitIdentifier == -1 ? 0 : this.readPositions.get(splitIdentifier)); return this.readPositions.size() - 1; } /** * Make room for more data (and drop data that has been read by all readers) */ private void readJustBuffer() { final int from = Collections.min(this.readPositions); final int whereTo = Collections.max(this.readPositions); final int newLength = Math.max((whereTo - from) * 2, MINIMUM_BUFFER); // System.out.println("New Length: " + newLength); if (newLength > MAXIMUM_BUFFER) { throw new CoreException("The SplittableInputStream buffer is blowing up. " + "Make sure all the split streams (including the original one " + "from which the splits originate!) are read at a similar pace."); } final int[] newBuf = new int[newLength]; System.arraycopy(this.buffer, from, newBuf, 0, whereTo - from); for (int i = 0; i < this.readPositions.size(); i++) { this.readPositions.set(i, this.readPositions.get(i) - from); } this.writePosition -= from; this.buffer = newBuf; } } // Non-root fields private final MultiplexedSource multiSource; private final int myId; /** * Public constructor: Used for first SplittableInputStream * * @param source * the source {@link InputStream} */ public SplittableInputStream(final InputStream source) { this.multiSource = new MultiplexedSource(source); this.myId = this.multiSource.addSource(-1); } /** * Private constructor: Used in split() * * @param multiSource * The multiplexed source * @param splitId * The split identifier */ private SplittableInputStream(final MultiplexedSource multiSource, final int splitId) { this.multiSource = multiSource; this.myId = multiSource.addSource(splitId); } @Override public int read() throws IOException { return this.multiSource.read(this.myId); } /** * @return a new InputStream that will read bytes from this position onwards. */ public SplittableInputStream split() { return new SplittableInputStream(this.multiSource, this.myId); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/Streams.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.Closeable; import java.io.Flushable; import org.openstreetmap.atlas.exception.CoreException; /** * Stream utility * * @author matthieun */ public final class Streams { /** * Safe close of a {@link Closeable} item. * * @param stream * The stream to close. */ public static void close(final Closeable stream) { if (stream == null) { return; } try { stream.close(); } catch (final Exception e) { throw new CoreException("Could not close stream", e); } } /** * Safe flush of a {@link Flushable} item. * * @param stream * The stream to flush. */ public static void flush(final Flushable stream) { if (stream == null) { return; } try { stream.flush(); } catch (final Exception e) { throw new CoreException("Could not flush stream", e); } } private Streams() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/StringInputStream.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.IOException; import java.io.InputStream; /** * An {@link InputStream} that reads from a {@link String} for convenience * * @author matthieun */ public class StringInputStream extends InputStream { private final String source; private int index; public StringInputStream(final String source) { this.source = source; this.index = 0; } @Override public int read() throws IOException { if (this.index < this.source.length()) { final int result = this.source.charAt(this.index); this.index++; return result; } return -1; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/StringOutputStream.java ================================================ package org.openstreetmap.atlas.streaming; import java.io.IOException; import java.io.OutputStream; /** * {@link OutputStream} backed by a {@link StringBuilder} * * @author matthieun */ public class StringOutputStream extends OutputStream { private final StringBuilder builder = new StringBuilder(); @Override public String toString() { return this.builder.toString(); } @Override public void write(final int byteValue) throws IOException { this.builder.append(String.valueOf((char) byteValue)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/compression/Compressor.java ================================================ package org.openstreetmap.atlas.streaming.compression; import java.io.IOException; import java.io.OutputStream; import java.util.zip.GZIPOutputStream; import org.openstreetmap.atlas.exception.CoreException; /** * Compressor for an {@link OutputStream} * * @author matthieun */ public interface Compressor { Compressor NONE = new Compressor() { @Override public OutputStream compress(final OutputStream out) { return out; } @Override public String toString() { return "NONE"; } }; Compressor GZIP = out -> { try { return new GZIPOutputStream(out); } catch (final IOException e) { throw new CoreException("Cannot create compressor.", e); } }; /** * @param out * The {@link OutputStream} to compress * @return The compressed {@link OutputStream} */ OutputStream compress(OutputStream out); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/compression/Decompressor.java ================================================ package org.openstreetmap.atlas.streaming.compression; import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPInputStream; import org.openstreetmap.atlas.exception.CoreException; /** * Decompressor for an {@link InputStream} * * @author matthieun */ public interface Decompressor { Decompressor NONE = new Decompressor() { @Override public InputStream decompress(final InputStream input) { return input; } @Override public String toString() { return "NONE"; } }; Decompressor GZIP = input -> { try { return new GZIPInputStream(input); } catch (final IOException e) { throw new CoreException("Cannot create decompressor.", e); } }; /** * @param input * The {@link InputStream} to decompress * @return The decompressed {@link InputStream} */ InputStream decompress(InputStream input); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/CsvLine.java ================================================ package org.openstreetmap.atlas.streaming.readers; import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.collections.StringList; import au.com.bytecode.opencsv.CSVParser; /** * A Csv line * * @author matthieun */ public final class CsvLine implements Iterable { private final CsvSchema schema; private final String[] items; public static CsvLine build(final CsvSchema schema, final String line) { final String[] items; try { items = new CSVParser().parseLine(line); } catch (final IOException e) { throw new CoreException("Could not parse line " + line, e); } if (items != null && schema != null) { if (items.length == schema.size()) { return new CsvLine(schema, items); } throw new CoreException("Line -- " + line + " -- has " + items.length + " arguments instead of " + schema.size() + " expected in the schema."); } throw new CoreException("line or schema was null."); } /** * Force use of factory method */ private CsvLine(final CsvSchema schema, final String[] line) { this.schema = schema; this.items = line; } /** * Get an item in this line * * @param index * The index at which the item is in the line * @return The item. */ public Object get(final int index) { verifyIndex(index); return this.schema.get(this, index); } @Override public Iterator iterator() { return new Iterator() { private int index = 0; @Override public boolean hasNext() { return this.index < CsvLine.this.items.length; } @Override public Object next() { if (!hasNext()) { throw new NoSuchElementException(); } return get(this.index++); } }; } @Override public String toString() { final StringList list = new StringList(() -> new Iterator() { private final Iterator objects = CsvLine.this.iterator(); @Override public boolean hasNext() { return this.objects.hasNext(); } @Override public String next() { final Object next = this.objects.next(); return "\"" + next.toString() + "\""; } }); return list.join(","); } protected String getValue(final int index) { verifyIndex(index); return this.items[index]; } private void verifyIndex(final int index) { if (index < 0 || index >= this.items.length) { throw new CoreException( "Item index " + index + " is out of range: 0 -> " + this.items.length); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/CsvReader.java ================================================ package org.openstreetmap.atlas.streaming.readers; import java.util.Iterator; import org.openstreetmap.atlas.streaming.resource.AbstractResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Reader for a Csv {@link Resource}. This is using openCsv * * @author tony * @author matthieun */ public class CsvReader implements Iterator { public static final Logger logger = LoggerFactory.getLogger(CsvReader.class); private final String comment; private final Iterator lineIterator; private final CsvSchema schema; public CsvReader(final CsvSchema schema, final AbstractResource resource) { this.lineIterator = resource.lines().iterator(); this.schema = schema; this.comment = "#"; } /** * @param resource * The resource to read * @param schema * The Csv schema to use * @param comment * The lines starting with this will be ignored. */ public CsvReader(final CsvSchema schema, final AbstractResource resource, final String comment) { this.lineIterator = resource.lines().iterator(); this.schema = schema; this.comment = comment; } @Override public boolean hasNext() { return this.lineIterator.hasNext(); } @Override public CsvLine next() { CsvLine result = null; String candidate; do { candidate = this.lineIterator.next(); if (candidate == null) { return null; } result = candidate.startsWith(this.comment) ? null : parse(candidate); } while (this.lineIterator.hasNext() && result == null); return result; } private CsvLine parse(final String candidate) { try { return CsvLine.build(this.schema, candidate); } catch (final Exception e) { logger.warn("Ignoring malformed line: -- {} --. Reason: {}", candidate, e.getMessage(), e); return null; } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/CsvSchema.java ================================================ package org.openstreetmap.atlas.streaming.readers; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.conversion.StringConverter; /** * Schema for a Csv {@link Resource}. Each item in a {@link CsvLine} is represented by its * {@link StringConverter}. All the {@link StringConverter}s must be supplied in proper order. * * @author matthieun */ public class CsvSchema { private final List> converters; public CsvSchema(final Iterable> converters) { this.converters = Iterables.asList(converters); } @SafeVarargs public CsvSchema(final StringConverter... converters) { this.converters = Iterables.asList(Iterables.iterable(converters)); } /** * Get an item * * @param line * The line to extract the item from * @param index * The index at which the item is in the line * @return The item */ protected Object get(final CsvLine line, final int index) { verifyIndex(index); return this.converters.get(index).convert(line.getValue(index)); } /** * The number of columns in this schema * * @return The number of columns in this schema */ protected int size() { return this.converters.size(); } private void verifyIndex(final int index) { if (index < 0 || index >= size()) { throw new CoreException( "Index " + index + " out of CsvSchema bounds of 0 -> " + size()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/GeoJsonReader.java ================================================ package org.openstreetmap.atlas.streaming.readers; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Iterator; import java.util.NoSuchElementException; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.readers.json.deserializers.LocatedDeserializer; import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated; import org.openstreetmap.atlas.streaming.resource.Resource; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; /** * Basic GeoJson stream reader. It reads only Point, LineString and Polygon. It ignores the rest, * including CRS. * * @author matthieun */ public class GeoJsonReader implements Iterator { private final InputStream input; private final JsonReader reader; private final Gson gson; public GeoJsonReader(final Resource source) { this.input = source.read(); this.reader = new JsonReader(new InputStreamReader(this.input)); final GsonBuilder builder = new GsonBuilder(); builder.registerTypeAdapter(Located.class, new LocatedDeserializer()); this.gson = builder.create(); try { this.reader.beginObject(); if (!"type".equals(this.reader.nextName()) || !"FeatureCollection".equals(this.reader.nextString())) { throw new CoreException("Malformed feature collection"); } String features = this.reader.nextName(); while (!"features".equals(features)) { // Read and skip the object (crs for example) this.reader.skipValue(); features = this.reader.nextName(); } this.reader.beginArray(); } catch (final Exception e) { Streams.close(this.input); throw new CoreException("Error reading GeoJson stream", e); } } @Override public boolean hasNext() { try { final boolean hasNext = this.reader.hasNext() && !this.reader.peek().equals(JsonToken.END_ARRAY); if (!hasNext) { Streams.close(this.input); } return hasNext; } catch (final IOException e) { Streams.close(this.input); throw new CoreException("Error reading GeoJson stream", e); } } @Override public PropertiesLocated next() { if (!hasNext()) { throw new NoSuchElementException(); } try { Located geometry = null; JsonObject properties = null; this.reader.beginObject(); while (this.reader.hasNext()) { final String name = this.reader.nextName(); if ("properties".equals(name)) { // Populate the properties properties = this.gson.fromJson(this.reader, JsonObject.class); } else if ("geometry".equals(name)) { geometry = this.gson.fromJson(this.reader, Located.class); } else { this.reader.skipValue(); } } this.reader.endObject(); if (geometry == null || properties == null) { throw new CoreException("Geometry or properties were null."); } return new PropertiesLocated(geometry, properties); } catch (final IOException e) { Streams.close(this.input); throw new CoreException("Error reading GeoJson stream", e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/csv/converters/CsvLineConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.csv.converters; import org.openstreetmap.atlas.streaming.readers.CsvLine; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * Converter from a {@link CsvLine} * * @param * The type to be converted * @author matthieun */ public interface CsvLineConverter extends Converter { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/converters/MultiPolyLineCoordinateConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.converters; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.JsonArray; import com.google.gson.JsonElement; /** * Used for {@link MultiPolyLine}s * * @author chunzhu */ public class MultiPolyLineCoordinateConverter implements Converter { private final PointCoordinateConverter coordinateConverter = new PointCoordinateConverter(); @Override public JsonArray convert(final MultiPolyLine object) { final JsonArray result = new JsonArray(); object.forEach(polyline -> { final JsonArray locations = new JsonArray(); polyline.forEach(location -> locations.add(this.coordinateConverter.convert(location))); result.add(locations); }); return result; } public Converter revert() { return jsonArray -> { final List polyLines = new ArrayList<>(); jsonArray.forEach(polyline -> { final JsonArray points = (JsonArray) polyline; final List locations = new ArrayList<>(); final Iterator locationIterator = points.iterator(); while (locationIterator.hasNext()) { final JsonArray coordinate = (JsonArray) locationIterator.next(); locations.add(this.coordinateConverter.revert().convert(coordinate)); } final PolyLine convertedPolyLine = new PolyLine(locations); polyLines.add(convertedPolyLine); }); if (polyLines.isEmpty()) { throw new CoreException("Cannot have an empty polyLines."); } return new MultiPolyLine(polyLines); }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/converters/MultiPolygonCoordinateConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.converters; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.Converter; import org.openstreetmap.atlas.utilities.maps.MultiMap; import com.google.gson.JsonArray; import com.google.gson.JsonElement; /** * Used for MultiPolygons * * @author brian_l_davis */ public class MultiPolygonCoordinateConverter implements Converter { private final PointCoordinateConverter coordinateConverter = new PointCoordinateConverter(); @Override public JsonArray convert(final MultiPolygon object) { final JsonArray result = new JsonArray(); object.outers().forEach(outerPolygon -> { final JsonArray outerLocations = new JsonArray(); outerPolygon.forEach( location -> outerLocations.add(this.coordinateConverter.convert(location))); result.add(outerLocations); object.innersOf(outerPolygon).forEach(innerPolygon -> { final JsonArray innerLocations = new JsonArray(); innerPolygon.forEach( location -> innerLocations.add(this.coordinateConverter.convert(location))); result.add(innerLocations); }); }); return result; } public Converter revert() { return jsonArray -> { final MultiMap outerToInners = new MultiMap<>(); jsonArray.forEach(polygon -> { final JsonArray linearRings = (JsonArray) polygon; final Iterator linearRingsIterator = linearRings.iterator(); if (linearRingsIterator.hasNext()) { final List polygons = new ArrayList<>(); while (linearRingsIterator.hasNext()) { final JsonArray coordinates = (JsonArray) linearRingsIterator.next(); final List locations = new ArrayList<>(); coordinates.forEach(coordinate -> { final JsonArray points = (JsonArray) coordinate; locations.add(this.coordinateConverter.revert().convert(points)); }); if (locations.isEmpty() || !locations.get(0).equals(locations.get(locations.size() - 1))) { throw new CoreException( "Invalidly formatted Geojson Polygon within Multipolygon"); } // in valid geojson the first point is repeated at the end and does not need // to be for our polygons locations.remove(locations.size() - 1); polygons.add(new Polygon(locations)); } if (polygons.isEmpty()) { throw new CoreException("Cannot have an empty MultiPolygon."); } final Polygon outer = polygons.remove(0); outerToInners.put(outer, new ArrayList<>()); polygons.forEach(inner -> { if (!outer.fullyGeometricallyEncloses(inner)) { throw new CoreException( "MultiPolygon inner ring not enclosed by outer ring."); } outerToInners.add(outer, inner); }); } }); if (outerToInners.isEmpty()) { throw new CoreException("Cannot have an empty MultiPolygon."); } return new MultiPolygon(outerToInners); }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/converters/PointCoordinateConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.converters; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.JsonArray; import com.google.gson.JsonPrimitive; /** * Converter that converts a {@link Location} object into its coordinate representation in the Geo * Json model: "coordinates" : [longitude, latitude] * * @author matthieun */ public class PointCoordinateConverter implements Converter { private static final double DATE_LINE_LONGITUDE_WEST = -180.0; private static final double DATE_LINE_LONGITUDE_EAST = 180.0; @Override public JsonArray convert(final Location location) { final JsonArray coordinates = new JsonArray(); coordinates.add(new JsonPrimitive(location.getLongitude().asDegrees())); coordinates.add(new JsonPrimitive(location.getLatitude().asDegrees())); return coordinates; } public Converter revert() { return jsonArray -> { final double latitude = jsonArray.get(1).getAsDouble(); double longitude = jsonArray.get(0).getAsDouble(); if (longitude == DATE_LINE_LONGITUDE_EAST) { longitude = DATE_LINE_LONGITUDE_WEST; } return new Location(Latitude.degrees(latitude), Longitude.degrees(longitude)); }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/converters/PolyLineCoordinateConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.converters; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.JsonArray; /** * Used for {@link PolyLine}s * * @author matthieun */ public class PolyLineCoordinateConverter implements Converter, JsonArray> { private final PointCoordinateConverter coordinateConverter = new PointCoordinateConverter(); @Override public JsonArray convert(final Iterable object) { final JsonArray result = new JsonArray(); object.forEach(location -> result.add(this.coordinateConverter.convert(location))); return result; } public Converter> revert() { return jsonArray -> { final List result = new ArrayList<>(); jsonArray.forEach(jsonElement -> { final JsonArray array = (JsonArray) jsonElement; result.add(this.coordinateConverter.revert().convert(array)); }); return result; }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/converters/PolygonCoordinateConverter.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.converters; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.utilities.conversion.Converter; import com.google.gson.JsonArray; /** * Used for {@link Polygon}s * * @author matthieun */ public class PolygonCoordinateConverter implements Converter, JsonArray> { private final PointCoordinateConverter coordinateConverter = new PointCoordinateConverter(); @Override public JsonArray convert(final Iterable object) { final JsonArray result = new JsonArray(); final JsonArray inner = new JsonArray(); object.forEach(location -> inner.add(this.coordinateConverter.convert(location))); result.add(inner); return result; } public Converter> revert() { return jsonArray -> { final List result = new ArrayList<>(); jsonArray.forEach(jsonElement -> { final JsonArray array = (JsonArray) jsonElement; array.forEach(element -> { final JsonArray array2 = (JsonArray) element; result.add(this.coordinateConverter.revert().convert(array2)); }); }); if (result.isEmpty() || !result.get(0).equals(result.get(result.size() - 1))) { throw new CoreException("Invalidly formatted Geojson Polygon"); } // in valid Geojson the first point is repeated at the end where it does not need to be // for our polygons result.remove(result.size() - 1); return result; }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/LocatedDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.geojson.GeoJsonType; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonParseException; /** * Deserializer that is clever about re-directing to the proper Located type. * * @author matthieun */ public class LocatedDeserializer implements JsonDeserializer { @Override public Located deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final GeoJsonType type = GeoJsonType.forJson(json.getAsJsonObject()); if (GeoJsonType.POINT == type) { return new LocationDeserializer().deserialize(json, typeOfT, context); } else if (GeoJsonType.LINESTRING == type) { return new PolyLineDeserializer().deserialize(json, typeOfT, context); } else if (GeoJsonType.POLYGON == type) { return new PolygonDeserializer().deserialize(json, typeOfT, context); } else if (GeoJsonType.MULTI_POLYGON == type) { return new MultiPolygonDeserializer().deserialize(json, typeOfT, context); } else if (GeoJsonType.MULTI_LINESTRING == type) { return new MultiPolyLineDeserializer().deserialize(json, typeOfT, context); } throw new CoreException("Unknown/unsupported geometry type: " + type); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/LocationDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.streaming.readers.json.converters.PointCoordinateConverter; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; /** * @author matthieun */ public class LocationDeserializer implements JsonDeserializer { private final PointCoordinateConverter coordinateConverter = new PointCoordinateConverter(); @Override public Location deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final JsonArray coordinates = (JsonArray) ((JsonObject) json).get("coordinates"); return this.coordinateConverter.revert().convert(coordinates); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/MultiPolyLineDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.MultiPolyLine; import org.openstreetmap.atlas.streaming.readers.json.converters.MultiPolyLineCoordinateConverter; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; /** * @author chunzhu */ public class MultiPolyLineDeserializer implements JsonDeserializer { private final MultiPolyLineCoordinateConverter multiMultiCoordinateConverter = new MultiPolyLineCoordinateConverter(); @Override public MultiPolyLine deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final JsonArray coordinates = (JsonArray) ((JsonObject) json).get("coordinates"); return this.multiMultiCoordinateConverter.revert().convert(coordinates); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/MultiPolygonDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.streaming.readers.json.converters.MultiPolygonCoordinateConverter; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; /** * @author brian_l_davis */ public class MultiPolygonDeserializer implements JsonDeserializer { private final MultiPolygonCoordinateConverter multiMultiCoordinateConverter = new MultiPolygonCoordinateConverter(); @Override public MultiPolygon deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final JsonArray coordinates = (JsonArray) ((JsonObject) json).get("coordinates"); return this.multiMultiCoordinateConverter.revert().convert(coordinates); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/PolyLineDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.streaming.readers.json.converters.PolyLineCoordinateConverter; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; /** * @author matthieun */ public class PolyLineDeserializer implements JsonDeserializer { private final PolyLineCoordinateConverter multiCoordinateConverter = new PolyLineCoordinateConverter(); @Override public PolyLine deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final JsonArray coordinates = (JsonArray) ((JsonObject) json).get("coordinates"); return new PolyLine(this.multiCoordinateConverter.revert().convert(coordinates)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/deserializers/PolygonDeserializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.deserializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.streaming.readers.json.converters.PolygonCoordinateConverter; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; /** * @author matthieun */ public class PolygonDeserializer implements JsonDeserializer { private final PolygonCoordinateConverter multiMultiCoordinateConverter = new PolygonCoordinateConverter(); /** * @deprecated Currently doesn't return accurate geometric representations for Geojson polygons * with inner rings use {@link MultiPolygonDeserializer} and check to see if the * returned multipolygon is a simple polygon **/ @Deprecated @Override public Polygon deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { final JsonArray coordinates = (JsonArray) ((JsonObject) json).get("coordinates"); return new Polygon(this.multiMultiCoordinateConverter.revert().convert(coordinates)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/serializers/LocationSerializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.serializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.streaming.readers.json.converters.PointCoordinateConverter; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; /** * {@link JsonSerializer} for a {@link Location} * * @author matthieun */ public class LocationSerializer implements JsonSerializer { @Override public JsonElement serialize(final Location location, final Type typeOfSrc, final JsonSerializationContext context) { final JsonObject result = new JsonObject(); result.add("type", new JsonPrimitive("Point")); result.add("coordinates", new PointCoordinateConverter().convert(location)); return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/serializers/MultiLocationSerializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.serializers; import java.lang.reflect.Type; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.streaming.readers.json.converters.PolyLineCoordinateConverter; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; /** * @param * The type of {@link PolyLine} to serialize. * @author matthieun */ public abstract class MultiLocationSerializer implements JsonSerializer { @Override public JsonElement serialize(final T polyLine, final Type typeOfSrc, final JsonSerializationContext context) { final JsonObject result = new JsonObject(); result.add("type", new JsonPrimitive(getType())); polyLine.forEach(location -> { }); result.add("coordinates", new PolyLineCoordinateConverter().convert(polyLine)); return result; } /** * @return The type of multi-location item */ protected abstract String getType(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/serializers/PolyLineSerializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.serializers; import org.openstreetmap.atlas.geography.PolyLine; import com.google.gson.JsonSerializer; /** * {@link JsonSerializer} for a {@link PolyLine} * * @author matthieun */ public class PolyLineSerializer extends MultiLocationSerializer { @Override protected String getType() { return "LineString"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/serializers/PolygonSerializer.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.serializers; import org.openstreetmap.atlas.geography.Polygon; import com.google.gson.JsonSerializer; /** * {@link JsonSerializer} for a {@link Polygon} * * @author matthieun */ public class PolygonSerializer extends MultiLocationSerializer { @Override protected String getType() { return "Polygon"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/readers/json/serializers/PropertiesLocated.java ================================================ package org.openstreetmap.atlas.streaming.readers.json.serializers; import org.openstreetmap.atlas.geography.Located; import com.google.gson.JsonObject; /** * Java bean containing a Located object and its properties from a GeoJson file * * @author matthieun */ public class PropertiesLocated { private final Located item; private final JsonObject properties; public PropertiesLocated(final Located item, final JsonObject properties) { this.item = item; this.properties = properties; } public Located getItem() { return this.item; } public JsonObject getProperties() { return this.properties; } @Override public String toString() { final StringBuilder result = new StringBuilder(); result.append(this.item.getClass().getSimpleName()); result.append(": "); result.append(this.item.toString()); result.append(" -- Properties: "); result.append(this.properties); return result.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/AbstractResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.compression.Decompressor; /** * Base implementation for a {@link Resource} * * @author matthieun */ public abstract class AbstractResource implements Resource { private Decompressor decompressor = Decompressor.NONE; private String name = null; public Decompressor getDecompressor() { return this.decompressor; } @Override public String getName() { if (this.name == null) { return Resource.super.getName(); } return this.name; } @Override public long length() { try (InputStream input = new BufferedInputStream(read())) { long length = 0; while (input.read() >= 0) { length++; } return length; } catch (final IOException e) { throw new CoreException("Resource Length can't be obtained.", e); } } @Override public final InputStream read() { if (this.decompressor == null) { return this.onRead(); } return this.decompressor.decompress(this.onRead()); } public void setDecompressor(final Decompressor decompressor) { this.decompressor = decompressor; } public void setName(final String name) { this.name = name; } @Override public String toString() { if (getName() != null) { return getName(); } return super.toString(); } /** * @return The raw stream from the resource */ protected abstract InputStream onRead(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/AbstractWritableResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.OutputStream; import org.openstreetmap.atlas.streaming.compression.Compressor; /** * Implementation for a {@link WritableResource} * * @author matthieun */ public abstract class AbstractWritableResource extends AbstractResource implements WritableResource { private Compressor compressor = Compressor.NONE; public Compressor getCompressor() { return this.compressor; } public void setCompressor(final Compressor compressor) { this.compressor = compressor; } @Override public final OutputStream write() { if (this.compressor == null) { return this.onWrite(); } return this.compressor.compress(this.onWrite()); } /** * @return The stream where to write raw bytes to the resource */ protected abstract OutputStream onWrite(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/ByteArrayResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.arrays.ByteArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link WritableResource} backed by a large {@link ByteArray}. * * @author matthieun */ public class ByteArrayResource extends AbstractWritableResource { private static final Logger logger = LoggerFactory.getLogger(ByteArrayResource.class); private static final int BYTE_MASK = 0xFF; private final ByteArray array; public ByteArrayResource() { this.array = new ByteArray(Long.MAX_VALUE); this.array.setName("ByteArrayResource"); } /** * @param initialSize * An initial size to help avoiding resizings. */ public ByteArrayResource(final long initialSize) { final int blockSize = (int) (initialSize <= Integer.MAX_VALUE ? initialSize : Integer.MAX_VALUE); this.array = new ByteArray(Long.MAX_VALUE, blockSize, Integer.MAX_VALUE); this.array.setName("ByteArrayResource"); } @Override public long length() { return this.array.size(); } public ByteArrayResource withName(final String name) { setName(name); this.array.setName(name); return this; } @Override protected InputStream onRead() { return new InputStream() { private long index = 0L; private boolean readOpen = true; @Override public void close() { this.readOpen = false; } @Override public int read() throws IOException { if (!this.readOpen) { throw new CoreException("Cannot read a closed stream"); } if (this.index >= ByteArrayResource.this.array.size()) { return -1; } return ByteArrayResource.this.array.get(this.index++) & BYTE_MASK; } }; } @Override protected OutputStream onWrite() { return new OutputStream() { private boolean writeOpen = true; @Override public void close() { this.writeOpen = false; logger.trace("Closed writer after {} bytes.", ByteArrayResource.this.array.size()); } @Override public void write(final int byteValue) throws IOException { if (!this.writeOpen) { throw new CoreException("Cannot write to a closed stream"); } ByteArrayResource.this.array.add((byte) (byteValue & BYTE_MASK)); } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/ClassResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Type; import java.util.Properties; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.Streams; import com.google.gson.Gson; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; /** * @author cuthbertm */ public class ClassResource extends AbstractResource { private final String resource; public ClassResource(final String resource) { this.resource = resource; } /** * Converts the resource into an object. It is expected that the format of the resource would be * JSON * * @param gson * A Gson object to convert the json resource to the specified type. This could have * special type adapters registered with it so that you can deserialize the resource * correctly * @param classType * The type of object to convert the resource json into * @param * The type T of the object being returned * @return The instantiated object */ public T getJSONResourceObject(final Gson gson, final Type classType) { InputStream input = null; try { input = onRead(); final JsonReader reader = new JsonReader(new InputStreamReader(onRead())); return gson.fromJson(reader, classType); } catch (final JsonIOException | JsonSyntaxException e) { throw new CoreException("Failed to load json file from resource file {}", this.resource, e); } finally { Streams.close(input); } } /** * Converts the resource into an object. It is expected that the format of the resource would be * JSON * * @param classType * The type of object to convert the resource json into * @param * The type T of the object being returned * @return The instantiated objecte */ public T getJSONResourceObject(final Type classType) { return this.getJSONResourceObject(new Gson(), classType); } /** * Given a resource file location will load from jar/classpath and return a Properties file. * Resource file should follow pattern for Properties file * * @return A properties file */ public Properties getResourceAsPropertyFile() { InputStream input = null; try { input = onRead(); final Properties props = new Properties(); props.load(input); return props; } catch (final IOException ioe) { throw new CoreException("Failed to load properties from resource file {}", this.resource, ioe); } finally { Streams.close(input); } } @Override protected InputStream onRead() { final ClassLoader classloader = Thread.currentThread().getContextClassLoader(); if (classloader == null) { throw new CoreException("Context Class loader could not be initialized."); } final InputStream input = classloader.getResourceAsStream(this.resource); if (input == null) { throw new CoreException("Resource, {}, not found.", this.resource); } return input; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/File.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileAttribute; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.compression.Compressor; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * File from a local file system as an {@link AbstractWritableResource}. * * @author matthieun * @author lcram */ public class File extends AbstractWritableResource implements Comparable { private static final Logger logger = LoggerFactory.getLogger(File.class); private static final String COULD_NOT_CREATE_DIRECTORIES_FOR_PATH = "Could not create directories for path {}"; private static final String JAVA_TEMPORARY_DIRECTORY; static { final String property = System.getProperty("java.io.tmpdir"); if (property == null) { throw new CoreException("Could not get System property java.io.tmpdir"); } else { JAVA_TEMPORARY_DIRECTORY = property; } } private final Path path; private String name = null; /** * Get a {@link TemporaryFile} in the default location using the default {@link FileSystem}. * This method is deprecated in favor of {@link File#temporary(FileSystem)}, which should allow * test cases much more control and flexibility over their environments. If your code needs to * use the default filesystem, then call {@link File#temporary(FileSystem)} with * {@link FileSystems#getDefault()}. However, your code would be more flexible if it were * file-system-agnostic and received the {@link FileSystem} as an input from calling code. See * the unit tests for the {@link File} class for details on how to use jimfs as an alternative * {@link FileSystem} in testing cases. * * @return the temporary file * @deprecated please use {@link File#temporary(FileSystem)} instead. */ @Deprecated public static TemporaryFile temporary() { return temporary(FileSystems.getDefault()); } /** * Get a {@link TemporaryFile} in the default location using the default {@link FileSystem}, but * with a specified prefix and suffix. This method is deprecated in favor of * {@link File#temporary(FileSystem)}, which should allow test cases much more control and * flexibility over their environments. If your code needs to use the default filesystem, then * call {@link File#temporary(FileSystem)} with {@link FileSystems#getDefault()}. However, your * code would be more flexible if it were file-system-agnostic and received the * {@link FileSystem} as an input from calling code. See the unit tests for the {@link File} * class for details on how to use jimfs as an alternative {@link FileSystem} in testing cases. * * @param prefix * the prefix to use * @param suffix * the suffix to use * @return the temporary file * @deprecated please use {@link File#temporary(FileSystem, String, String)} instead. */ @Deprecated public static TemporaryFile temporary(final String prefix, final String suffix) { return temporary(FileSystems.getDefault(), prefix, suffix); } /** * Create a temporary file at the system default temporary location. The name of the file will * be generated randomly and will have the suffix given by {@link FileSuffix#TEMPORARY}. * * @param fileSystem * the {@link FileSystem} to use for this {@link TemporaryFile}, use * {@link FileSystems#getDefault()} for the default local {@link FileSystem} * @return the file's {@link TemporaryFile} */ public static TemporaryFile temporary(final FileSystem fileSystem) { return temporary(fileSystem, null, FileSuffix.TEMPORARY.toString()); } /** * Create a temporary file with a given prefix and suffix at the system default temporary * location. The name of the file will be generated randomly and will be prefixed by the given * prefix. The file will have a suffix given by the suffix parameter. * * @param fileSystem * the {@link FileSystem} to use for this {@link TemporaryFile}, use * {@link FileSystems#getDefault()} for the default local {@link FileSystem} * @param prefix * a string prefix to use for the temporary file * @param suffix * a string suffix to use for the temporary file * @return the file's {@link TemporaryFile} */ public static TemporaryFile temporary(final FileSystem fileSystem, final String prefix, final String suffix) { final Path directory = fileSystem.getPath(JAVA_TEMPORARY_DIRECTORY); return temporary(directory, prefix, suffix); } /** * Create a temporary file with a given prefix and suffix at a given directory. The name of the * file will be generated randomly and will be prefixed by the given prefix. The file will have * a suffix given by the suffix parameter. * * @param directory * the parent directory for this temporary file * @param prefix * a string prefix to use for the temporary file * @param suffix * a string suffix to use for the temporary file * @return the file's {@link TemporaryFile} */ public static TemporaryFile temporary(final Path directory, final String prefix, final String suffix) { /* * Create the directory and all parents if it/they do not exist. Since Files.createTempFile * will not actually create parent directories, this step is necessary in some cases. */ new File(directory).mkdirs(); try { return new TemporaryFile(Files.createTempFile(directory, prefix, suffix)); } catch (final IOException exception) { throw new CoreException( "Unable to create a temporary file with prefix '{}' and suffix '{}' at {}", prefix, suffix, directory.toAbsolutePath(), exception); } } /** * Create a temporary folder at the system default temporary location. The name of the folder * will be generated randomly. * * @param fileSystem * the {@link FileSystem} to use for this {@link TemporaryFile}, use * {@link FileSystems#getDefault()} for the default local {@link FileSystem} * @return the folder's {@link TemporaryFile} */ public static TemporaryFile temporaryFolder(final FileSystem fileSystem) { return temporaryFolder(fileSystem, null); } /** * Create a temporary folder with a given prefix at the system default temporary location. The * name of the folder will be generated randomly and will be prefixed by the given prefix. * * @param fileSystem * the {@link FileSystem} to use for this {@link TemporaryFile}, use * * {@link FileSystems#getDefault()} for the default local {@link FileSystem} * @param prefix * a string prefix to use for the temporary folder * @return the folder's {@link TemporaryFile} */ public static TemporaryFile temporaryFolder(final FileSystem fileSystem, final String prefix) { final Path directory = fileSystem.getPath(JAVA_TEMPORARY_DIRECTORY); return temporaryFolder(directory, prefix); } /** * Get a {@link TemporaryFile} folder in the default location using the default * {@link FileSystem}. This method is deprecated in favor of * {@link File#temporaryFolder(FileSystem)}, which should allow test cases much more control and * flexibility over their environments. If your code needs to use the default filesystem, then * call {@link File#temporaryFolder(FileSystem)} with {@link FileSystems#getDefault()}. However, * your code would be more flexible if it were file-system-agnostic and received the * {@link FileSystem} as an input from calling code. See the unit tests for the {@link File} * class for details on how to use jimfs as an alternative {@link FileSystem} in testing cases. * * @return the temporary folder * @deprecated please use {@link File#temporaryFolder(FileSystem)} instead. */ @Deprecated public static TemporaryFile temporaryFolder() { return temporaryFolder(FileSystems.getDefault()); } /** * Create a temporary folder with a given prefix at a given directory. The name of the folder * will be generated randomly and will be prefixed by the given prefix. * * @param directory * the parent directory for this temporary folder * @param prefix * a string prefix to use for the temporary folder * @return the folder's {@link TemporaryFile} */ public static TemporaryFile temporaryFolder(final Path directory, final String prefix) { /* * Create the directory and all parents if it/they do not exist. Since Files.createTempFile * will not actually create parent directories, this step is necessary in some cases. */ new File(directory).mkdirs(); try { return new TemporaryFile(Files.createTempDirectory(directory, prefix)); } catch (final IOException exception) { throw new CoreException("Unable to create a temporary folder with prefix '{}' at {}", prefix, directory.toAbsolutePath(), exception); } } /** * Create a new {@link File} from a {@link java.io.File}, creating all necessary parent * directories. * * @param file * the {@link java.io.File} to use * @deprecated please use {@link File#File(Path)} */ @Deprecated public File(final java.io.File file) { this(file, true); } /** * Create a new {@link File} from a given {@link Path}, creating all necessary parent * directories. * * @param path * the {@link Path} to use */ public File(final Path path) { this(path, true); } /** * Create a new {@link File} from a given {@link Path}, optionally creating all necessary parent * directories. * * @param path * the {@link Path} to use * @param createParentDirectories * whether or not to create necessary parent directories */ public File(final Path path, final boolean createParentDirectories) { this.path = path; if (path.toAbsolutePath().toString().endsWith(FileSuffix.GZIP.toString())) { this.setCompressor(Compressor.GZIP); this.setDecompressor(Decompressor.GZIP); } if (this.path.getParent() != null && createParentDirectories) { /* * We must explicitly check the case where the direct parent already exists and is a * symbolic link. This is due to the fact that Files#createDirectories does not treat * symlinks to directories as directories themselves. So attempting to create parent * directories for the path "foo/bar/baz/bat" where baz is a symlink results in a * FileAlreadyExistsException on baz, even if baz points to a directory such that * "foo/bar/baz/bat" is still a valid path. This exception is not behaviour we want, * since all subsequent file operations will function normally. Rather, in this case, we * can simply refrain from even attempting to create the parent directories. */ if (Files.isSymbolicLink(this.path.getParent())) { logger.debug( "{} already existed and was a symbolic link, skipping parent directory creation", this.path.getParent()); } else { createDirectoriesForPath(this.path.getParent()); } } } /** * Create a new {@link File} from a {@link java.io.File}, optionally creating all necessary * parent directories. * * @param file * the {@link java.io.File} to use * @param createParentDirectories * whether or not to create necessary parent directories * @deprecated please use {@link File#File(Path, boolean)} */ @Deprecated public File(final java.io.File file, final boolean createParentDirectories) { this(file.toPath(), createParentDirectories); } /** * Create a new {@link File} from a given path string, using the given {@link FileSystem} to * resolve the path string into an actual {@link Path}. Automatically create all necessary * parent directories. * * @param pathString * the path string to the file * @param fileSystem * the {@link FileSystem} to use for resolution */ public File(final String pathString, final FileSystem fileSystem) { this(pathString, fileSystem, true); } /** * Create a new {@link File} from a given path string, using the given {@link FileSystem} to * resolve the path string into an actual {@link Path}. Optionally create all necessary parent * directories. * * @param pathString * the path string to the file * @param fileSystem * the {@link FileSystem} to use for resolution * @param createParentDirectories * whether or not to create necessary parent directories */ public File(final String pathString, final FileSystem fileSystem, final boolean createParentDirectories) { this(fileSystem.getPath(pathString), createParentDirectories); } /** * Create a new {@link File} from a given path string, using the default {@link FileSystem} * (i.e. {@link FileSystems#getDefault()}) to resolve the path string into an actual * {@link Path}. Automatically create all necessary parent directories. * * @param pathString * the path string to the file * @deprecated please use {@link File#File(String, FileSystem)} instead */ @Deprecated public File(final String pathString) { this(pathString, FileSystems.getDefault(), true); } /** * Create a new {@link File} from a given path string, using the default {@link FileSystem} * (i.e. {@link FileSystems#getDefault()}) to resolve the path string into an actual * {@link Path}. Optionally create all necessary parent directories. * * @param pathString * the path string to the file * @param createParentDirectories * whether or not to create necessary parent directories * @deprecated please use {@link File#File(String, FileSystem, boolean)} instead */ @Deprecated public File(final String pathString, final boolean createParentDirectories) { this(pathString, FileSystems.getDefault(), createParentDirectories); } /** * Get the basename of this {@link File} object as given by the underlying {@link FileSystem}. * * @return the basename of this {@link File}. */ public String basename() { return this.path.getFileName().toString(); } /** * Get a child {@link File} object for this file with a given name. Note this method will not * actually create the child file in the underlying {@link FileSystem}. However, it will attempt * to create all necessary directories such that "child" can resolve successfully when actually * created, whether that be through {@link File#writeAndClose(String)}, {@link File#mkdirs()}, * or something else. This method will fail if this {@link File} object is not resolvable to a * directory. * * @param name * the name of the desired child * @return the child {@link File} object */ public File child(final String name) { /* * We must explicitly check the case that this.path is a symbolic link. If it is, then we * want to skip the directory creation step. Why? Because the Files#createDirectories call * used in createDirectoriesForPath will fail when this.path is a symlink, even if it points * to a directory. However in this case, our Files.isDirectory() and Path#resolve() calls * later on will follow symlinks for us by default. So in the end, things resolve properly * in all cases. */ if (Files.isSymbolicLink(this.path)) { logger.debug("{} already existed and was a symbolic link, skipping directory creation", this.path); } else { createDirectoriesForPath(this.path); } if (!Files.isDirectory(this.path)) { throw new CoreException( "Cannot create the child of file {} since it did not resolve to a directory", this.path); } return new File(this.path.resolve(name)); } /** * This comparison uses the underlying {@link Path#compareTo(Path)} (Object)} implementation for * each {@link File} object. This means that the file contents are not necessarily considered. * If you are looking to compare files strictly using contents, you may consider implementing * your own {@link Comparator} that uses something like {@link File#all()}. * * @param other * the other {@link File} * @return the comparison value between the two {@link File}s */ @Override public int compareTo(final File other) { return this.path.compareTo(other.toPath()); } /** * Delete this {@link File} from the underlying {@link FileSystem}. This will fail to delete * non-empty directories. If you need that behaviour, see {@link File#deleteRecursively()}. */ public void delete() { try { Files.delete(this.path); } catch (final IOException exception) { throw new CoreException("Cannot delete file {}", this.path, exception); } } /** * Delete this {@link File} from the underlying {@link FileSystem}. If this {@link File} is a * non-empty directory, recursively delete all contents before deleting this {@link File}. */ public void deleteRecursively() { try { Files.walkFileTree(this.path, new SimpleFileVisitor() { @Override public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { Files.delete(dir); return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { Files.delete(file); return FileVisitResult.CONTINUE; } }); } catch (final IOException exception) { throw new CoreException("Cannot delete folder {} recursively", this.path, exception); } } /** * This equals check uses the underlying {@link Path#equals(Object)} implementation for each * {@link File} object. This means that the file contents are not necessarily considered. If you * are looking to compare files strictly using contents, you may consider comparing the values * of {@link File#all()}. * * @param other * the other {@link File} * @return if the two {@link File} objects are equal as specified by * {@link Path#equals(Object)}. */ @Override public boolean equals(final Object other) { if (other instanceof File) { return this.path.equals(((File) other).toPath()); } return false; } /** * Check if this {@link File} actually exists in the underlying {@link FileSystem}. * * @return true if this {@link File} exists */ public boolean exists() { return Files.exists(this.path); } /** * Return a string version of this {@link File}'s absolute path. * * @return the absolute path of this {@link File} as a string */ public String getAbsolutePathString() { return this.toAbsolutePath().toString(); } /** * Get a {@link java.io.File} version of this {@link File} resource. Note that this will force * the file to be resolved against the default filesystem. Please avoid using this method, it is * here for backwards-compatibility purposes. * * @return a {@link java.io.File} version of this {@link File} resource * @deprecated please move away from this method, and consider using either this class or * {@link Files} for file operations */ @Deprecated public java.io.File getFile() { return new java.io.File(this.path.toAbsolutePath().toString()); } /** * Get the name of this {@link File} {@link Resource}, as specified by the {@link Resource} * interface. By default, this implementation defers to the name of the {@link File} as given by * the underlying {@link FileSystem}. However, through the use of {@link File#withName(String)}, * it is possible this name may deviate from this {@link File}'s true basename. For the * ground-truth file system basename, try {@link File#basename()}. We recommend you use * {@link File#basename()} in all cases where the actual {@link File} name is what you care * about. * * @return the name of this {@link File} {@link Resource} */ @Override public String getName() { if (this.name != null) { return this.name; } return this.basename(); } /** * Return a string version of this {@link File}'s parent path. This may be absolute or relative, * depending on the status of the underlying {@link Path}. For the example file * File("foo/bar/baz"), this method will return "foo/bar". * * @return the parent path of this {@link File} as a string */ public String getParentPathString() { return this.toParentPath().toString(); } /** * Return a string version of this {@link File}'s path. This may be absolute or relative, * depending on the status of the underlying {@link Path}. * * @return the path of this {@link File} as a string */ public String getPathString() { return this.toPath().toString(); } @Override public int hashCode() { return this.path.hashCode(); } /** * Determine if this {@link File} is a directory. See * {@link Files#isDirectory(Path, LinkOption...)} for more details. * * @return if this {@link File} is a directory */ public boolean isDirectory() { return Files.isDirectory(this.path); } /** * Get the size in bytes of this {@link File}. This method defers to the implementation of the * underlying {@link FileSystem} to compute the file size. See {@link Files#size(Path)} for * details. * * @return the size in bytes of this {@link File} */ @Override public long length() { try { return Files.size(this.path); } catch (final IOException exception) { throw new CoreException("Could not get length of file {}", this.path, exception); } } /** * Get a {@link List} of all {@link File} objects at this {@link File}'s path, excluding any * directories. If this {@link File} is a regular file, then return a {@link List} containing * only itself. If this {@link File} does not exist, return an empty {@link List}. For example, * suppose this {@link File} is a directory called "foo/" containing "bar", "baz/", "bat/", and * "fred". This method would return the following {@link List}: [bar, fred]. * * @return a {@link List} of all file-only {@link File}s (not directories) at this * {@link File}'s path */ public List listFiles() { return listFiles(false); } /** * Get a {@link List} of all {@link File} objects at this {@link File}'s path, optionally * including or excluding any directories. If this {@link File} is a regular file, then return a * {@link List} containing only itself. If this {@link File} does not exist, return an empty * {@link List}. For example, suppose this {@link File} is a directory called "foo/" containing * "bar", "baz/", "bat/", and "fred". When includeDirectories is specified "true", this method * would return the following {@link List}: [bar, baz, bat, fred]. When includeDirectories is * specified "false", this method would instead return the following {@link List}: [bar, fred]. * * @param includeDirectories * whether or not to include directories in the list * @return a {@link List} of all {@link File}s at this {@link File}'s path */ public List listFiles(final boolean includeDirectories) { final List result = new ArrayList<>(); if (!this.exists()) { return result; } if (!this.isDirectory()) { result.add(this); return result; } try (Stream pathStream = Files.list(this.path)) { pathStream.forEach(path0 -> { final File file = new File(path0, false); if (file.isDirectory()) { if (includeDirectories) { result.add(file); } } else { result.add(file); } }); } catch (final IOException exception) { throw new CoreException("Could not list files at {}", this.path); } return result; } /** * If this {@link File} is a directory, recursively list all {@link File}s contained by this * {@link File}. Subdirectories will themselves be listed recursively. If this {@link File} is a * file, simply return a singleton list containing this {@link File}. If this {@link File} does * not exist, return an empty {@link List}. The 'includeDirectories' parameter will only control * whether directories are included in the final list, not whether subdirectories are expanded. * All subdirectories are always expanded. * * @param includeDirectories * whether or not to include directories in the list * @return the {@link List} of all {@link File}s contained in this {@link File} */ public List listFilesRecursively(final boolean includeDirectories) { final List result = new ArrayList<>(); if (!this.exists()) { return result; } if (!this.isDirectory()) { result.add(this); return result; } for (final File file : this.listFiles(true)) { final File listedFile = new File(file.toAbsolutePath()); if (listedFile.isDirectory()) { if (includeDirectories) { result.add(listedFile); } // We need to carry through the value of includeDirectories result.addAll(listedFile.listFilesRecursively(includeDirectories)); } else { result.add(listedFile); } } return result; } /** * If this {@link File} is a directory, recursively list all {@link File}s contained by this * {@link File}. Subdirectories will themselves be listed recursively. The final result will * contain only leaf {@link File} objects. If this {@link File} is a file, simply return a * singleton list containing this {@link File}. If this {@link File} does not exist, return an * empty {@link List}. * * @return the {@link List} of all {@link File}s contained in this {@link File} */ public List listFilesRecursively() { return listFilesRecursively(false); } /** * Create a directory for this {@link File}'s {@link Path} by creating all non-existent parent * directories first. See {@link Files#createDirectories(Path, FileAttribute[])}. For example, * if this {@link File} is specified by the {@link Path} "/foo/bar/baz", then this method will * first create "foo" if necessary, followed by "bar", and then finally "baz". */ public void mkdirs() { createDirectoriesForPath(this.path); } /** * Get a {@link File} object representing the parent directory of this {@link File}. * * @return a {@link File} object of the parent directory */ public File parent() { return new File(this.toAbsolutePath().getParent()); } /** * Get the absolute {@link Path} object associated with this {@link File}. * * @return the absolute {@link Path} for this {@link File} */ public Path toAbsolutePath() { if (this.path.isAbsolute()) { return this.path; } return this.path.toAbsolutePath(); } /** * Get the parent {@link Path} of the {@link Path} object associated with this {@link File}. * * @return the parent {@link Path} of this {@link File}. */ public Path toParentPath() { return this.parent().toPath(); } /** * Get the {@link Path} object associated with this {@link File}. * * @return the {@link Path} for this {@link File} */ public Path toPath() { return this.path; } @Override public String toString() { return this.getAbsolutePathString(); } /** * Utilize a given {@link Compressor} when writing to this {@link File}. * * @param compressor * the {@link Compressor} to use * @return this {@link File} for chaining */ public File withCompressor(final Compressor compressor) { this.setCompressor(compressor); return this; } /** * Utilize a given {@link Decompressor} when reading from this {@link File}. * * @param decompressor * the {@link Decompressor} to use * @return this {@link File} for chaining */ public File withDecompressor(final Decompressor decompressor) { this.setDecompressor(decompressor); return this; } /** * Update the name of this {@link File} {@link Resource}. Note that this simply changes the name * metadata in the {@link Resource} object, it *will not* change the actual name of the file in * the filesystem. Generally, users should avoid this method as it may cause confusion in * downstream code. * * @param name * the new name * @return this {@link File} for chaining */ public File withName(final String name) { this.name = name; return this; } @Override protected InputStream onRead() { try { return new BufferedInputStream(Files.newInputStream(this.path)); } catch (final IOException exception) { throw new CoreException("Cannot read file {}", this.path, exception); } } @Override protected OutputStream onWrite() { try { return new BufferedOutputStream(Files.newOutputStream(this.path)); } catch (final IOException exception) { throw new CoreException("Cannot write to file {}", this.path, exception); } } /** * Create a directory for a given {@link Path} by creating all non-existent parent directories * first. See documentation for {@link Files#createDirectories(Path, FileAttribute[])}. * * @param path * the given {@link Path} */ private void createDirectoriesForPath(final Path path) { try { Files.createDirectories(path); } catch (final IOException exception) { throw new CoreException(COULD_NOT_CREATE_DIRECTORIES_FOR_PATH, path, exception); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/FileSuffix.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.nio.file.Path; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; import com.google.common.base.Joiner; /** * @author mgostintsev */ public enum FileSuffix { ATLAS(".atlas"), CHANGESET(".cs"), CSV(".csv"), GEO_JSON(".geojson"), GZIP(".gz"), // extended csv EXTENDED(".ext"), JSON(".json"), NONE(""), OSMPBF(".osm.pbf"), OSM(".osm"), PBF(".pbf"), PROTOATLAS(".patlas"), TEMPORARY(".tmp"), TEXT(".txt"), ZIP(".zip"), WKT(".wkt"), WKB(".wkb"); private final String suffix; public static FileSuffix getEnum(final String value) { return suffixFor(value).orElseThrow( () -> new IllegalArgumentException("No file suffix found for " + value)); } public static Predicate pathFilter(final FileSuffix... listOfSuffixes) { final String suffix = Joiner.on("").join(listOfSuffixes); return path -> path.getFileName().toString().toLowerCase().endsWith(suffix); } public static Predicate resourceFilter(final FileSuffix... listOfSuffixes) { final String suffix = Joiner.on("").join(listOfSuffixes); return resource -> resource.getName().endsWith(suffix); } public static Optional suffixFor(final String value) { final String compareMe = value.toLowerCase(); return Stream.of(FileSuffix.values()) .filter(suffix -> compareMe.endsWith(suffix.toString())).findFirst(); } FileSuffix(final String suffix) { this.suffix = suffix; } public boolean matches(final Resource resource) { final Optional foundSuffix = suffixFor(resource.getName()); return foundSuffix.isPresent() && foundSuffix.get() == this; } @Override public String toString() { return this.suffix; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/GeoJsonFile.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.util.Iterator; import org.openstreetmap.atlas.streaming.readers.GeoJsonReader; import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated; /** * File that contains GeoJson data. * * @author matthieun */ public class GeoJsonFile extends File implements Iterable { public GeoJsonFile(final String path) { super(path); } @Override public Iterator iterator() { return new GeoJsonReader(this); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/InputStreamResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.InputStream; import java.util.function.Supplier; import org.openstreetmap.atlas.streaming.compression.Decompressor; /** * Readable resource from an {@link InputStream}. This is readable once only when using the * {@link InputStream} constructor, and readable as many times as needed when using the * {@link Supplier} constructor. * * @author matthieun */ public class InputStreamResource extends AbstractResource { private final Supplier inputStreamSupplier; /** * The supplier given to this constructor should return a new input stream at each invocation to * avoid any read-once gotchas. E.g. of a good supplier: () -> new FileInputStream("foo.txt") * * @param inputStreamSupplier * the stream supplier */ public InputStreamResource(final Supplier inputStreamSupplier) { this.inputStreamSupplier = inputStreamSupplier; } public InputStreamResource withDecompressor(final Decompressor decompressor) { this.setDecompressor(decompressor); return this; } public InputStreamResource withName(final String name) { this.setName(name); return this; } @Override protected InputStream onRead() { return this.inputStreamSupplier.get(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/InputStreamResourceCloseable.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.InputStream; import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Stream; /** * Create an InputStream resource that depends upon other resources that should not be closed until * the caller is done with the InputStream * * @author Taylor Smock */ public class InputStreamResourceCloseable extends InputStreamResource implements ResourceCloseable { private final AutoCloseable[] dependencies; /** * Create a new InputStreamWritableResource * * @param inputStreamSupplier * The InputStream to write to * @param dependencies * The dependencies that should be closed on finish */ @SafeVarargs public InputStreamResourceCloseable(final Supplier inputStreamSupplier, final AutoCloseable... dependencies) { super(inputStreamSupplier); this.dependencies = dependencies != null ? Stream.of(dependencies).filter(Objects::nonNull).toArray(AutoCloseable[]::new) : new AutoCloseable[0]; } @Override public AutoCloseable[] getDependencies() { return this.dependencies; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/LineFilteredResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.InputStream; import java.util.function.Predicate; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * A resource that has a built-in filter for the lines method. * * @author matthieun */ public class LineFilteredResource implements Resource { private final Resource source; private final Predicate lineFilter; public LineFilteredResource(final Resource source, final Predicate lineFilter) { this.source = source; this.lineFilter = lineFilter; } @Override public long length() { return this.source.length(); } @Override public Iterable lines() { return Iterables.stream(this.source.lines()).filter(this.lineFilter).collect(); } @Override public InputStream read() { return this.source.read(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/LineWriter.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.Charset; import org.openstreetmap.atlas.exception.CoreException; /** * Write lines to a {@link WritableResource} * * @author matthieun */ public class LineWriter extends BufferedWriter { private static final Charset CHARSET = Charset.forName("UTF-8"); private static final String LINE_SEPARATOR = System.getProperty("line.separator"); private final WritableResource writableResource; public LineWriter(final WritableResource writableResource) { super(new OutputStreamWriter(writableResource.write(), CHARSET)); this.writableResource = writableResource; } public void writeLine(final String line) { try { write(line); write(LINE_SEPARATOR); } catch (final IOException e) { throw new CoreException("Unable to write line to {}", this.writableResource, e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/OutputStreamWritableResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.InputStream; import java.io.OutputStream; import org.openstreetmap.atlas.exception.CoreException; /** * Writable resource from an {@link OutputStream} * * @author matthieun */ public class OutputStreamWritableResource extends AbstractWritableResource { private final OutputStream out; public OutputStreamWritableResource(final OutputStream out) { this.out = out; } @Override protected InputStream onRead() { throw new CoreException("This resource cannot be read."); } @Override protected OutputStream onWrite() { return this.out; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/OutputStreamWritableResourceCloseable.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.OutputStream; import java.util.Objects; import java.util.stream.Stream; /** * Create an OutputStream resource that depends upon other resources that should not be closed until * the caller is done with the OutputStream * * @author Taylor Smock */ public class OutputStreamWritableResourceCloseable extends OutputStreamWritableResource implements WritableResourceCloseable { private final AutoCloseable[] dependencies; /** * Create a new OutputStreamWritableResource * * @param out * The OutputStream to write to * @param dependencies * The dependencies that should be closed on finish */ public OutputStreamWritableResourceCloseable(final OutputStream out, final AutoCloseable... dependencies) { super(out); this.dependencies = dependencies != null ? Stream.of(dependencies).filter(Objects::nonNull).toArray(AutoCloseable[]::new) : new AutoCloseable[0]; } @Override public AutoCloseable[] getDependencies() { return this.dependencies; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/README.md ================================================ # FAQ/Notes on the recent [File](https://github.com/osmlab/atlas/blob/dev/src/main/java/org/openstreetmap/atlas/streaming/resource/File.java) refactor --- ### FAQ 1. **What has been updated?** Our [`org.openstreetmap.atlas.streaming.resource.File`](https://github.com/osmlab/atlas/blob/dev/src/main/java/org/openstreetmap/atlas/streaming/resource/File.java) implementation (hereafter referred to simply as `File`) has been refactored to entirely use [`FileSystem`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html) and [`java.nio.file`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/package-summary.html) operations. It no longer depends on [`java.io.File`](https://docs.oracle.com/javase/8/docs/api/java/io/File.html) for any functionality. Additionally, the class is now [heavily tested](https://github.com/osmlab/atlas/blob/dev/src/test/java/org/openstreetmap/atlas/streaming/resource/FileTest.java) using [`jimfs`](https://github.com/google/jimfs). For the curious, here are a few links discussing the differences between `java.io.File` and the file manipulation code in `java.nio.file`: https://docs.oracle.com/javase/tutorial/essential/io/legacy.html https://docs.oracle.com/javase/8/docs/api/java/nio/file/package-summary.html#package.description 2. **Why were these updates made?** Before these updates, any `File` object (through its dependence on `java.io.File`) was at the mercy of the default `FileSystem`. Code components which used `File` were difficult to test because they automatically relied on file system state external to the test environment. Even modular code that operated on caller-provided paths still had to be grounded in the default `FileSystem`. Because of this, unit/integration tests often relied on creating temporary files and directories at the system default temporary location. This approach is brittle, since testing code ultimately has no control over the state of the system default temporary location. This is especially true when running local tests, since a poorly crafted test could inadvertently be affected by file system state left from previous test runs. These changes make it much easier to decouple file manipulation code from a concrete file system. Unit and integration tests can now easily utilize a mock file system (like [`jimfs`](https://github.com/google/jimfs)) for the code they are testing. 3. **My code uses a method that is now deprecated. What should I do?** Your initial approach should be to decouple the file manipulation parts of your code from any specific `FileSystem` implementation. This allows clients of your code to provide whatever implementation they would like. See the below samples. Coupled code using the old `File` implementation: ```java public class MyOldCoupledComponent { /* * This "computeSomething" uses the old deprecated methods in File. * Thus, MyOldCoupledComponent automatically uses the default FileSystem, * and there is nothing MyOldClient can do to change this. */ public void computeSomething(String pathstring) { int i = 2 * 2; // Store result at user's home folder if (pathstring == null) { String userHome = System.getProperty("user.home"); File result = new File(userHome).child("result"); result.writeAndClose("result: " + i); } // Store result at alternate location else { File result = new File(pathstring).child("result"); result.writeAndClose("result: " + i); } } } public class MyOldClient { public void useComponent() { MyOldCoupledComponent c = new MyOldCoupledComponent(); // Store the result in the home folder c.computeSomething(null); // Also store it somewhere else c.computeSomething("/foo/bar") } } ``` Decoupled code using the new `File` implementation: ```java public class MyNewDecoupledComponent { /* * Below are two different ways you might refactor "computeSomething" * to utilize the new File implementation in a FileSystem agnostic way. */ // This version uses the pathstring like before, but also receives a // FileSystem argument to contextualize the pathstring public void computeSomethingString(FileSystem filesystem, String pathstring) { int i = 2 * 2; // Store result at user's home folder in the provided FileSystem if (pathstring == null) { String userHome = System.getProperty("user.home"); File result = new File(userHome, filesystem).child("result"); result.writeAndClose("result: " + i); } // Store result at alternate location else { File result = new File(pathstring, filesystem).child("result"); result.writeAndClose("result: " + i); } } // This version takes a Path, which is already implicitly tied to // a specific FileSystem. In this case, it would be the client's // responsibility to ensure the Path is tied to the FileSystem they want public void computeSomethingPath(Path path) { int i = 2 * 2; // Store result at user's home folder if (path == null) { // We cannot infer a FileSystem when no Path is given, // so just assume default String userHome = System.getProperty("user.home"); File result = new File(userHome, FileSystems.getDefault()).child("result"); result.writeAndClose("result: " + i); } // Store result at alternate location else { // No need to specify a FileSystem here, since it is already part of // the Path object File result = new File(path).child("result"); result.writeAndClose("result: " + i); } } } public class MyNewClient { /* * Notice that MyNewClient has complete control over the FileSystem used by * the File class in MyNewDecoupledComponent. This means that no matter how * complex the component is, we can always easily test it. It also means that * we can construct arbitrarily complex file system states in our unit tests, * making them that much more effective at identifying edge cases and brittle code. */ public void useComponent() { MyNewDecoupledComponent c = new MyNewDecoupledComponent(); // Store the result in the home folder on the default FileSystem c.computeSomethingString(FileSystems.getDefault(), null) c.computeSomethingPath(null); // Some ways to store it at "/foo/bar" in the default FileSystem c.computeSomethingString(FileSystems.getDefault(), "/foo/bar"); c.computeSomethingPath(Paths.get("/foo/bar")); c.computeSomethingPath(Path.of("/foo", "bar")); c.computeSomethingPath(FileSystems.getDefault().getPath("/foo/bar")); // Store it at "/baz/bat" in an alternate filesystem (jimfs) try (FileSystem jimfsFileSystem = Jimfs.newFileSystem(Configuration.osX())) { c.computeSomethingString(jimfsFileSystem, "/baz/bat"); c.computeSomethingPath(jimfsFileSystem.getPath("/baz/bat")) } catch (final IOException exception) { // ... } } } ``` 4. **I don't want to refactor my code. What can I do to remove the deprecated warnings?** If you really want to avoid refactoring your code like in the above sample, then you can just use the new version of each deprecated method that takes a `FileSystem`, and simply provide `FileSystems.getDefault()` as an argument. So `new File("/foo")` becomes `new File("/foo", FileSystems.getDefault())`, `File.temporary()` becomes `File.temporary(FileSystems.getDefault())`, etc. For code that uses the deprecated method `getFile`, you *will* have to change your code to perform file operations using the `java.nio.file` package. For more information on this, [see the documentation here.](https://docs.oracle.com/javase/tutorial/essential/io/legacy.html) ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/Resource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import org.apache.commons.io.IOUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.NotifyingIOUtils; import org.openstreetmap.atlas.streaming.NotifyingIOUtils.IOProgressListener; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.utilities.collections.StringList; /** * A resource that contains data and is readable by default. * * @author matthieun */ public interface Resource { int BYTE_MASK = 0x00FF; /** * @return The full contents of the {@link Resource} as a {@link String} */ default String all() { return new StringList(lines()).join("\n"); } /** * Copy all the contents of this {@link Resource} to a {@link WritableResource} * * @param output * The output {@link WritableResource} */ default void copyTo(final WritableResource output) { try (InputStream inputStream = read(); OutputStream outputStream = output.write()) { IOUtils.copy(inputStream, outputStream); } catch (final Exception e) { throw new CoreException("Unable to copy {} to {}.", this, output, e); } } /** * Copy all of the contents of {@link Resource} to a {@link WritableResource} while notifying a * progress listener * * @param output * The output {@link WritableResource} * @param listener * The notification {@link IOProgressListener} called as data is being copied */ default void copyTo(final WritableResource output, final NotifyingIOUtils.IOProgressListener listener) { try (InputStream inputStream = read(); OutputStream outputStream = output.write()) { NotifyingIOUtils.copy(inputStream, outputStream, listener); } catch (final Exception e) { throw new CoreException("Unable to copy {} to {}.", this, output, e); } } /** * @return The first line in this resource */ default String firstLine() { try (BufferedReader reader = reader()) { return reader.readLine(); } catch (final IOException e) { throw new CoreException("Unable to read first line of {}", this, e); } } /** * @return The optional name of the resource. */ default String getName() { return getClass().getName() + "@" + Integer.toHexString(hashCode()); } /** * @return True if the name of this resource lets believe that the resource contains Gzipped * contents. */ default boolean isGzipped() { return FileSuffix.GZIP.matches(this); } /** * @return The raw length on the {@link Resource} (as stored, regardless of compression). * Depending on the type of the {@link Resource}, it can be really slow if there is no * direct length meta data. */ long length(); /** * @return A String {@link Iterable} of all the lines in this resource. */ default Iterable lines() { final BufferedReader reader = reader(); return () -> { try { return new Iterator() { private String line = reader.readLine(); @Override public boolean hasNext() { return this.line != null; } @Override public String next() { if (!hasNext()) { throw new NoSuchElementException(); } final String result = this.line; populateNextLine(); return result; } private void populateNextLine() { try { this.line = reader.readLine(); } catch (final IOException e) { Streams.close(reader); throw new CoreException("Could not read resource line", e); } if (this.line == null) { Streams.close(reader); } } }; } catch (final IOException e) { Streams.close(reader); throw new CoreException("Could not read resource line", e); } }; } /** * @return A {@link StringList} of all the lines in this resource. */ default StringList linesList() { return new StringList(lines()); } /** * @return An {@link InputStream} streaming the contents of the resource */ InputStream read(); /** * @return The contents of the resource as a String */ default String readAndClose() { final StringList builder = new StringList(); lines().forEach(builder::add); return builder.join(System.lineSeparator()); } /** * @return The contents of the resource as a byte[] */ default byte[] readBytesAndClose() { final List byteContents = new ArrayList<>(); int kyte; try (InputStream input = new BufferedInputStream(read())) { while ((kyte = input.read()) >= 0) { byteContents.add((byte) (kyte & BYTE_MASK)); } } catch (final IOException e) { throw new CoreException("Unable to read the bytes from {}.", this, e); } final byte[] contents = new byte[byteContents.size()]; for (int index = 0; index < byteContents.size(); index++) { contents[index] = byteContents.get(index); } return contents; } /** * @return A {@link BufferedReader} on this resource. */ default BufferedReader reader() { return new BufferedReader(new InputStreamReader(this.read(), StandardCharsets.UTF_8)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/ResourceCloseable.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.util.ArrayList; import java.util.Collection; import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; /** * Use when the returned resource has some items that cannot be closed prior to finishing with this * resource. * * @author Taylor Smock */ public interface ResourceCloseable extends Resource, AutoCloseable { @Override default void close() throws Exception { final Collection exceptions = new ArrayList<>(); for (final AutoCloseable closeable : Stream.of(getDependencies()).filter(Objects::nonNull) .collect(Collectors.toList())) { try { closeable.close(); } catch (final Exception exception) { exceptions.add(exception); } } if (!exceptions.isEmpty()) { throw new CoreException( "{} exceptions thrown while closing {}, only showing the first thrown exception", exceptions.size(), this.getName(), exceptions.iterator().next()); } } /** * Get the {@link AutoCloseable} resources that this resource depends upon. * * @return An array of {@link AutoCloseable} dependencies */ default AutoCloseable[] getDependencies() { return new AutoCloseable[0]; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/StreamOfResourceStreams.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.SequenceInputStream; import java.util.Collections; import java.util.stream.Collectors; import java.util.stream.Stream; /** * Piggybacking on that classic IO class for concatenating the contents of several streams into one * * @author cstaylor */ public class StreamOfResourceStreams extends SequenceInputStream { public StreamOfResourceStreams(final Resource... resources) { super(Collections.enumeration(Stream.of(resources).map(resource -> resource.read()) .collect(Collectors.toList()))); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/StringResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.InputStream; import java.io.OutputStream; import java.util.function.Supplier; import org.openstreetmap.atlas.streaming.StringInputStream; import org.openstreetmap.atlas.streaming.StringOutputStream; /** * A {@link Resource} that relies on a {@link String} for convenience. * * @author matthieun */ public class StringResource extends AbstractWritableResource { private String source; private StringOutputStream out; /** * A {@link StringResource} for Writing, then Reading */ public StringResource() { this.out = new StringOutputStream(); } public StringResource(final AbstractResource source) { final StringBuilder builder = new StringBuilder(); source.lines().forEach(line -> { builder.append(line); builder.append("\n"); }); this.source = builder.toString(); } public StringResource(final Supplier sourceSupplier) { this(new InputStreamResource(sourceSupplier)); } public StringResource(final String source) { this.source = source; } @Override public long length() { if (this.source != null) { return this.source.length(); } return super.length(); } public StringResource withName(final String name) { this.setName(name); return this; } public String writtenString() { return this.out.toString(); } @Override protected InputStream onRead() { return new StringInputStream(this.source != null ? this.source : writtenString()); } @Override protected OutputStream onWrite() { if (this.out == null) { this.out = new StringOutputStream(); } return this.out; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/TemporaryFile.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.Closeable; import java.nio.file.FileSystem; import java.nio.file.Path; /** * A special type of {@link File} that automatically cleans up after itself. Please use * {@link File#temporary(FileSystem)}, {@link File#temporaryFolder(FileSystem)}, and the related * methods to obtain {@link TemporaryFile}s. You may take advantage of the automatic cleanup by * using it in a try-with-resources like so: * * 
 * try (TemporaryFile directory = File.temporaryFolder(fileSystem))
 * {
 *     File foo = directory.child("foo");
 *     // do something with foo
 * }
 * // the temporary directory is automatically deleted here
 *
* * @author matthieun * @author lcram */ public class TemporaryFile extends File implements Closeable { /** * Construct a new {@link TemporaryFile} with the given {@link Path}. All parent directories * will be automatically created. * * @param path * the path */ TemporaryFile(final Path path) { super(path); } /** * Clean up this {@link TemporaryFile}. If it is a regular file, simply delete it. If it is a * directory, we will delete it and all its contents recursively. */ @Override public void close() { if (this.isDirectory()) { this.deleteRecursively(); } else { this.delete(); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/WritableResource.java ================================================ package org.openstreetmap.atlas.streaming.resource; import java.io.BufferedOutputStream; import java.io.BufferedWriter; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.NotifyingIOUtils.IOProgressListener; import org.openstreetmap.atlas.streaming.writers.SafeBufferedWriter; /** * A resource that can be written to * * @author matthieun */ public interface WritableResource extends Resource { /** * Copy all the contents of another {@link Resource} to this {@link WritableResource} * * @param input * The input {@link Resource} */ default void copyFrom(final Resource input) { input.copyTo(this); } /** * Copy all the contents of another {@link Resource} to this {@link WritableResource} while * notifying a progress listener * * @param input * The input {@link Resource} * @param listener * The notification {@link IOProgressListener} called as data is being copied */ default void copyFrom(final Resource input, final IOProgressListener listener) { input.copyTo(this, listener); } /** * @return An {@link OutputStream} that streams data to this resource */ OutputStream write(); /** * Write to this resource and close it. * * @param value * The value to write. */ default void writeAndClose(final byte[] value) { try (BufferedOutputStream output = new BufferedOutputStream(write())) { output.write(value); } catch (final Exception e) { throw new CoreException("Could not write to {}", this, e); } } /** * Write to this resource and close it. * * @param value * The value to write. */ default void writeAndClose(final String value) { try (BufferedWriter writer = writer()) { writer.write(value); } catch (final Exception e) { throw new CoreException("Could not write to {}", this, e); } } /** * @return A {@link BufferedWriter} on this resource. */ default SafeBufferedWriter writer() { return new SafeBufferedWriter(new OutputStreamWriter(write(), StandardCharsets.UTF_8)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/WritableResourceCloseable.java ================================================ package org.openstreetmap.atlas.streaming.resource; /** * Use for WritableResources that have dependent AutoCloseable resources * * @author Taylor Smock */ public interface WritableResourceCloseable extends WritableResource, ResourceCloseable { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/http/DeleteResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.http; import java.net.URI; import org.apache.http.client.methods.HttpDelete; /** * Same usage as found in {@link HttpResource} * * @author cuthbertm */ public class DeleteResource extends HttpResource { public DeleteResource(final String uri) { this(URI.create(uri)); } public DeleteResource(final URI uri) { super(uri); setRequest(new HttpDelete(uri)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/http/GetResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.http; import java.net.URI; import org.apache.http.client.methods.HttpGet; /** * Same usage as found in {@link HttpResource} * * @author cuthbertm */ public class GetResource extends HttpResource { public GetResource(final String uri) { this(URI.create(uri)); } public GetResource(final URI uri) { super(uri); setRequest(new HttpGet(uri)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/http/HttpResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.http; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.Optional; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.HttpClientUtils; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.StringInputStream; import org.openstreetmap.atlas.streaming.resource.AbstractResource; import org.openstreetmap.atlas.streaming.resource.ResourceCloseable; /** * Base Http resource object that will handle most of the http request information. Sub classes * generally will set the type of request and possibly a couple of request-specific parameters. For * instance POST will require to post body data in the request. Example Usage: URI uri = new * URIBuilder("http://localhost:2020/path/to/location").build(); HttpResource post = new GetResource * // get t(uri, body); //read the response post.lines().foreach(System.out.println(x)); //get * status code int code = post.getStatusCode(); * * @author cuthbertm */ public abstract class HttpResource extends AbstractResource implements ResourceCloseable { private HttpRequestBase request; private final URI uri; private CloseableHttpResponse response = null; private Optional creds = Optional.empty(); private Optional proxy = Optional.empty(); private static HttpClientContext createBasicAuthCache(final HttpHost target, final HttpClientContext context) { // Create AuthCache instance final AuthCache authCache = new BasicAuthCache(); // Generate BASIC scheme object and add it to the local // auth cache final BasicScheme basicAuth = new BasicScheme(); authCache.put(target, basicAuth); // Add AuthCache to the execution context context.setAuthCache(authCache); return context; } public HttpResource(final String uri) { this(URI.create(uri)); } public HttpResource(final URI uri) { this.uri = uri; } @Override public void close() { HttpClientUtils.closeQuietly(this.response); } /** * If you want to execute the request, call this. All other attempts in an HttpResource will * first check to see if the response object has been retrieved. This will null out the response * object and execute it again. */ public void execute() { this.response = null; onRead(); } public Header[] getHeader(final String headerKey) { // make sure that a connection attempt has been made onRead(); return this.response.getHeaders(headerKey); } public HttpRequestBase getRequest() { return this.request; } public String getRequestBodyAsString() { // make sure that a connection attempt has been made final StringBuilder builder = new StringBuilder(); lines().forEach(builder::append); return builder.toString(); } // ------------------------------------// public CloseableHttpResponse getResponse() { // make sure that a connection attempt has been made onRead(); return this.response; } // ---- HTTP Helper Functions ---------// public int getStatusCode() { // make sure that a connection attempt has been made onRead(); return this.response.getStatusLine().getStatusCode(); } public URI getURI() { return this.uri; } public void setAuth(final String user, final String pass) { this.creds = Optional.of(new UsernamePasswordCredentials(user, pass)); } public void setHeader(final String name, final String value) { this.request.setHeader(name, value); } public void setProxy(final HttpHost proxy) { this.proxy = Optional.ofNullable(proxy); } public void setRequest(final HttpRequestBase request) { this.request = request; } @Override protected InputStream onRead() { try { if (this.response == null) { final HttpHost target = new HttpHost(this.uri.getHost(), this.uri.getPort(), this.uri.getScheme()); HttpClientContext context = HttpClientContext.create(); HttpClientBuilder clientBuilder = HttpClients.custom(); if (this.creds.isPresent()) { final CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials( new AuthScope(target.getHostName(), target.getPort()), this.creds.get()); clientBuilder = clientBuilder.setDefaultCredentialsProvider(credsProvider); } if (this.proxy.isPresent()) { clientBuilder = clientBuilder.setProxy(this.proxy.get()); } final CloseableHttpClient client = clientBuilder.build(); context = createBasicAuthCache(target, context); this.response = client.execute(target, this.request, context); } if (this.response.getEntity() == null) { return new StringInputStream(""); } return this.response.getEntity().getContent(); } catch (final IOException ioe) { throw new CoreException(ioe.getMessage(), ioe); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/http/PostResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.http; import java.io.UnsupportedEncodingException; import java.net.URI; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; /** * Usage: byte[] body = "{\"test\":\"test\"}".getBytes(); URI uri = new * URIBuilder("http://localhost:2020/path/to/location").build(); HttpResource post = new * PostResource(uri, body); //read the response post.lines().foreach(System.out.println(x)); // get * the status code int code = post.getStatusCode(); * * @author cuthbertm */ public class PostResource extends HttpResource { public PostResource(final String uri) { this(URI.create(uri)); } public PostResource(final URI uri) { super(uri); setRequest(new HttpPost(uri)); } public void setEntity(final HttpEntity entity) { ((HttpEntityEnclosingRequestBase) getRequest()).setEntity(entity); } public void setStringBody(final String body, final ContentType contentType) throws UnsupportedEncodingException { // using HttpEntityEnclosingRequestBase, so that resources like Put and Path can // extend from it. final HttpEntityEnclosingRequestBase base = (HttpEntityEnclosingRequestBase) getRequest(); base.addHeader(HttpHeaders.CONTENT_TYPE, contentType.getMimeType()); final StringEntity entity = new StringEntity(body, contentType); base.setEntity(entity); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/http/PutResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.http; import java.net.URI; import org.apache.http.client.methods.HttpPut; /** * Same usage as {@link PostResource} * * @author cuthbertm */ public class PutResource extends PostResource { public PutResource(final String uri) { this(URI.create(uri)); } public PutResource(final URI uri) { super(uri); setRequest(new HttpPut(uri)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/zip/ZipFileWritableResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.zip; import java.io.BufferedInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Zip wrapper for a {@link File} resource. This enables random lookups which are not available in * the case of a {@link ZipResource} * * @author matthieun */ public class ZipFileWritableResource extends ZipWritableResource { public ZipFileWritableResource(final File source) { super(source); } @Override public Iterable entries() { try { final ZipInputStream iteratorStream = getZipInputStream(); ZipEntry currentZipEntry; final List resources = new ArrayList<>(); while ((currentZipEntry = iteratorStream.getNextEntry()) != null) { final String entryName = currentZipEntry.getName(); resources.add(new InputStreamResource(() -> { final ZipInputStream zipStream = getZipInputStream(); seekStreamAheadToEntry(zipStream, entryName); return zipStream; }).withName(entryName)); } return resources; } catch (final IOException exception) { throw new CoreException("Could not read entries for {}", getFileSource().getAbsolutePathString(), exception); } } public Resource entryForName(final String name) { return new InputStreamResource(() -> { final ZipInputStream zipStream = getZipInputStream(); seekStreamAheadToEntry(zipStream, name); return zipStream; }).withName(name); } @Override public ZipFileWritableResource withWriteCompression(final boolean compression) { setWriteCompression(compression); return this; } protected File getFileSource() { return (File) getWritableSource(); } protected ZipInputStream getZipInputStream() { return new ZipInputStream(new BufferedInputStream(getFileSource().read())); } private void seekStreamAheadToEntry(final ZipInputStream streamToSeekThrough, final String entryNameToSeek) { if (entryNameToSeek == null) { throw new CoreException("Cannot seek for null entry name"); } String currentEntryName = null; while (!Objects.equals(currentEntryName, entryNameToSeek)) { try { final ZipEntry currentEntry = streamToSeekThrough.getNextEntry(); if (currentEntry == null) { break; } currentEntryName = currentEntry.getName(); } catch (final IOException exception) { throw new CoreException("IOException while getting next entry", exception); } } /* * If we make it here, then we didn't find the entry we were looking for - these aren't the * entries you're looking for. */ if (!Objects.equals(currentEntryName, entryNameToSeek)) { throw new CoreException("No such entry {} found in stream", entryNameToSeek); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/zip/ZipResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.zip; import java.io.BufferedInputStream; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.Streams; import org.openstreetmap.atlas.streaming.resource.AbstractResource; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Zipped {@link Resource} flavored wrapper using {@link ZipInputStream} * * @author matthieun */ public class ZipResource { /** * @author matthieun */ public static class ZipIterator implements Iterator, Closeable { private final Resource source; private final ZipInputStream input; private ZipEntry nextEntry = null; private boolean doneReading = true; public ZipIterator(final Resource source) { this.source = source; this.input = new ZipInputStream(new BufferedInputStream(this.source.read())); } @Override public void close() { Streams.close(this.input); } @Override public boolean hasNext() { try { if (this.nextEntry == null) { this.nextEntry = this.input.getNextEntry(); } if (this.nextEntry == null) { close(); } return this.nextEntry != null; } catch (final IOException e) { throw new CoreException("Unable to go to next Zip Entry!", e); } } @Override public Resource next() { if (!this.doneReading) { throw new CoreException(PREMATURE_READ_ERROR_MESSAGE); } if (hasNext()) { this.doneReading = false; final Resource result = new AbstractResource() { private final String name = ZipIterator.this.nextEntry.getName(); @Override public String getName() { return this.name; } @Override protected InputStream onRead() { return new InputStream() { @Override public void close() { // Trick to make sure the resource is read fully before moving // to the next one. ZipIterator.this.doneReading = true; } @Override public int read() throws IOException { return ZipIterator.this.input.read(); } @Override public int read(final byte[] buffer, final int offset, final int length) throws IOException { return ZipIterator.this.input.read(buffer, offset, length); } }; } }; this.nextEntry = null; return result; } else { throw new NoSuchElementException(); } } } public static final String PREMATURE_READ_ERROR_MESSAGE = "Cannot go to the next ZipEntry before the previous one has been fully read."; private final Resource source; public ZipResource(final Resource source) { this.source = source; } /** * @return The entries of the file as an {@link Iterable} of {@link Resource}s. This works only * if each resource is sequentially read. */ public Iterable entries() { return () -> new ZipIterator(getSource()); } public String getName() { return this.source.getName(); } @Override public String toString() { return this.getName(); } protected Resource getSource() { return this.source; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/resource/zip/ZipWritableResource.java ================================================ package org.openstreetmap.atlas.streaming.resource.zip; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.commons.compress.utils.IOUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Zipped {@link WritableResource} flavored wrapper using {@link ZipOutputStream} * * @author matthieun */ public class ZipWritableResource extends ZipResource { private static final int ZIP_MAXIMUM_COMPRESSION_LEVEL = 9; private boolean compression = true; public ZipWritableResource(final WritableResource source) { super(source); } /** * @param compression * True to compress the zip archive when writing. */ public void setWriteCompression(final boolean compression) { this.compression = compression; } /** * @param compression * True to compress the zip archive when writing. * @return The same object */ public ZipWritableResource withWriteCompression(final boolean compression) { setWriteCompression(compression); return this; } /** * Write a set of {@link Resource}s to the zip resource and close the stream. * * @param entries * The {@link Resource}s to write to the zip resource. */ public void writeAndClose(final Iterable entries) { try (ZipOutputStream output = new ZipOutputStream( new BufferedOutputStream(getWritableSource().write()))) { output.setLevel( this.compression ? ZIP_MAXIMUM_COMPRESSION_LEVEL : Deflater.NO_COMPRESSION); int counter = 0; for (final Resource resource : entries) { String name = resource.getName(); if (name == null) { name = "Entry " + counter; } final ZipEntry entry = new ZipEntry(name); output.putNextEntry(entry); try (InputStream input = resource.read()) { IOUtils.copy(input, output); counter++; } catch (final Exception e) { throw new CoreException("Unable to read resource {}", resource, e); } } } catch (final IOException e) { throw new CoreException("Unable to write next ZipEntry!", e); } } /** * Write a set of {@link Resource}s to the zip resource and close the stream. * * @param entries * The {@link Resource}s to write to the zip resource. */ public void writeAndClose(final Resource... entries) { writeAndClose(Iterables.asList(entries)); } protected WritableResource getWritableSource() { return (WritableResource) getSource(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/writers/JsonWriter.java ================================================ package org.openstreetmap.atlas.streaming.writers; import java.io.BufferedWriter; import java.io.Closeable; import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.WritableResource; import com.google.gson.JsonObject; /** * Write Json objects * * @author matthieun */ public class JsonWriter implements Closeable { private final BufferedWriter writer; public JsonWriter(final WritableResource resource) { this.writer = new BufferedWriter( new OutputStreamWriter(resource.write(), StandardCharsets.UTF_8)); } @Override public void close() { try { this.writer.close(); } catch (final IOException e) { throw new CoreException("Cannot close JsonWriter", e); } } public void flush() { try { this.writer.flush(); } catch (final IOException e) { close(); throw new CoreException("Cannot flush JsonWriter", e); } } public void write(final JsonObject object) { final String value = object.toString(); try { this.writer.write(value); } catch (final IOException e) { close(); throw new CoreException("Could not write String to JsonWriter", e); } } public void writeLine(final JsonObject object) { final String value = object.toString(); writeLine(value); } private void writeLine(final String stringValue) { try { this.writer.write(stringValue); this.writer.newLine(); } catch (final IOException e) { close(); throw new CoreException("Could not write String to JsonWriter", e); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/streaming/writers/SafeBufferedWriter.java ================================================ package org.openstreetmap.atlas.streaming.writers; import java.io.BufferedWriter; import java.io.IOException; import java.io.Writer; import org.openstreetmap.atlas.exception.CoreException; /** * @author matthieun */ public class SafeBufferedWriter extends BufferedWriter { public SafeBufferedWriter(final Writer out) { super(out); } @Override public void write(final String string) { try { super.write(string); } catch (final IOException e) { throw new CoreException("Could not write.", e); } } public void writeLine(final String line) { write(line + "\n"); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AbandonedAerowayTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM abandoned:aeroway tag * * @author cstaylor */ @Tag(with = { AerowayTag.class }, taginfo = "http://taginfo.openstreetmap.org/keys/abandoned%3Aaeroway#values", osm = "https://wiki.openstreetmap.org/wiki/Key:abandoned:") public interface AbandonedAerowayTag { @TagKey String KEY = "abandoned:aeroway"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AbandonedAmenityTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM abandoned:amenity tag * * @author cstaylor */ @Tag(with = { AmenityTag.class }, taginfo = "http://taginfo.openstreetmap.org/keys/abandoned%3Aamenity#values", osm = "https://wiki.openstreetmap.org/wiki/Key:abandoned:") public interface AbandonedAmenityTag { @TagKey String KEY = "abandoned:amenity"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AbandonedArtworkTypeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM abandoned:artwork_type tag * * @author cstaylor */ @Tag(with = { ArtworkTypeTag.class }, taginfo = "http://taginfo.openstreetmap.org/keys/abandoned%3Aartwork_type#values", osm = "https://wiki.openstreetmap.org/wiki/Key:abandoned:") public interface AbandonedArtworkTypeTag { @TagKey String KEY = "abandoned:artwork_type"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AccessTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM access tag * * @author robert_stack * @author matthieun * @author pmi */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/access#values", osm = "http://wiki.openstreetmap.org/wiki/Key:access") public enum AccessTag { YES, PRIVATE, NO, PERMISSIVE, AGRICULTURAL, USE_SIDEPATH, DELIVERY, DESIGNATED, DISMOUNT, DISCOURAGED, FORESTRY, DESTINATION, CUSTOMERS, PROHIBITED, PUBLIC, RESTRICTED, UNKNOWN; @TagKey public static final String KEY = "access"; private static final EnumSet PRIVATE_ACCESS = EnumSet.of(CUSTOMERS, NO, PRIVATE, RESTRICTED, PROHIBITED); public static boolean isNo(final Taggable taggable) { return Validators.isOfType(taggable, AccessTag.class, AccessTag.NO); } public static boolean isPrivate(final Taggable taggable) { final Optional access = Validators.from(AccessTag.class, taggable); return access.isPresent() && PRIVATE_ACCESS.contains(access.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressCityTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:city tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Acity#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressCityTag { @TagKey String KEY = "addr:city"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressCountryTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:country tag * * @author cstaylor */ @Tag(value = Validation.ISO2_COUNTRY, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Acountry#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressCountryTag { @TagKey String KEY = "addr:country"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressFlatsTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:flats tag * * @author mgostintsev */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/search?q=addr%3Aflats", osm = "http://wiki.openstreetmap.org/wiki/Key:addr:flats") public interface AddressFlatsTag { @TagKey String KEY = "addr:flats"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressFullTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:full tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Afull#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressFullTag { @TagKey String KEY = "addr:full"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressHousenameTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM addr:housename tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Ahousename#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressHousenameTag { @TagKey(KeyType.LOCALIZED) String KEY = "addr:housename"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressHousenumberTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:housenumber tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Ahousenumber#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressHousenumberTag { @TagKey String KEY = "addr:housenumber"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressInterpolationTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:interpolation tag * * @author pmi */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Ainterpolation#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressInterpolationTag { @TagKey String KEY = "addr:interpolation"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressPlaceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:place tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Aplace#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressPlaceTag { @TagKey String KEY = "addr:place"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressPostcodeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:postcode tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Apostcode#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressPostcodeTag { @TagKey String KEY = "addr:postcode"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressProvinceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:province tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Aprovince#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressProvinceTag { @TagKey String KEY = "addr:province"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressStateTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM addr:state tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Astate#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressStateTag { @TagKey String KEY = "addr:state"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AddressStreetTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM addr:street tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/addr%3Astreet#values", osm = "http://wiki.openstreetmap.org/wiki/Key:addr") public interface AddressStreetTag { @TagKey(KeyType.LOCALIZED) String KEY = "addr:street"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AdministrativeLevelTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.OrdinalExtractor; /** * OSM admin_level tag. Works in conjunction with the {@link BoundaryTag} administrative value. * * @author matthieun */ @Tag(value = Validation.ORDINAL, range = @Range(min = 1, max = 11), taginfo = "https://taginfo.openstreetmap.org/keys/admin_level#values", osm = "http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative#10_admin_level_values_for_specific_countries") public interface AdministrativeLevelTag { @TagKey String KEY = "admin_level"; /** * Validate the tag and return the Administrative Level tag as an integer between 1 and 11. * * @param taggable * The object to parse * @return the Administrative Level tag as an integer between 1 and 11, if validated. */ static Optional getAdministrativeLevel(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final OrdinalExtractor extractor = new OrdinalExtractor(); return extractor.validateAndExtract(tagValue.get(), AdministrativeLevelTag.class.getDeclaredAnnotation(Tag.class)); } return Optional.empty(); } static long maximumAdministrativeLevelValue() { final Tag tag = AdministrativeLevelTag.class.getDeclaredAnnotation(Tag.class); final Range range = tag.range(); return range.max(); } static long minimumAdministrativeLevelValue() { final Tag tag = AdministrativeLevelTag.class.getDeclaredAnnotation(Tag.class); final Range range = tag.range(); return range.min(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AerialWayTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM aerialway tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/aerialway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:aerialway") public enum AerialWayTag { CABLE_CAR, GONDOLA, CHAIR_LIFT, MIXED_LIFT, DRAG_LIFT, T_BAR, J_BAR, PLATTER, ROPE_TOW, MAGIC_CARPET, ZIP_LINE, PYLON, STATION; @TagKey public static final String KEY = "aerialway"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AerowayTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM aeroway tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/aeroway#values", osm = "http://wiki.openstreetmap.org/wiki/Aeroways") public enum AerowayTag { TAXIWAY, AERODROME, RUNWAY, HELIPAD, APRON, HANGAR, GATE, PARKING_POSITION, TERMINAL, HOLDING_POSITION, WINDSOCK, NAVIGATIONAID, MARKING; @TagKey public static final String KEY = "aeroway"; public static Optional get(final Taggable taggable) { return Validators.from(AerowayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AmenityTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM amenity tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/amenity#values", osm = "http://wiki.openstreetmap.org/wiki/Key:amenity") public enum AmenityTag { PARKING, PARKING_ENTRANCE, MOTORCYCLE_PARKING, PLACE_OF_WORSHIP, SCHOOL, BENCH, RESTAURANT, FUEL, CAFE, FAST_FOOD, BANK, POST_BOX, GRAVE_YARD, KINDERGARTEN, RECYCLING, PHARMACY, WASTE_BASKET, BICYCLE_PARKING, TOILETS, HOSPITAL, SHELTER, POST_OFFICE, PUB, DRINKING_WATER, PUBLIC_BUILDING, TELEPHONE, ATM, BAR, POLICE, FIRE_STATION, TOWNHALL, HUNTING_STAND, PARKING_SPACE, VENDING_MACHINE, FOUNTAIN, LIBRARY, DOCTORS, SWIMMING_POOL, SOCIAL_FACILITY, UNIVERSITY, BICYCLE_RENTAL, EMERGENCY_PHONE, WASTE_DISPOSAL, FESTIVAL_GROUNDS, COLLEGE, COMMUNITY_CENTRE, COMMUNITY_CENTER, MARKETPLACE, FERRY_TERMINAL; @TagKey public static final String KEY = "amenity"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AreaTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM area tag * * @author ihillberg */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/area#values", osm = "http://wiki.openstreetmap.org/wiki/Key:area") public enum AreaTag { YES, NO; @TagKey public static final String KEY = "area"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ArtworkTypeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM artwork_type tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/artwork_type#values", osm = "http://wiki.openstreetmap.org/wiki/Key:artwork_type") public enum ArtworkTypeTag { SCULPTURE, STATUE, MURAL, ARCHITECTURE, STONE, PAINTING, BUST, INSTALLATION, MOSAIC; @TagKey public static final String KEY = "artwork_type"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/AtlasTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; /** * Atlas created tags * * @author tony */ public final class AtlasTag { public static final Set TAGS_FROM_OSM = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(LastEditTimeTag.KEY, LastEditUserIdentifierTag.KEY, LastEditUserNameTag.KEY, LastEditVersionTag.KEY, LastEditChangesetTag.KEY))); public static final Set TAGS_FROM_ATLAS = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(ISOCountryTag.KEY, SyntheticBoundaryNodeTag.KEY, SyntheticDuplicateOsmNodeTag.KEY, SyntheticGeometrySlicedTag.KEY, SyntheticInvalidMultiPolygonRelationMembersRemovedTag.KEY, SyntheticInvalidWaySectionTag.KEY, SyntheticRelationMemberAdded.KEY, SyntheticRelationRoleUpdated.KEY, SyntheticSyntheticRelationMemberTag.KEY, SyntheticInvalidGeometryTag.KEY))); private AtlasTag() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BarrierTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM barrier tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/barrier#values", osm = "http://wiki.openstreetmap.org/wiki/Key:barrier") public enum BarrierTag { FENCE, WALL, GATE, HEDGE, BOLLARD, LIFT_GATE, RETAINING_WALL, STILE, CYCLE_BARRIER, KERB, YES, ENTRANCE, BLOCK, TOLL_BOOTH, CATTLE_GRID, DITCH, KISSING_GATE, CITY_WALL, GUARD_RAIL, HEDGE_BANK, WIRE_FENCE, LINE, SWING_GATE, CHAIN, TURNSTILE, EMBANKMENT, FIELD_BOUNDARY, BORDER_CONTROL, SALLY_PORT, DOOR, HAMPSHIRE_GATE, WOOD_FENCE, BUMP_GATE, BUS_TRAP, JERSEY_BARRIER, SLIDING_GATE, HEIGHT_RESTRICTOR, HANDRAIL, LOG, AVALANCHE_PROTECTION, MEDIAN_STRIP, DEBRIS, TRAFFIC_ISLAND, WICKET_GATE, ROPE, @TagValueAs("full-height_turnstile") FULL_HEIGHT_TURNSTILE, PLANTER, PROPERTY_BOUNDARY, NO, CABLE_BARRIER, RAILING, SUMP_BUSTER, @TagValueAs("sump-buster") SUMPBUSTER; @TagKey public static final String KEY = "barrier"; private static final EnumSet BARRIERS = EnumSet.of(FENCE, WALL, GATE, HEDGE, BOLLARD, LIFT_GATE, RETAINING_WALL, STILE, CYCLE_BARRIER, KERB, YES, ENTRANCE, BLOCK, TOLL_BOOTH, CATTLE_GRID, DITCH, KISSING_GATE, CITY_WALL, GUARD_RAIL, HEDGE_BANK, WIRE_FENCE, LINE, SWING_GATE, CHAIN, TURNSTILE, EMBANKMENT, FIELD_BOUNDARY, BORDER_CONTROL, SALLY_PORT, DOOR, HAMPSHIRE_GATE, WOOD_FENCE, BUMP_GATE, BUS_TRAP, JERSEY_BARRIER, SLIDING_GATE, HEIGHT_RESTRICTOR, HANDRAIL, LOG, AVALANCHE_PROTECTION, MEDIAN_STRIP, DEBRIS, TRAFFIC_ISLAND, WICKET_GATE, ROPE, FULL_HEIGHT_TURNSTILE, PLANTER, PROPERTY_BOUNDARY, NO, CABLE_BARRIER, RAILING, SUMP_BUSTER, SUMPBUSTER); public static boolean isBarrier(final Taggable taggable) { final Optional barrier = Validators.from(BarrierTag.class, taggable); return barrier.isPresent() && BARRIERS.contains(barrier.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BicycleTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM bicycle tag. Also see http://wiki.openstreetmap.org/wiki/Bicycle#Bicycle_Restrictions for * further tagging detail. * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/bicycle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:bicycle") public enum BicycleTag { YES, DESIGNATED, USE_SIDEPATH, NO, PERMISSIVE, DESTINATION, DISMOUNT, PRIVATE, OFFICIAL, UNKNOWN; @TagKey public static final String KEY = "bicycle"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BoundaryTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM Boundary Tag. Also works with the {@link AdministrativeLevelTag}. * * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/boundary#values", osm = "http://wiki.openstreetmap.org/wiki/Key:boundary") public enum BoundaryTag { ADMINISTRATIVE, HISTORIC, MARITIME, NATIONAL_PARK, POLITICAL, POSTAL_CODE, RELIGIOUS_ADMINISTRATION, PROTECTED_AREA; @TagKey public static final String KEY = "boundary"; public static boolean isAdministrative(final Taggable taggable) { return Validators.isOfType(taggable, BoundaryTag.class, ADMINISTRATIVE); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BrandTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM brand tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/brand#values", osm = "http://wiki.openstreetmap.org/wiki/Key:brand") public interface BrandTag { @TagKey String KEY = "brand"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BreakfastTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM breakfast tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/breakfast#values", osm = "http://wiki.openstreetmap.org/wiki/Key:breakfast") public enum BreakfastTag { YES, BUFFET, NO, FREE; @TagKey public static final String KEY = "breakfast"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BridgeTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM bridge tag * * @author cstaylor * @author pmi */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/bridge#values", osm = "http://wiki.openstreetmap.org/wiki/Key:bridge") public enum BridgeTag { YES, VIADUCT, NO, AQUEDUCT, BOARDWALK, MOVABLE, SUSPENSION, CULVERT, ABANDONED, LOW_WATER_CROSSING, SIMPLE_BRUNNEL, COVERED; // Left out bridge=no and bridge=simple_brunnel private static final EnumSet BRIDGE_WAYS = EnumSet.of(YES, VIADUCT, AQUEDUCT, BOARDWALK, MOVABLE, SUSPENSION, CULVERT, ABANDONED, LOW_WATER_CROSSING, SIMPLE_BRUNNEL, COVERED); @TagKey public static final String KEY = "bridge"; public static Optional get(final Taggable taggable) { return Validators.from(BridgeTag.class, taggable); } public static boolean isBridge(final Taggable taggable) { final Optional bridge = get(taggable); return bridge.isPresent() && BRIDGE_WAYS.contains(bridge.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingHeightTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.AltitudeExtractor; /** * OSM building:height tag: https://taginfo.openstreetmap.org/keys/building%3Aheight#values. OSM * Wiki indicates that height is the standard key, but taginfo usage suggest prevalent use of * building:height as well * * @author isabellehillberg * @author bbreithaupt */ @Tag(value = Validation.LENGTH, taginfo = "https://taginfo.openstreetmap.org/keys/building%3Aheight#values", osm = "http://wiki.openstreetmap.org/wiki/Key:height") public interface BuildingHeightTag { @TagKey String KEY = "building:height"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return AltitudeExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingLevelsTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.DoubleValidator; /** * OSM building:levels tag * * @author robert_stack */ @Tag(value = Validation.ORDINAL, taginfo = "http://taginfo.openstreetmap.org/keys/building:levels#values", osm = "http://wiki.openstreetmap.org/wiki/Key:building:levels") public interface BuildingLevelsTag { @TagKey String KEY = "building:levels"; DoubleValidator validator = new DoubleValidator(); static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent() && validator.isValid(tagValue.get())) { return Optional.of(Double.valueOf(tagValue.get())); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingMinLevelTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.DoubleValidator; /** * OSM building:min_level tag * * @author ajayaswal */ @Tag(value = Validation.ORDINAL, taginfo = "http://taginfo.openstreetmap.org/keys/building:min_level#values", osm = "http://wiki.openstreetmap.org/wiki/Key:building:min_level") public interface BuildingMinLevelTag { @TagKey String KEY = "building:min_level"; DoubleValidator validator = new DoubleValidator(); static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent() && validator.isValid(tagValue.get())) { return Optional.of(Double.valueOf(tagValue.get())); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingPartTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM building:part tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/building%3Apart#values", osm = "http://wiki.openstreetmap.org/wiki/Key:building:part") public enum BuildingPartTag { YES, ROOF, APARTMENTS, COLUMN, NO, COMMERCIAL, BASE, STEPS, RESIDENTIAL, GARAGE, HOUSE, RETAIL, SCHOOL, ROOT, INDUSTRIAL, DEFAULT, STILOBATE, OFFICE, SUPERSTRUCTURE, RUINS, OUTLINE, UNIVERSITY; @TagKey public static final String KEY = "building:part"; public String getTagValue() { return name().toLowerCase().intern(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingRoofTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; /** * OSM building:roof tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/building%3Aroof#values", osm = "http://wiki.openstreetmap.org/wiki/Proposed_features/Building_attributes") public enum BuildingRoofTag { TILE, TRADITIONAL, FLAT, CONCRETE, PERMANENT, TIN, METAL, TILES, SLATE, @TagValueAs("semi-permanent") SEMI_PERMANENT, ASBESTOS, PITCHED, IRONSHEETS, HIPPED, GABLED; @TagKey public static final String KEY = "building:roof"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/BuildingTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM building tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/building#values", osm = "http://wiki.openstreetmap.org/wiki/Buildings") public enum BuildingTag { YES, RESIDENTIAL, COMMERCIAL, SHOP, HOUSE, GARAGE, APARTMENTS, HUT, INDUSTRIAL, DETACHED, ROOF, SHED, TERRACE, SCHOOL, RETAIL, FARM_AUXILIARY, CHURCH, BARN, CONSTRUCTION, GREENHOUSE, SERVICE, MANUFACTURE, CABIN, FARM, WAREHOUSE, CIVIC, COLLAPSED, OFFICE, NO, UNIVERSITY, HOTEL, DORMITORY, BUNGALOW, CHAPEL, MOSQUE, KINDERGARTEN, HOSPITAL, STADIUM, TRAIN_STATION, TRANSPORTATION, PUBLIC, BUNKER, GARAGES, HANGAR, STABLE, TRANSFORMER_TOWER, RUINS, ENTRANCE, FACTORY, STORAGE_TANK, PAVILION, STORE, KIOSK, COWSHED, COLLEGE, SUPERMARKET, TANK, ADMINISTRATIVE, ABANDONED, SEMIDETACHED_HOUSE, CATHEDRAL, TEMPLE, SHELTER, POLICLINIC, GREENHOUSE_HORTICULTURE, STANDS, TOWER, KITCHEN, SILO, PARKING, OFFICES, STATION, SHACK, UNCLASSIFIED, GLASSHOUSE, GAZEBO, POLICE, COTTAGE, CASTLE_WALL, COVER, CELLAR, HEAT_STATION, CLINIC, PART, MILITARY, GRANDSTAND, UNDEFINED, CASTLE_TOWER, SHEDS, SPORT, HOME, SAUNA, DISUSED, TRIBUNE, POWER, BANK, ELEVATOR, PUBLIC_BUILDING, WATER_TOWER, MALL, MUSEUM, FIRE_STATION, MODEL, TOILETS, RAILWAY_STATION, FUEL, BRIDGE, THEATRE, CAFE, DAMAGED, STAND, ALLOTMENT_HOUSE, GOVERNMENT, SPORTS_CENTRE; private static final EnumSet VALID_BUILDINGS = EnumSet.of(YES, RESIDENTIAL, COMMERCIAL, SHOP, HOUSE, GARAGE, APARTMENTS, HUT, INDUSTRIAL, DETACHED, ROOF, SHED, TERRACE, SCHOOL, RETAIL, FARM_AUXILIARY, CHURCH, BARN, CONSTRUCTION, GREENHOUSE, SERVICE, MANUFACTURE, CABIN, FARM, WAREHOUSE, CIVIC, COLLAPSED, OFFICE, UNIVERSITY, HOTEL, DORMITORY, BUNGALOW, CHAPEL, MOSQUE, KINDERGARTEN, HOSPITAL, STADIUM, TRAIN_STATION, TRANSPORTATION, PUBLIC, BUNKER, GARAGES, HANGAR, STABLE, TRANSFORMER_TOWER, RUINS, FACTORY, STORAGE_TANK, PAVILION, STORE, KIOSK, COWSHED, COLLEGE, SUPERMARKET, TANK, ADMINISTRATIVE, ABANDONED, SEMIDETACHED_HOUSE, CATHEDRAL, TEMPLE, SHELTER, POLICLINIC, GREENHOUSE_HORTICULTURE, STANDS, TOWER, KITCHEN, SILO, PARKING, OFFICES, STATION, SHACK, UNCLASSIFIED, GLASSHOUSE, GAZEBO, POLICE, COTTAGE, CASTLE_WALL, COVER, CELLAR, HEAT_STATION, CLINIC, PART, MILITARY, GRANDSTAND, UNDEFINED, CASTLE_TOWER, SHEDS, SPORT, HOME, SAUNA, DISUSED, TRIBUNE, POWER, BANK, ELEVATOR, PUBLIC_BUILDING, WATER_TOWER, MALL, MUSEUM, FIRE_STATION, MODEL, TOILETS, RAILWAY_STATION, FUEL, BRIDGE, THEATRE, CAFE, DAMAGED, STAND, ALLOTMENT_HOUSE, GOVERNMENT, SPORTS_CENTRE); @TagKey public static final String KEY = "building"; public static final String BUILDING_ROLE_OUTLINE = "outline"; public static final String BUILDING_ROLE_PART = "part"; public static boolean isBuilding(final String value) { return isBuilding(taggable -> Optional.of(value)); } public static boolean isBuilding(final Taggable taggable) { final Optional building = Validators.from(BuildingTag.class, taggable); return building.isPresent() && VALID_BUILDINGS.contains(building.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CheckDateTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM check_date tag * * @author brianjor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/check_date#values", osm = "https://wiki.openstreetmap.org/wiki/Key:check_date") public interface CheckDateTag { @TagKey String KEY = "check_date"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ConstructionDateTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM construction:date tag * * @author brianjor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/construction:date#values", osm = "https://wiki.openstreetmap.org/wiki/Item:Q9553") public interface ConstructionDateTag { @TagKey String KEY = "construction:date"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ConstructionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Construction tag. Inherits values from Bridge, Highway, LandUse and Railway tags. * * @author bbreithaupt */ @Tag(with = { BuildingTag.class, HighwayTag.class, LandUseTag.class, RailwayTag.class }, taginfo = "https://taginfo.openstreetmap.org/keys/construction", osm = "https://wiki.openstreetmap.org/wiki/Key:construction") @SuppressWarnings("squid:S1214") public interface ConstructionTag { @TagKey String KEY = "construction"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactDiasporaTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:diaspora tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Adiaspora#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactDiasporaTag { @TagKey String KEY = "contact:diaspora"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactEmailTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:email tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Aemail#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactEmailTag { @TagKey String KEY = "contact:email"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactFacebookTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:facebook tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Afacebook#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactFacebookTag { @TagKey String KEY = "contact:facebook"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactFaxTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:fax tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Afax#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactFaxTag { @TagKey String KEY = "contact:fax"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactGooglePlusTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:google_plug tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Agoogle_plus#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactGooglePlusTag { @TagKey String KEY = "contact:google_plus"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactInstagramTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:instagram tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Ainstagram#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactInstagramTag { @TagKey String KEY = "contact:instagram"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactLinkedInTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:linkedin tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Alinkedin#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactLinkedInTag { @TagKey String KEY = "contact:linkedin"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactMobileTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:mobile tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Amobile#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactMobileTag { @TagKey String KEY = "contact:mobile"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactPhoneTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:phone tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Aphone#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactPhoneTag { @TagKey String KEY = "contact:phone"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactTwitterTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:twitter tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Atwitter#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactTwitterTag { @TagKey String KEY = "contact:twitter"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactWebsiteTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:website tag * * @author cstaylor */ @Tag(value = Validation.URI, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Awebsite#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactWebsiteTag { @TagKey String KEY = "contact:website"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ContactXingTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM contact:xing tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/contact%3Axing#values", osm = "http://wiki.openstreetmap.org/wiki/Key:contact") public interface ContactXingTag { @TagKey String KEY = "contact:xing"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CoveredTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Covered Tag * * @author cuthbertm */ @Tag(value = Tag.Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/covered", osm = "http://wiki.openstreetmap.org/wiki/Key:covered") public enum CoveredTag { YES, NO, ARCADE, COLONNADE; @TagKey public static final String KEY = "covered"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CuisineTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM cuisine tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/cuisine#values", osm = "http://wiki.openstreetmap.org/wiki/Key:cuisine") public interface CuisineTag { @TagKey String KEY = "cuisine"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CyclewayLaneTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM cycleway lane tag * * @author james_gage */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Alane#values", osm = "http://wiki.openstreetmap.org/wiki/Key:cycleway:lane") public enum CyclewayLaneTag { ADVISORY, EXCLUSIVE; @TagKey public static final String KEY = "cycleway:lane"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CyclewayLeftTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM cycleway left tag * * @author james_gage */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Aleft#values", osm = "http://wiki.openstreetmap.org/wiki/Key:cycleway:left") public enum CyclewayLeftTag { LANE, TRACK, OPPOSITE_LANE, OPPOSITE_SHARE_BUSWAY, SHARED_LANE, SHARED, SHARED_BUSWAY; @TagKey public static final String KEY = "cycleway:left"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CyclewayRightTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM cycleway right tag * * @author james_gage */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Aright#values", osm = "http://wiki.openstreetmap.org/wiki/Key:cycleway:right") public enum CyclewayRightTag { LANE, TRACK, OPPOSITE_LANE, OPPOSITE_SHARE_BUSWAY, SHARED_LANE, SHARED, SHARED_BUSWAY; @TagKey public static final String KEY = "cycleway:right"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/CyclewayTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM cycleway tag * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/cycleway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:cycleway") public enum CyclewayTag { LANE, OPPOSITE_LANE, OPPOSITE, SHARED_LANE, SHARE_BUSWAY, SHARED, TRACK, OPPOSITE_TRACK, ASL, SHOULDER, NO, YES, LEFT, RIGHT, OPPOSITE_SHARE_BUSWAY, SEGREGATED, NONE, SEPARATE; @TagKey public static final String KEY = "cycleway"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationForwardTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination:forward tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination%3Aforward") public interface DestinationForwardTag { @TagKey String KEY = "destination:forward"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationIntRefTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination:int_ref tag * * @author sbhalekar */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination%3Aint_ref") public interface DestinationIntRefTag { @TagKey String KEY = "destination:int_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationRefTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination:ref tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination%3Aref") public interface DestinationRefTag { @TagKey String KEY = "destination:ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationRefToTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination:ref:to tag * * @author sbhalekar */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination%3Aref%3Ato") public interface DestinationRefToTag { @TagKey String KEY = "destination:ref:to"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationStreetTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination:street tag * * @author sbhalekar */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination%3Astreet") public interface DestinationStreetTag { @TagKey String KEY = "destination:street"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DestinationTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM destination tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/destination") public interface DestinationTag { @TagKey String KEY = "destination"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DirectionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM direction tag * * @author MonicaBrandeis */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/direction", osm = "https://wiki.openstreetmap.org/wiki/Key:direction") public enum DirectionTag { FORWARD, BACKWARD, CLOCKWISE, ANTICLOCKWISE, BOTH; @TagKey public static final String KEY = "direction"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DisusedRailwayTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM disused: Railway tag * * @author Vladimir Lemberg */ @Tag(with = { ShopTag.class }, taginfo = "https://taginfo.openstreetmap.org/keys/disused%3Arailway#values", osm = "https://wiki.openstreetmap.org/wiki/Key:disused:railway") public enum DisusedRailwayTag { YES, RAIL, LIGHT_RAIL, CROSSING, LEVEL_CROSSING, STATION, HALT, PLATFORM, TRAM; @TagKey public static final String KEY = "disused:railway"; private static final EnumSet DISUSED_RAILWAY_CROSSINGS = EnumSet.of(CROSSING, LEVEL_CROSSING); public static Optional get(final Taggable taggable) { return Validators.from(DisusedRailwayTag.class, taggable); } public static boolean isDisusedRailwayCrossing(final Taggable taggable) { final Optional disusedRailway = get(taggable); return disusedRailway.isPresent() && DISUSED_RAILWAY_CROSSINGS.contains(disusedRailway.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/DisusedShopTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM disused:shop tag *

* Note how we're using the with optional parameter to Tag. This lets us bring in all of the values * defined in the ShopTag enum * * @author cstaylor */ @Tag(with = { ShopTag.class }, taginfo = "http://taginfo.openstreetmap.org/keys/disused%3Ashop#values", osm = "http://wiki.openstreetmap.org/wiki/Key:disused:") public interface DisusedShopTag { @TagKey String KEY = "disused:shop"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ElevationTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM elevation tag. This tag is used to express height above sea level of a point in meters. * * @author mgostintsev */ @Tag(value = Validation.DOUBLE, taginfo = "https://taginfo.openstreetmap.org/keys/ele#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ele") public interface ElevationTag { @TagKey String KEY = "ele"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/EmbankmentTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM embankment tag * * @author mkalender */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/embankment#values", osm = "http://wiki.openstreetmap.org/wiki/Key:embankment") public enum EmbankmentTag { YES, NO; @TagKey public static final String KEY = "embankment"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/EntranceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM entrance tag * * @author savannahostrowski */ @Tag(taginfo = "https://wiki.openstreetmap.org/wiki/Key:entrance#values", osm = "https://wiki.openstreetmap.org/wiki/Key:entrance") public enum EntranceTag { EMERGENCY, EXIT, HOME, MAIN, SERVICE, STAIRCASE, YES; @TagKey public static final String KEY = "entrance"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/EstimatedWidthTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.LengthExtractor; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * OSM estimated width tag * * @author bbreithaupt */ @Tag(value = Validation.DOUBLE, range = @Range(min = 0, max = Integer.MAX_VALUE), taginfo = "https://taginfo.openstreetmap.org/keys/est_width", osm = "https://wiki.openstreetmap.org/wiki/Key:est_width") public interface EstimatedWidthTag { @TagKey String KEY = "est_width"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return LengthExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ExitToLeftTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM exit_to:left tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/exit_to%3Aleft") public interface ExitToLeftTag { @TagKey String KEY = "exit_to:left"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ExitToRightTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM exit_to:right tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/exit_to%3Aright") public interface ExitToRightTag { @TagKey String KEY = "exit_to:right"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ExitToTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM exit_to tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/exit_to") public interface ExitToTag { @TagKey String KEY = "exit_to"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FaxTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM fax tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/fax#values", osm = "http://wiki.openstreetmap.org/wiki/Key:phone") public interface FaxTag { @TagKey String KEY = "fax"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FerryTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM Ferry Tag * * @author isabellehillberg */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/ferry#values", osm = "https://wiki.openstreetmap.org/wiki/Key:ferry") public enum FerryTag { YES, NO, MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, SERVICE, FOOTWAY, PEDESTRIAN, TRACK; @TagKey public static final String KEY = "ferry"; private static final EnumSet CAR_NAVIGABLE = EnumSet.of(MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, SERVICE); private static final EnumSet PEDESTRIAN_NAVIGABLE = EnumSet.of(FOOTWAY, TRACK, PEDESTRIAN); public static boolean isCarNavigableFerry(final Taggable taggable) { final Optional ferry = Validators.from(FerryTag.class, taggable); return ferry.isPresent() && CAR_NAVIGABLE.contains(ferry.get()); } public static boolean isPedestrianNavigableFerry(final Taggable taggable) { final Optional ferry = Validators.from(FerryTag.class, taggable); return ferry.isPresent() && PEDESTRIAN_NAVIGABLE.contains(ferry.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FixMeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM fix me tag. Check OSM Wiki below for more information. * * @author v-garei */ @Tag(value = Tag.Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/fixme#values", osm = "https://wiki.openstreetmap.org/wiki/Key:fixme") public interface FixMeTag { @TagKey String KEY = "fixme"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FootTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM foot tag * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/foot#values", osm = "http://wiki.openstreetmap.org/wiki/Key:foot") public enum FootTag { YES, NO, DESIGNATED, OFFICIAL, PRIVATE, PERMISSIVE, DESTINATION, USE_SIDEPATH, CUSTOMERS, UNKNOWN; private static final EnumSet PRIVATE_ACCESS = EnumSet.of(NO, PRIVATE); private static final EnumSet PEDESTRIAN_ACCESS = EnumSet.of(YES, DESIGNATED, OFFICIAL, PERMISSIVE, DESTINATION); @TagKey public static final String KEY = "foot"; public static boolean isPedestrianAccessible(final Taggable taggable) { final Optional foot = Validators.from(FootTag.class, taggable); return foot.isPresent() && PEDESTRIAN_ACCESS.contains(foot.get()); } public static boolean isPrivate(final Taggable taggable) { final Optional foot = Validators.from(FootTag.class, taggable); return foot.isPresent() && PRIVATE_ACCESS.contains(foot.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FootwayTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM footway tag * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/footway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:footway") public enum FootwayTag { SIDEWALK, CROSSING; @TagKey public static final String KEY = "footway"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FordTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM Ford Tag. * * @author sayas01 */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/ford#values", osm = "https://wiki.openstreetmap.org/wiki/Key:ford") public enum FordTag { YES, STEPPING_STONES, SEASONAL, NO, STREAM, LEVEL_CROSSING, BOAT; @TagKey public static final String KEY = "ford"; public static boolean isYes(final Taggable taggable) { return Validators.isOfType(taggable, FordTag.class, FordTag.YES); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/FourWheelDriveOnlyTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * This tag denotes roads that are only navigable by vehicles with Four wheel drive * * @author kkonishi2 */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/?key=4wd_only", osm = "https://wiki.openstreetmap.org/wiki/Key:4wd_only") public enum FourWheelDriveOnlyTag { YES, NO; @TagKey public static final String KEY = "4wd_only"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/HarbourTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Harbour tag * * @author mgostintsev */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/harbour#values", osm = "http://wiki.openstreetmap.org/wiki/Harbour") public enum HarbourTag { YES; @TagKey public static final String KEY = "harbour"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/HeightTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.AltitudeExtractor; /** * OSM height tag: http://taginfo.openstreetmap.org/keys/height#values * * @author cstaylor * @author bbreithaupt */ @Tag(value = Validation.LENGTH, taginfo = "http://taginfo.openstreetmap.org/keys/height#values", osm = "http://wiki.openstreetmap.org/wiki/Key:height") public interface HeightTag { @TagKey String KEY = "height"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return AltitudeExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/HighResolutionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM hires tag * * @author mgostintsev */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/hires#values") public enum HighResolutionTag { YES, NO, MAPBOX, BING; @TagKey public static final String KEY = "hires"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/HighwayTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import com.google.common.collect.EnumBiMap; /** * OSM highway tag * * @author cstaylor * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/highway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:highway") public enum HighwayTag { MOTORWAY, MOTORWAY_LINK, TRUNK, TRUNK_LINK, PRIMARY, PRIMARY_LINK, SECONDARY, SECONDARY_LINK, TERTIARY, TERTIARY_LINK, UNCLASSIFIED, RESIDENTIAL, SERVICE, TRACK, LIVING_STREET, PEDESTRIAN, BUS_GUIDEWAY, RACEWAY, ROAD, CYCLEWAY, FOOTWAY, BRIDLEWAY, STEPS, PATH, PROPOSED, CONSTRUCTION, ESCAPE, BUS_STOP, CROSSING, ELEVATOR, EMERGENCY_ACCESS_POINT, EMERGENCY_BAY, GIVE_WAY, MINI_ROUNDABOUT, MOTORWAY_JUNCTION, PASSING_PLACE, REST_AREA, SPEED_CAMERA, STREET_LAMP, SERVICES, STOP, TRAILHEAD, TRAFFIC_MIRROR, TRAFFIC_SIGNALS, TURNING_CIRCLE, PLATFORM, MILESTONE, TURNING_LOOP, CORRIDOR, NO, TOLL_GANTRY; @TagKey public static final String KEY = "highway"; private static final EnumSet CORE_WAYS = EnumSet.of(MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, SERVICE, MOTORWAY_LINK, TRUNK_LINK, PRIMARY_LINK, SECONDARY_LINK, TERTIARY_LINK, LIVING_STREET, PEDESTRIAN, TRACK, BUS_GUIDEWAY, RACEWAY, ROAD, FOOTWAY, BRIDLEWAY, STEPS, PATH, CYCLEWAY, ESCAPE); private static final EnumSet METRICS_HIGHWAYS = EnumSet.of(MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, SERVICE, MOTORWAY_LINK, TRUNK_LINK, PRIMARY_LINK, SECONDARY_LINK, TERTIARY_LINK, LIVING_STREET, PEDESTRIAN, TRACK, BUS_GUIDEWAY, FOOTWAY, BRIDLEWAY, STEPS, PATH, CYCLEWAY, ESCAPE); private static final EnumSet CAR_NAVIGABLE_HIGHWAYS = EnumSet.of(MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, SERVICE, MOTORWAY_LINK, TRUNK_LINK, PRIMARY_LINK, SECONDARY_LINK, TERTIARY_LINK, LIVING_STREET, TRACK, ROAD); private static final EnumSet PEDESTRIAN_NAVIGABLE_HIGHWAYS = EnumSet.of(PEDESTRIAN, FOOTWAY, STEPS, PATH, CROSSING, PLATFORM, ELEVATOR, CORRIDOR); private static final EnumSet NODE_ONLY_HIGHWAYTAGS = EnumSet.of(BUS_STOP, CROSSING, EMERGENCY_ACCESS_POINT, GIVE_WAY, MILESTONE, MINI_ROUNDABOUT, MOTORWAY_JUNCTION, PASSING_PLACE, SPEED_CAMERA, STREET_LAMP, STOP, TRAFFIC_MIRROR, TRAFFIC_SIGNALS, TRAILHEAD, TURNING_LOOP, TOLL_GANTRY); private static final EnumSet WAY_ONLY_HIGHWAYTAGS = EnumSet.of(MOTORWAY, TRUNK, PRIMARY, SECONDARY, TERTIARY, UNCLASSIFIED, RESIDENTIAL, MOTORWAY_LINK, TRUNK_LINK, PRIMARY_LINK, SECONDARY_LINK, TERTIARY_LINK, LIVING_STREET, SERVICE, PEDESTRIAN, TRACK, BUS_GUIDEWAY, ESCAPE, RACEWAY, ROAD, FOOTWAY, BRIDLEWAY, STEPS, CORRIDOR, PATH, CYCLEWAY, PROPOSED, CONSTRUCTION); private static final EnumBiMap HIGHWAY_LINKS = EnumBiMap .create(HighwayTag.class, HighwayTag.class); static { HIGHWAY_LINKS.put(MOTORWAY, MOTORWAY_LINK); HIGHWAY_LINKS.put(TRUNK, TRUNK_LINK); HIGHWAY_LINKS.put(PRIMARY, PRIMARY_LINK); HIGHWAY_LINKS.put(SECONDARY, SECONDARY_LINK); HIGHWAY_LINKS.put(TERTIARY, TERTIARY_LINK); } public static Optional highwayTag(final Taggable taggable) { return Validators.from(HighwayTag.class, taggable); } public static boolean isCarNavigableHighway(final HighwayTag tag) { return CAR_NAVIGABLE_HIGHWAYS.contains(tag); } public static boolean isCarNavigableHighway(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && CAR_NAVIGABLE_HIGHWAYS.contains(highway.get()); } public static boolean isCoreWay(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && CORE_WAYS.contains(highway.get()); } /** * Looking for (highway=pedestrian or highway=footway) and (building = * or area=yes) tag * combination. These are pedestrian plazas and can contain valid road intersections. * * @param taggable * The taggable object being test * @return Whether the taggable object is a highway area or not */ public static boolean isHighwayArea(final Taggable taggable) { return Validators.isOfType(taggable, HighwayTag.class, HighwayTag.PEDESTRIAN, HighwayTag.FOOTWAY) && (Validators.isOfType(taggable, AreaTag.class, AreaTag.YES) || Validators.hasValuesFor(taggable, BuildingTag.class)); } public static boolean isHighwayWithLink(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && HIGHWAY_LINKS.containsKey(highway.get()); } public static boolean isLinkHighway(final HighwayTag tag) { return HIGHWAY_LINKS.containsValue(tag); } public static boolean isLinkHighway(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && HIGHWAY_LINKS.containsValue(highway.get()); } public static boolean isMetricHighway(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && METRICS_HIGHWAYS.contains(highway.get()); } public static boolean isNodeOnlyTag(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && NODE_ONLY_HIGHWAYTAGS.contains(highway.get()); } public static boolean isPedestrianCrossing(final Taggable taggable) { return Validators.isOfType(taggable, HighwayTag.class, CROSSING); } public static boolean isPedestrianNavigableHighway(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && PEDESTRIAN_NAVIGABLE_HIGHWAYS.contains(highway.get()); } public static boolean isWayOnlyTag(final Taggable taggable) { final Optional highway = highwayTag(taggable); return highway.isPresent() && WAY_ONLY_HIGHWAYTAGS.contains(highway.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(HighwayTag.class, taggable); } /** * Checks if the current highway type has a complementary link type * * @return true if has a link type for highway */ public boolean canHaveLink() { return HIGHWAY_LINKS.containsKey(this); } /** * Gets the highway type from a link type. So PRIMARY_LINK will return PRIMARY * * @return an Optional {@link HighwayTag}, if highway type does not have a link will return * Optional.empty */ public Optional getHighwayFromLink() { return Optional.ofNullable(HIGHWAY_LINKS.inverse().get(this)); } /** * Gets the highway_link type from the highway type. So PRIMARY will return PRIMARY_LINK * * @return an Optional {@link HighwayTag}, if no link for highway available will return * Optional.empty */ public Optional getLinkFromHighway() { return Optional.ofNullable(HIGHWAY_LINKS.get(this)); } public String getTagValue() { return name().toLowerCase().intern(); } /** * Checks to see if one highway type has the identical classification as another. * * @param tag * The {@link HighwayTag} that you are comparing this to * @return {@code true} if class is the same */ public boolean isIdenticalClassification(final HighwayTag tag) { return this == tag; } public boolean isLessImportantThan(final HighwayTag other) { return this.compareTo(other) > 0; } public boolean isLessImportantThanOrEqualTo(final HighwayTag other) { return this.compareTo(other) >= 0; } /** * Checks to see if the highway type is a link type * * @return true if it is link type */ public boolean isLink() { return HIGHWAY_LINKS.containsValue(this); } public boolean isMoreImportantThan(final HighwayTag other) { return this.compareTo(other) < 0; } public boolean isMoreImportantThanOrEqualTo(final HighwayTag other) { return this.compareTo(other) <= 0; } /** * Checks to see if one highway type has an equal classification as another. This either * indicates that it is the exact same type, or that one is a link and the other the same * non-link type. e.g. TRUNK and TRUNK_LINK. * * @param tag * The {@link HighwayTag} that you are comparing this to * @return {@code true} if class the same */ public boolean isOfEqualClassification(final HighwayTag tag) { return this == tag || HIGHWAY_LINKS.get(this) == tag || HIGHWAY_LINKS.inverse().get(this) == tag; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/HistoricTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM historic tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/historic#values", osm = "http://wiki.openstreetmap.org/wiki/Key:historic") public enum HistoricTag { AIRCRAFT, ARCHAEOLOGICAL_SITE, BATTLEFIELD, BOUNDARY_STONE, BUILDING, CANNON, CASTLE, CITY_GATE, CITYWALLS, FARM, FORT, GALLOWS, LOCOMOTIVE, MANOR, MEMORIAL, MILESTONE, MONASTERY, MONUMENT, OPTICAL_TELEGRAPH, PILLORY, RUINS, RUNE_STONE, SHIP, TOMB, WAYSIDE_CROSS, WAYSIDE_SHRINE, WRECK; @TagKey public static final String KEY = "historic"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ISOCountryTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.Collection; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.utilities.collections.Iterables; import com.google.common.collect.ImmutableList; /** * Annotated class representing the iso_country_code tag in Atlas data * * @author cstaylor * @author ahsieh * @author bbreithaupt */ @Tag(Validation.ISO3_COUNTRY) public interface ISOCountryTag { @TagKey String KEY = "iso_country_code"; String COUNTRY_MISSING = "N/A"; String COUNTRY_DELIMITER = ","; static Iterable all(final Taggable taggable) { final Optional countryCode = taggable.getTag(ISOCountryTag.class, Optional.empty()); if (countryCode.isPresent()) { return Arrays.asList(countryCode.get().split(COUNTRY_DELIMITER)); } return ImmutableList.of(); } /** * Tests if all of the item's iso_country_codes can be found in a list of countries * * @param countries * A set of countries we want to check in * @return Predicate used to test if all of the item's iso_country_codes can be found in the * list of countries */ static Predicate allIn(final Collection countries) { if (countries.isEmpty()) { return taggable -> false; } return taggable -> { for (final String country : all(taggable)) { // if any don't match, return false if (!countries.contains(country)) { return false; } } return true; }; } /** * Returns the first country code from the Taggable's iso_country_code tag. BEWARE: This method * may be non-deterministic as it depends on the ordering of the country codes * * @param taggable * The {@link Taggable} to get the first country code from * @return The first country code for the item if it exists */ static Optional first(final Taggable taggable) { return Iterables.first(all(taggable)); } static Predicate isIn(final Set countries) { if (countries.isEmpty()) { return taggable -> false; } return taggable -> { for (final String country : all(taggable)) { // if any one matches, return true if (countries.contains(country)) { return true; } } return false; }; } static Predicate isIn(final String countryToMatch) { if (countryToMatch == null || countryToMatch.isEmpty()) { return taggable -> false; } return taggable -> { for (final String country : all(taggable)) { // if any one matches, return true if (countryToMatch.equals(country)) { return true; } } return false; }; } static String join(final Collection countries) { return String.join(COUNTRY_DELIMITER, countries); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/IceRoadTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM ice_road tag. * * @author jacker */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/ice_road#values", osm = "https://wiki.openstreetmap.org/wiki/Key:ice_road") public enum IceRoadTag { YES, SNOWMOBILE, NO; @TagKey public static final String KEY = "ice_road"; public static boolean isYes(final Taggable taggable) { return Validators.isOfType(taggable, IceRoadTag.class, IceRoadTag.YES, IceRoadTag.SNOWMOBILE); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/IndustrialTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Industrial Tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/industrial#values", osm = "http://wiki.openstreetmap.org/wiki/Key:industrial") public enum IndustrialTag { ALUMINUM_SMELTING, BAKERY, BREWERY, BRICKYARD, DEPOT, DISTRIBUTOR, FACTORY, FURNITURE, GAS, GRINDING_MILL, HEATING_STATION, ICE_FACTORY, MACHINE_SHOP, MINE, MOBILE_EQUIPMENT, OIL_MILL, OIL, PORT, SALT_POND, SAWMILL, SCRAP_YARD, SHIPYARD, SLAUGHTERHOUSE, STEELMAKING, WAREHOUSE, WELL_CLUSTER, WELLSITE; @TagKey public static final String KEY = "industrial"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/IntermittentTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM's intermittent tags * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/intermittent#values", osm = "http://wiki.openstreetmap.org/wiki/Key:intermittent") public enum IntermittentTag { YES, NO, DRY; @TagKey public static final String KEY = "intermittent"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/InternetAccessFeeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM internet_access:fee tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/internet_access:fee#values", osm = "http://wiki.openstreetmap.org/wiki/Key:internet_access") public enum InternetAccessFeeTag { NO, YES; @TagKey public static final String KEY = "internet_access:fee"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/InternetAccessTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM internet_access tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/internet_access#values", osm = "http://wiki.openstreetmap.org/wiki/Key:internet_access") public enum InternetAccessTag { WLAN, NO, YES, TERMINAL; @TagKey public static final String KEY = "internet_access"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/Iso31662CountryTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.IsoCountryExtractor; /** * OSM's 2 digit ISO Country tag * * @author matthieun */ @Tag(Validation.ISO2_COUNTRY) public interface Iso31662CountryTag { @TagKey String KEY = "ISO3166-1:alpha2"; static Iterable all(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final IsoCountryExtractor extractor = new IsoCountryExtractor(); final Optional> countries = extractor.validateAndExtract( tagValue.get(), Iso31662CountryTag.class.getDeclaredAnnotation(Tag.class)); return countries.isPresent() ? countries.get() : new ArrayList<>(); } return new ArrayList<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/Iso31663CountryTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.IsoCountryExtractor; /** * OSM's 3 digit ISO Country tag * * @author matthieun */ @Tag(Validation.ISO3_COUNTRY) public interface Iso31663CountryTag { @TagKey String KEY = "ISO3166-1:alpha3"; static Iterable all(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final IsoCountryExtractor extractor = new IsoCountryExtractor(); final Optional> countries = extractor.validateAndExtract( tagValue.get(), Iso31663CountryTag.class.getDeclaredAnnotation(Tag.class)); return countries.isPresent() ? countries.get() : new ArrayList<>(); } return new ArrayList<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/Iso3166DefaultCountryTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.IsoCountryExtractor; /** * OSM's default 2 digit ISO Country tag * * @author matthieun */ @Tag(Validation.ISO2_COUNTRY) public interface Iso3166DefaultCountryTag { @TagKey String KEY = "ISO3166-1"; static Iterable all(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final IsoCountryExtractor extractor = new IsoCountryExtractor(); final Optional> countries = extractor.validateAndExtract( tagValue.get(), Iso3166DefaultCountryTag.class.getDeclaredAnnotation(Tag.class)); return countries.isPresent() ? countries.get() : new ArrayList<>(); } return new ArrayList<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/JunctionTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM junction tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/junction#values", osm = "http://wiki.openstreetmap.org/wiki/Junctions") public enum JunctionTag { ROUNDABOUT, CIRCULAR; @TagKey public static final String KEY = "junction"; public static boolean isCircular(final Taggable taggable) { final Optional junction = Validators.from(JunctionTag.class, taggable); return junction.isPresent() && CIRCULAR == junction.get(); } public static boolean isRoundabout(final Taggable taggable) { final Optional junction = Validators.from(JunctionTag.class, taggable); return junction.isPresent() && ROUNDABOUT == junction.get(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LandUseTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM landuse tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/landuse#values", osm = "http://wiki.openstreetmap.org/wiki/Landuse") public enum LandUseTag { ALLOTMENTS, AQUACULTURE, BASIN, BROWNFIELD, CEMETERY, COMMERCIAL, CONSTRUCTION, FARMLAND, FARMYARD, FOREST, GARAGES, GRASS, GREENFIELD, GREENHOUSE_HORTICULTURE, INDUSTRIAL, LANDFILL, MEADOW, MILITARY, ORCHARD, PEAT_CUTTING, PLANT_NURSERY, PORT, QUARRY, RAILWAY, RECREATION_GROUND, RESERVOIR, RESIDENTIAL, RETAIL, SALT_POND, VILLAGE_GREEN, VINEYARD, POND; @TagKey public static final String KEY = "landuse"; public static Optional get(final Taggable taggable) { return Validators.from(LandUseTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LandcoverTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM landcover tag * * @author stephencerqueira */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/landcover#values", osm = "https://wiki.openstreetmap.org/wiki/Proposed_features/landcover") public enum LandcoverTag { GRASS, GRAVEL, TREES; @TagKey public static final String KEY = "landcover"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LanesTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.OrdinalExtractor; /** * OSM lanes tag * * @author robert_stack */ @Tag(value = Validation.ORDINAL, range = @Range(min = 1, max = 50), taginfo = "http://taginfo.openstreetmap.org/keys/lanes#values", osm = "http://wiki.openstreetmap.org/wiki/Lanes") public interface LanesTag { @TagKey String KEY = "lanes"; static Optional numberOfLanes(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final OrdinalExtractor extractor = new OrdinalExtractor(); return extractor.validateAndExtract(tagValue.get(), LanesTag.class.getDeclaredAnnotation(Tag.class)); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LastEditChangesetTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Annotated class representing the last_edit_changeset tag in Atlas data. Though a tag in an atlas, * this property does not come from a tag in an OSM Entity. It comes from the OSM Entity attribute: * changeset. Every atlas entity will have this tag. * * @author nhallahan */ @Tag(Validation.LONG) public interface LastEditChangesetTag { @TagKey String KEY = "last_edit_changeset"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LastEditTimeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Annotated class representing the last_edit_time tag in Atlas data. Though a tag in an atlas, this * property does not come from a tag in an OSM Entity. It comes from the OSM Entity attribute: * timestamp. Every atlas entity will have this tag. * * @author cstaylor */ @Tag(Validation.TIMESTAMP) public interface LastEditTimeTag { @TagKey String KEY = "last_edit_time"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LastEditUserIdentifierTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Annotated class representing the last_edit_user_id tag in Atlas data. Though a tag in an atlas, * this property does not come from a tag in an OSM Entity. It comes from the OSM Entity attribute: * uid. Every atlas entity will have this tag. * * @author tony */ @Tag(Validation.LONG) public interface LastEditUserIdentifierTag { @TagKey String KEY = "last_edit_user_id"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LastEditUserNameTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Annotated class representing the last_edit_user_name tag in Atlas data. Though a tag in an atlas, * this property does not come from a tag in an OSM Entity. It comes from the OSM Entity attribute: * user. Every atlas entity will have this tag. * * @author cstaylor */ @Tag(Validation.NON_EMPTY_STRING) public interface LastEditUserNameTag { @TagKey String KEY = "last_edit_user_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LastEditVersionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Annotated class representing the last_edit_version tag in Atlas data. Though a tag in an atlas, * this property does not come from a tag in an OSM Entity. It comes from the OSM Entity attribute: * version. Every atlas entity will have this tag. * * @author nhallahan */ @Tag(Validation.LONG) public interface LastEditVersionTag { @TagKey String KEY = "last_edit_version"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LayerTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.LongExtractor; /** * OSM layer tag * * @author cstaylor * @author brian_l_davis * @author bbreithaupt */ @Tag(value = Validation.LONG, range = @Range(min = -5, max = 5, exclude = { 0 }), taginfo = "http://taginfo.openstreetmap.org/keys/layer#values", osm = "http://wiki.openstreetmap.org/wiki/Layer") public interface LayerTag { Long ZERO = 0L; @TagKey String KEY = "layer"; /** * Checks if two Taggable objects are on the same layer or not. According to OSM wiki, objects * with no explicit LayerTag are assumed to have layer 0. * * @param taggableOne * first object to compare * @param taggableTwo * second object to compare * @return true if the two objects have same layer tag, false otherwise */ static boolean areOnSameLayer(final Taggable taggableOne, final Taggable taggableTwo) { return LayerTag.getTaggedOrImpliedValue(taggableOne, ZERO) .equals(LayerTag.getTaggedOrImpliedValue(taggableTwo, ZERO)); } static long getMaxValue() { return LayerTag.class.getDeclaredAnnotation(Tag.class).range().max(); } static long getMinValue() { return LayerTag.class.getDeclaredAnnotation(Tag.class).range().min(); } static Long getTaggedOrImpliedValue(final Taggable taggable, final Long impliedValue) { final Optional taggedValue = getTaggedValue(taggable); // Return the layer tag if there is one if (taggedValue.isPresent()) { return taggedValue.get(); } // Else return 1 if taggable is a bridge if (BridgeTag.isBridge(taggable)) { return 1L; } // Else return -1 if taggable is a tunnel if (TunnelTag.isTunnel(taggable)) { return -1L; } // Else return the implied value return impliedValue; } static Optional getTaggedValue(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final LongExtractor extractor = new LongExtractor(); return extractor.validateAndExtract(tagValue.get(), LayerTag.class.getDeclaredAnnotation(Tag.class)); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LeisureTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM leisure tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/leisure#values", osm = "http://wiki.openstreetmap.org/wiki/Key:leisure") public enum LeisureTag { PITCH, SWIMMING_POOL, PARK, PLAYGROUND, GARDEN, SPORTS_CENTRE, NATURE_RESERVE, COMMON, TRACK, STADIUM, GOLF_COURSE, RECREATION_GROUND, SLIPWAY, PICNIC_TABLE, MARINA, WATER_PARK, DOG_PARK, FIREPIT, SAUNA, MINIATURE_GOLF, FISHING, HORSE_RIDING, FITNESS_STATION, ICE_RINK, BEACH_RESORT, YES, BIRD_HIDE, RESORT, DANCE, ADULT_GAMING_CENTRE, CLUB, CLIMBING, BLEACHERS, OUTDOOR_SEATING, TANNING_SALON, BANDSTAND, SOCIAL_CLUB, FESTIVAL_GROUNDS; @TagKey public static final String KEY = "leisure"; public static Optional get(final Taggable taggable) { return Validators.from(LeisureTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LevelTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM level tag * * @author sayas01 */ @Tag(value = Tag.Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/level#values", osm = "http://wiki.openstreetmap.org/wiki/Level") @SuppressWarnings("squid:S1214") public interface LevelTag { @TagKey String KEY = "level"; /** * Checks if two Taggable objects are on the same level or not. As per * https://wiki.openstreetmap.org/wiki/Key:level, level=0 is not always at street level and so * unlike LayerTag, if the LevelTag is not explicitly given, we cannot imply that the object is * at level 0. * * @param taggableOne * first object to compare * @param taggableTwo * second object to compare * @return true if object one and object two are on the same level */ static boolean areOnSameLevel(final Taggable taggableOne, final Taggable taggableTwo) { final Optional levelTagEdgeOne = LevelTag.getTaggedValue(taggableOne); final Optional levelTagEdgeTwo = LevelTag.getTaggedValue(taggableTwo); if (levelTagEdgeOne.isPresent() && levelTagEdgeTwo.isPresent()) { return levelTagEdgeOne.get().equals(levelTagEdgeTwo.get()); } return !levelTagEdgeOne.isPresent() && !levelTagEdgeTwo.isPresent(); } static String getTaggedOrImpliedValue(final Taggable taggable, final String impliedValue) { final Optional taggedValue = getTaggedValue(taggable); return taggedValue.isPresent() ? taggedValue.get() : impliedValue; } static Optional getTaggedValue(final Taggable taggable) { return taggable.getTag(KEY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LivingStreetTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM's living_street tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/living_street#values", osm = "http://wiki.openstreetmap.org/wiki/Proposed_features/Tag:living_street%3Dyes") public enum LivingStreetTag { YES, NO, RESIDENTIAL; @TagKey public static final String KEY = "living_street"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LocalizedTagNameWithOptionalDate.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import java.util.stream.Collectors; import org.openstreetmap.atlas.locale.IsoLanguage; import org.openstreetmap.atlas.utilities.collections.StringList; import com.google.common.base.Joiner; /** * Some tags in OSM have optional languages and even date ranges. *

* This class is immutable, so it's threadsafe * * @author cstaylor */ public class LocalizedTagNameWithOptionalDate { private final String name; private Optional language = Optional.empty(); // For now, we just leave this be. In the future we may parse and use this portion of the name private Optional untouchedDateRange = Optional.empty(); public LocalizedTagNameWithOptionalDate(final String key) { if (key == null) { throw new IllegalArgumentException("key can't be null"); } final StringList list = StringList.split(key, ":"); // This is the count of items in the string list that are part of the name and not optional // language or date range fields int nameBlocks = list.size(); if (nameBlocks > 1) { // Check if we have a '-' character in the last item // This would make it a date range if (list.get(nameBlocks - 1).contains("-")) { // We don't need the date range in the name nameBlocks--; this.untouchedDateRange = Optional.of(list.get(nameBlocks)); } // Check if we have the language portion at the end: it's possible nameBlocks is now 1, // so we need to check it again if (nameBlocks > 1) { this.language = IsoLanguage.forLanguageCode(list.get(nameBlocks - 1)); if (this.language.isPresent()) { nameBlocks--; } } } // Thank you streams! this.name = Joiner.on(":") .join(list.stream().limit(nameBlocks).collect(Collectors.toList())); } public Optional getDateRange() { return this.untouchedDateRange; } public Optional getLanguage() { return this.language; } public String getName() { return this.name; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/LocationTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM location tag * * @author mkalender */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/location#values", osm = "http://wiki.openstreetmap.org/wiki/Key:location") public enum LocationTag { INDOOR, KIOSK, OUTDOOR, OVERGROUND, OVERWATER, PLATFORM, ROOF, ROOFTOP, UNDERGROUND, UNDERWATER; @TagKey public static final String KEY = "location"; public static Optional get(final Taggable taggable) { return Validators.from(LocationTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ManMadeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM man_made tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/man_made#values", osm = "http://wiki.openstreetmap.org/wiki/Key:man_made") public enum ManMadeTag { ADIT, BEACON, BREAKWATER, BRIDGE, BUNKER_SILO, CAMPANILE, CHIMNEY, COMMUNICATIONS_TOWER, CRANE, CROSS, CUTLINE, CLEARCUT, EMBANKMENT, DYKE, FLAGPOLE, GASOMETER, GROYNE, HOT_WATER_TANK, KILN, LIGHTHOUSE, MAST, MINESHAFT, MONITORING_STATION, OBSERVATORY, OFFSHORE_PLATFORM, PETROLEUM_WELL, PIER, PIPELINE, PUMPING_STATION, RESERVOIR_COVERED, SILO, SNOW_FENCE, SNOW_NET, STORAGE_TANK, STREET_CABINET, SURVEILLANCE, SURVEY_POINT, TELESCOPE, TOWER, WASTEWATER_PLANT, WATERMILL, WATER_TOWER, WATER_WELL, WATER_TAP, WATER_WORKS, WILDLIFE_CROSSING, WINDMILL, WORKS; @TagKey public static final String KEY = "man_made"; public static boolean isBridge(final Taggable taggable) { return Validators.isOfType(taggable, ManMadeTag.class, BRIDGE); } public static boolean isPier(final Taggable taggable) { return Validators.isOfType(taggable, ManMadeTag.class, PIER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MaxHeightTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.TagValue.ValueType; import org.openstreetmap.atlas.tags.annotations.extraction.AltitudeExtractor; /** * OSM maxheight tag: http://taginfo.openstreetmap.org/keys/maxheight#values * * @author cstaylor * @author bbreithaupt */ @Tag(value = Validation.DOUBLE, taginfo = "http://taginfo.openstreetmap.org/keys/maxheight#values", osm = "http://wiki.openstreetmap.org/wiki/Key:maxheight") public interface MaxHeightTag { @TagKey String KEY = "maxheight"; @TagValue String DEFAULT = "default"; @TagValue String NONE = "none"; @TagValue(ValueType.REGEX) String METERS = "(\\d+(\\.\\d+)?|\\.\\d+)(\\sm)?"; @TagValue(ValueType.REGEX) String FEET = "\\d'\\d\""; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return AltitudeExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MaxSpeedBackwardTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.extraction.SpeedExtractor; import org.openstreetmap.atlas.utilities.scalars.Speed; /** * OSM maxspeed:backward tag * * @author matthieun */ @Tag(value = Validation.SPEED, taginfo = "http://taginfo.openstreetmap.org/keys/maxspeed#values", osm = "http://wiki.openstreetmap.org/wiki/Key:maxspeed") public interface MaxSpeedBackwardTag { @TagKey String KEY = "maxspeed:backward"; @TagValue String NONE = "none"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return SpeedExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } static boolean hasMaxSpeedBackward(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MaxSpeedForwardTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.extraction.SpeedExtractor; import org.openstreetmap.atlas.utilities.scalars.Speed; /** * OSM maxspeed:forward tag * * @author matthieun */ @Tag(value = Validation.SPEED, taginfo = "http://taginfo.openstreetmap.org/keys/maxspeed#values", osm = "http://wiki.openstreetmap.org/wiki/Key:maxspeed") public interface MaxSpeedForwardTag { @TagKey String KEY = "maxspeed:forward"; @TagValue String NONE = "none"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return SpeedExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } static boolean hasMaxSpeedForward(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MaxSpeedTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.extraction.SpeedExtractor; import org.openstreetmap.atlas.utilities.scalars.Speed; /** * OSM maxspeed tag * * @author cstaylor * @author matthieun */ @Tag(value = Validation.SPEED, taginfo = "http://taginfo.openstreetmap.org/keys/maxspeed#values", osm = "http://wiki.openstreetmap.org/wiki/Key:maxspeed") public interface MaxSpeedTag { @TagKey String KEY = "maxspeed"; @TagValue String NONE = "none"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return SpeedExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } static boolean hasExtendedMaxSpeed(final Taggable taggable) { return hasMaxSpeed(taggable) || MaxSpeedForwardTag.hasMaxSpeedForward(taggable) || MaxSpeedBackwardTag.hasMaxSpeedBackward(taggable); } static boolean hasMaxSpeed(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MaxWidthTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.TagValue.ValueType; import org.openstreetmap.atlas.tags.annotations.extraction.LengthExtractor; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * OSM maxwidth tag * * @author cstaylor * @author bbreithaupt */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/maxwidth#values", osm = "http://wiki.openstreetmap.org/wiki/Key:maxwidth") public interface MaxWidthTag { @TagKey String KEY = "maxwidth"; @TagValue(ValueType.REGEX) String METERS = "(\\d+(\\.\\d+)?|\\.\\d+)(\\sm)?"; @TagValue(ValueType.REGEX) String FEET = "\\d'\\d\""; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return LengthExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MilitaryTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM military tag * * @author cstaylor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/military#values", osm = "https://wiki.openstreetmap.org/wiki/Key%3Amilitary") public enum MilitaryTag { BUNKER, BARRACKS, NUCLEAR_EXPLOSION_SITE, RANGE, TRENCH, AIRFIELD, YES, CHECKPOINT, DANGER_AREA, NAVAL_BASE, TRAINING_AREA; @TagKey public static final String KEY = "military"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MinHeightTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.AltitudeExtractor; import org.openstreetmap.atlas.tags.annotations.validation.DoubleValidator; /** * OSM min_height tag * * @author ajayaswal * @author bbreithaupt */ @Tag(value = Validation.DOUBLE, taginfo = "https://taginfo.openstreetmap.org/keys/min_height#values", osm = "https://wiki.openstreetmap.org/wiki/Key:min_height") public interface MinHeightTag { @TagKey String KEY = "min_height"; DoubleValidator validator = new DoubleValidator(); static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return AltitudeExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MinSpeedTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM minspeed tag * * @author mgostintsev */ @Tag(value = Validation.SPEED, taginfo = "http://taginfo.openstreetmap.org/keys/minspeed#values", osm = "http://wiki.openstreetmap.org/wiki/Key:minspeed") public interface MinSpeedTag { @TagKey String KEY = "minspeed"; static boolean hasMinSpeed(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MotorVehicleTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM motor vehicle tag * * @author pmi */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/motor_vehicle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:motor_vehicle") public enum MotorVehicleTag { YES, NO, PRIVATE, AGRICULTURAL, PERMISSIVE, DESTINATION, DESIGNATED, DELIVERY, FORESTRY, OFFICIAL, CUSTOMERS, EMERGENCY, BUS, SERVICE, UNKNOWN; @TagKey public static final String KEY = "motor_vehicle"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MotorcarTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM motorcar tag * * @author mkalender */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/motorcar#values", osm = "http://wiki.openstreetmap.org/wiki/Key:motorcar") public enum MotorcarTag { YES, NO, PRIVATE, AGRICULTURAL, PERMISSIVE, DESTINATION, DESIGNATED, DELIVERY, FORESTRY, OFFICIAL, CUSTOMERS, EMERGENCY, BUS, SERVICE, UNKNOWN; @TagKey public static final String KEY = "motorcar"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/MotorcycleTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM motorcyle tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/motorcycle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:motorcycle") public enum MotorcycleTag { NO, YES, AGRICULTURAL, DESIGNATED, FORESTRY, PRIVATE, PERMISSIVE, UNKNOWN; @TagKey public static final String KEY = "motorcycle"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/NaturalTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM natural tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/natural#values", osm = "http://wiki.openstreetmap.org/wiki/Natural") public enum NaturalTag { WATER, TREE, WOOD, WETLAND, SCRUB, COASTLINE, PEAK, CLIFF, GRASSLAND, TREE_ROW, HEATH, ROCK, BARE_ROCK, BEACH, SAND, SPRING, HOT_SPRING, GEYSER, LAND, BAY, SCREE, RIDGE, GLACIER, CAVE_ENTRANCE, SADDLE, MARSH, FELL, REEF, MUD, STONE, LANDFORM, SHINGLE, VALLEY, CAPE, VOLCANO, CREVASSE, SINKHOLE; @TagKey public static final String KEY = "natural"; public static Optional get(final Taggable taggable) { return Validators.from(NaturalTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/NetworkTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM network tag: While there are some conventions indicated on the OSM wiki, there is a wide * variety of possible values based on taginfo so this is subject to validation as any string. * * @author robert_stack */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/network#values", osm = "http://wiki.openstreetmap.org/wiki/Key:network") public interface NetworkTag { @TagKey String KEY = "network"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/NoExitTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM noexit tag * * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/noexit#values", osm = "http://wiki.openstreetmap.org/wiki/Key:noexit") public enum NoExitTag { YES; @TagKey public static final String KEY = "noexit"; public static Optional get(final Taggable taggable) { return Validators.from(NoExitTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/NotesTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM notes tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/notes#values", osm = "http://wiki.openstreetmap.org/wiki/Notes") public interface NotesTag { @TagKey String KEY = "notes"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/OpenDateTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM open_date tag * * @author brianjor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/open_date#values") public interface OpenDateTag { @TagKey String KEY = "open_date"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/OpeningDateTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM opening_date tag * * @author brianjor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/opening_date#values", osm = "https://wiki.openstreetmap.org/wiki/Key:opening_date") public interface OpeningDateTag { @TagKey String KEY = "opening_date"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/OpeningHoursTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.TagValue.ValueType; /** * OSM opening_hours tag * * @author mcuthbert * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/opening_hours#values", osm = "http://wiki.openstreetmap.org/wiki/Key:opening_hours") public interface OpeningHoursTag { @TagKey String KEY = "opening_hours"; @TagValue String ALL = "24/7"; @TagValue(ValueType.REGEX) String TWENTY_FOUR_SEVEN = "24[/x]7"; @TagValue(ValueType.REGEX) String DATE_AND_TIME = "(((Mo|Tu|We|Th|Fr|Sa|Su)(-?|,{0,6})){1,2}[_ ]?(([0-9]{1,2}:[0-9]{2}_?-_?[0-9]{1,2}:[0-9]{2}),? ?)+,?)+"; @TagValue(ValueType.REGEX) String TIME = "(([0-9]{1,2}:[0-9]{2}_?-_?[0-9]{1,2}:[0-9]{2}),? ?)+"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/OrganicTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM organic tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/organic#values", osm = "http://wiki.openstreetmap.org/wiki/Key:organic") public enum OrganicTag { ONLY, YES, NO, LIMITED; @TagKey public static final String KEY = "organic"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ParkingTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; /** * OSM parking tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/parking#values", osm = "http://wiki.openstreetmap.org/wiki/Key:parking") public enum ParkingTag { SURFACE, UNDERGROUND, @TagValueAs("multi-storey") MULTI_STOREY; @TagKey public static final String KEY = "parking"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/PhoneTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM phone tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/phone#values", osm = "http://wiki.openstreetmap.org/wiki/Key:phone") public interface PhoneTag { @TagKey String KEY = "phone"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/PlaceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM place tag * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/place#values", osm = "http://wiki.openstreetmap.org/wiki/Key:place") public enum PlaceTag { COUNTRY, STATE, REGION, PROVINCE, DISTRICT, COUNTY, MUNICIPALITY, CITY, BOROUGH, SUBURB, QUARTER, NEIGHBOURHOOD, CITY_BLOCK, PLOT, TOWN, VILLAGE, HAMLET, ISOLATED_DWELLING, FARM, ALLOTMENTS, CONTINENT, ARCHIPELAGO, ISLAND, ISLET, LOCALITY, SQUARE, OCEAN, SEA; @TagKey public static final String KEY = "place"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/PowerTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM power tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/power#values", osm = "http://wiki.openstreetmap.org/wiki/Key:power") public enum PowerTag { TOWER, POLE, LINE, GENERATOR, MINOR_LINE, SUBSTATION, SUB_STATION, TRANSFORMER, STATION, SWITCH, CABLE_DISTRIBUTION_CABINET, BUSBAR, PORTAL, CABLE, HELIOSTAT, CATENARY_MAST, PLANT, INSULATOR, SWITCHGEAR, COMPENSATOR, TERMINAL; @TagKey public static final String KEY = "power"; public static Optional get(final Taggable taggable) { return Validators.from(PowerTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ProtectClassTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.OrdinalExtractor; /** * OSM protect_class tag * * @author bbreithaupt */ @Tag(value = Validation.ORDINAL, range = @Range(min = 1, max = 99), taginfo = "https://taginfo.openstreetmap.org/keys/protect_class", osm = "https://wiki.openstreetmap.org/wiki/Key:protect_class") public interface ProtectClassTag { @TagKey String KEY = "protect_class"; static Optional getValue(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { final OrdinalExtractor extractor = new OrdinalExtractor(); return extractor.validateAndExtract(tagValue.get(), ProtectClassTag.class.getDeclaredAnnotation(Tag.class)); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/PublicServiceVehiclesTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Public Service Vehicles (psv) tag * * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/psv#values", osm = "http://wiki.openstreetmap.org/wiki/Key:psv") public enum PublicServiceVehiclesTag { YES, NO, BUS, OFFICIAL, DESIGNATED; @TagKey public static final String KEY = "psv"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/README.md ================================================ # Tags This package helps with OSM tag parsing and validation. ## Enum or Interface Each recognized tag has a java instance named after itself. When the tag values are set in the OSM wiki, and well defined, we use enums (for example, [HighwayTag](HighwayTag.java)). When the values are more loosely defined, like using ranges, we use interfaces and validation annotations (for example, [LanesTag](LanesTag.java)). ## Annotations and Validation The [annotations](annotations) and [validation](annotations/validation) packages define the annotation values used to identify the key and type of tags. For example, the [MaxSpeedTag](MaxSpeedTag.java) defines its Validation as a Speed, and annotates the tag's key as "maxspeed". ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/RailwayTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM's railway tag * * @author cstaylor * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/railway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:railway") public enum RailwayTag { RAIL, LEVEL_CROSSING, ABANDONED, SWITCH, TRAVERSER, BUFFER_STOP, STATION, PLATFORM, TRAM, DISUSED, CROSSING, SIGNAL, TRAM_STOP, SUBWAY, HALT, NARROW_GAUGE, MILESTONE, SUBWAY_ENTRANCE, TRAM_LEVEL_CROSSING, LIGHT_RAIL, STOP, PRESERVED, RAZED, TRAM_CROSSING, CONSTRUCTION, RAILWAY_CROSSING, DISMANTLED, PROPOSED, DERAIL, MINIATURE, TURNTABLE, MONORAIL, FUNICULAR; @TagKey public static final String KEY = "railway"; private static final EnumSet RAILWAY_CROSSINGS = EnumSet.of(CROSSING, LEVEL_CROSSING, TRAM_LEVEL_CROSSING, TRAM_CROSSING); public static Optional get(final Taggable taggable) { return Validators.from(RailwayTag.class, taggable); } public static boolean isRailway(final Taggable taggable) { return get(taggable).isPresent(); } public static boolean isRailwayCrossing(final Taggable taggable) { final Optional railway = get(taggable); return railway.isPresent() && RAILWAY_CROSSINGS.contains(railway.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/RampBicycleTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM ramp bicycle tag * * @author james_gage */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/ramp:bicycle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ramp:bicycle") public enum RampBicycleTag { YES, NO; @TagKey public static final String KEY = "ramp:bicycle"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/RelationTypeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Used internally to Atlas for determining the type of a relation * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/type#values", osm = "http://wiki.openstreetmap.org/wiki/Types_of_relation") public enum RelationTypeTag { MULTIPOLYGON, BUILDING, RESTRICTION, ROUTE, ROUTE_MASTER, BOUNDARY, LAND_AREA, SITE, ASSOCIATEDSTREET, PUBLIC_TRANSPORT, STREET, DESTINATION_SIGN, WATERWAY, ENFORCEMENT, BRIDGE, TUNNEL; @TagKey public static final String KEY = "type"; public static final String MULTIPOLYGON_TYPE = "multipolygon"; public static final String MULTIPOLYGON_ROLE_INNER = "inner"; public static final String MULTIPOLYGON_ROLE_OUTER = "outer"; public static final String BUILDING_ROLE_OUTLINE = "outline"; public static final String BUILDING_ROLE_PART = "part"; public static final String RESTRICTION_ROLE_FROM = "from"; public static final String RESTRICTION_ROLE_VIA = "via"; public static final String RESTRICTION_ROLE_TO = "to"; public static final String ADMINISTRATIVE_BOUNDARY_ROLE_SUB_AREA = "subarea"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/RouteTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM route tag * * @author robert_stack * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/route#values", osm = "http://wiki.openstreetmap.org/wiki/Relation:route") public enum RouteTag { BICYCLE, BUS, CANOE, DETOUR, FERRY, FITNESS_TRAIL, HIKING, HORSE, INLINE_SKATES, LIGHT_RAIL, MTB, NORDIC_WALKING, PIPELINE, PISTE, POWER, RAILWAY, ROAD, RUNNING, SKI, TRAIN, TRAM; @TagKey public static final String KEY = "route"; public static boolean isFerry(final Taggable taggable) { return Validators.isOfType(taggable, RouteTag.class, FERRY); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SaltTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM salt tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/salt#values", osm = "http://wiki.openstreetmap.org/wiki/Key:salt") public enum SaltTag { YES, NO, UNKNOWN; @TagKey public static final String KEY = "salt"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SeasonalTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM seasonal tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/seasonal#values", osm = "http://wiki.openstreetmap.org/wiki/Key:seasonal") public enum SeasonalTag { NO, YES, DRY_SEASON, WET_SEASON, SPRING, SUMMER, AUTUMN, WINTER; @TagKey public static final String KEY = "seasonal"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ServiceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; /** * OSM service tag. Does not include the documented but rarely used waterway related service tags. * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/service#values", osm = "http://wiki.openstreetmap.org/wiki/Key:service") public enum ServiceTag { PARKING_AISLE, DRIVEWAY, ALLEY, EMERGENCY_ACCESS, @TagValueAs("drive-through") DRIVE_THROUGH, SPUR, YARD, SIDING, CROSSOVER; @TagKey public static final String KEY = "service"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ShopTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM shop tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/shop#values", osm = "http://wiki.openstreetmap.org/wiki/Key:shop") public enum ShopTag { CONVENIENCE, SUPERMARKET, CLOTHES, HAIRDRESSER, BAKERY, CAR_REPAIR, CAR, YES, KIOSK, DOITYOURSELF, BUTCHER, FLORIST, MALL, FURNITURE, SHOES, BICYCLE, ALCOHOL, ELECTRONICS, HARDWARE, BOOKS, BEAUTY, MOBILE_PHONE, JEWELRY, DEPARTMENT_STORE, OPTICIAN, GIFT, GREENGROCER, CAR_PARTS, CHEMIST, VARIETY_STORE, SPORTS, GARDEN_CENTRE, COMPUTER, STATIONERY, TRAVEL_AGENCY, LAUNDRY, CONFECTIONERY, BEVERAGES, DRY_CLEANING, TOYS, TAILOR, ART, BABY_GOODS, BATHROOM_FURNISHING, BOUTIQUE, CARPET, CHEESE, CHOCOLATE, COPY_SHOP, COSMETICS, DAIRY, DELI, FABRIC, FARM, FASHION, VACANT; @TagKey public static final String KEY = "shop"; public String getTagValue() { return name().toLowerCase().intern(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SidewalkLeftTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM sidewalk left tag * * @author Vladimir Lemberg */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/sidewalk:left#values", osm = "https://wiki.openstreetmap.org/wiki/Key:sidewalk:left") public enum SidewalkLeftTag { NO, YES, SEPARATE; @TagKey public static final String KEY = "sidewalk:left"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SidewalkRightTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM sidewalk right tag * * @author Vladimir Lemberg */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/sidewalk:right#values", osm = "https://wiki.openstreetmap.org/wiki/Key:sidewalk:right") public enum SidewalkRightTag { NO, YES, SEPARATE; @TagKey public static final String KEY = "sidewalk:right"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SidewalkTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM sidewalk tag * * @author isabellehillberg */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/sidewalk#values", osm = "http://wiki.openstreetmap.org/wiki/Key:sidewalk") public enum SidewalkTag { BOTH, NONE, NO, RIGHT, LEFT, SEPARATE, YES; @TagKey public static final String KEY = "sidewalk"; public static boolean isNo(final Taggable taggable) { return Validators.isOfType(taggable, SidewalkTag.class, SidewalkTag.NO, SidewalkTag.NONE); } public static boolean isYes(final Taggable taggable) { return Validators.isOfType(taggable, SidewalkTag.class, SidewalkTag.BOTH, SidewalkTag.RIGHT, SidewalkTag.LEFT, SidewalkTag.SEPARATE, SidewalkTag.YES); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SkiTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Ski Tag. * * @author sayas01 */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/ski#values", osm = "https://wiki.openstreetmap.org/wiki/Key:ski") public enum SkiTag { NO, YES, DESIGNATED, OFFICIAL, PERMISSIVE, CROSSING, PRIVATE, DOWNHILL, CUSTOMERS, NORDIC; @TagKey public static final String KEY = "ski"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SmokingTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM smoking tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/smoking#values", osm = "http://wiki.openstreetmap.org/wiki/Key:smoking") public enum SmokingTag { NO, OUTSIDE, YES, SEPARATED, ISOLATED, DEDICATED, UNKNOWN; @TagKey public static final String KEY = "smoking"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SmoothnessTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM smoothness tag * * @author bbreithaupt */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/smoothness", osm = "https://wiki.openstreetmap.org/wiki/Key:smoothness") public enum SmoothnessTag { EXCELLENT, GOOD, INTERMEDIATE, BAD, VERY_BAD, HORRIBLE, VERY_HORRIBLE, IMPASSABLE; @TagKey public static final String KEY = "smoothness"; public boolean isLessImportantThan(final SmoothnessTag other) { return this.compareTo(other) > 0; } public boolean isLessImportantThanOrEqualTo(final SmoothnessTag other) { return this.compareTo(other) >= 0; } public boolean isMoreImportantThan(final SmoothnessTag other) { return this.compareTo(other) < 0; } public boolean isMoreImportantThanOrEqualTo(final SmoothnessTag other) { return this.compareTo(other) <= 0; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SnowmobileTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Snowmobile Tag. * * @author sayas01 */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/snowmobile#values", osm = "https://wiki.openstreetmap.org/wiki/Key:snowmobile") public enum SnowmobileTag { NO, DESIGNATED, YES, PERMISSIVE, PRIVATE; @TagKey public static final String KEY = "snowmobile"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SourceTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM source tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/source#values", osm = "http://wiki.openstreetmap.org/wiki/Key:source") public interface SourceTag { @TagKey String KEY = "source"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SourceTypeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM source:type tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/source%3Atype#values") public interface SourceTypeTag { @TagKey String KEY = "source:type"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SourceURLTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM source_url tag * * @author cstaylor */ @Tag(value = Validation.URI, taginfo = "http://taginfo.openstreetmap.org/keys/source%3Aurl#values") public interface SourceURLTag { @TagKey String KEY = "source:url"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SportTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM sports tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/sport#values") public enum SportTag { TENNIS, GOLF, SKI, SOCCER, BILLIARD, MULTI, BADMINTON, EQUESTRIAN, RUNNING, ATHLETICS, MODEL_AERODROME, MOTOR, FISHING, WINGSTUN, HOCKEY, SWIMMING; @TagKey public static final String KEY = "sport"; public static Optional get(final Taggable taggable) { return Validators.from(SportTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SurfaceTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM's surface tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/surface#values", osm = "http://wiki.openstreetmap.org/wiki/Key:surface") public enum SurfaceTag { ASPHALT, UNPAVED, PAVED, GRAVEL, GROUND, DIRT, GRASS, CONCRETE, PAVING_STONES, SAND, COMPACTED, COBBLESTONE, WOOD, FINE_GRAVEL, EARTH, PEBBLESTONE, SETT, MUD, GRASS_PAVER, METAL, GRAVEL_TURF, ICE, SALT, SNOW, WOODCHIPS, TARTAN, ARTIFICIAL_TURF, DECOTURF, CLAY, METAL_GRID, @TagValueAs("cobblestone:flattened") COBBLESTONE_FLATTENED, @TagValueAs("concrete:lanes") CONCRETE_LANES, @TagValueAs("concrete:plates") CONCRETE_PLATES; @TagKey public static final String KEY = "surface"; public static Optional get(final Taggable taggable) { return Validators.from(SurfaceTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SwimmingPoolTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM swimming_pool tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/swimming_pool#values") public enum SwimmingPoolTag { NO, YES, ABOVE, WADING, ENTRANCE, PLUNGE; @TagKey public static final String KEY = "swimming_pool"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticBoundaryNodeTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag for boundary nodes that are created at boundaries. *

* This is not an OSM tag. * * @author matthieun */ @Tag(synthetic = true) public enum SyntheticBoundaryNodeTag { YES, EXISTING; @TagKey public static final String KEY = "synthetic_boundary_node"; private static EnumSet ALL_BOUNDARY_NODES = EnumSet.of(YES, EXISTING); private static EnumSet SYNTHETIC_BOUNDARY_NODES = EnumSet.of(YES); private static EnumSet EXISTING_BOUNDARY_NODES = EnumSet.of(EXISTING); public static boolean isBoundaryNode(final Taggable taggable) { final Optional boundary = Validators .from(SyntheticBoundaryNodeTag.class, taggable); return boundary.isPresent() && ALL_BOUNDARY_NODES.contains(boundary.get()); } public static boolean isExistingBoundaryNode(final Taggable taggable) { final Optional boundary = Validators .from(SyntheticBoundaryNodeTag.class, taggable); return boundary.isPresent() && EXISTING_BOUNDARY_NODES.contains(boundary.get()); } public static boolean isSyntheticBoundaryNode(final Taggable taggable) { final Optional boundary = Validators .from(SyntheticBoundaryNodeTag.class, taggable); return boundary.isPresent() && SYNTHETIC_BOUNDARY_NODES.contains(boundary.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticDuplicateOsmNodeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag identifying an OSM Node that contains duplicate Node on top of it. This usually signifies a * data error. This is NOT an OSM tag. * * @author mgostintsev */ @Tag(synthetic = true) public enum SyntheticDuplicateOsmNodeTag { YES; @TagKey public static final String KEY = "synthetic_duplicate_osm_node"; public static boolean isYes(final Taggable taggable) { return Validators.isOfType(taggable, SyntheticDuplicateOsmNodeTag.class, YES); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticGeometrySlicedTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag indicating an entity had its geometry sliced during country slicing *

* This is not an OSM tag. * * @author samg */ @Tag(synthetic = true) public enum SyntheticGeometrySlicedTag { YES; @TagKey public static final String KEY = "synthetic_geometry_sliced"; public static boolean isGeometrySliced(final Taggable taggable) { return Validators.from(SyntheticGeometrySlicedTag.class, taggable).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticInvalidGeometryTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag indicating an entity does not conform to the OGC geometry specification. *

* This is not an OSM tag. * * @author samg */ @Tag(synthetic = true) public enum SyntheticInvalidGeometryTag { YES; @TagKey public static final String KEY = "synthetic_invalid_geometry"; public static boolean isInvalidGeometry(final Taggable taggable) { return Validators.from(SyntheticInvalidGeometryTag.class, taggable).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticInvalidMultiPolygonRelationMembersRemovedTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Tag that tracks any invalid multipolygon members that were removed during country slicing *

* This is not an OSM tag. * * @author samg */ @Tag(synthetic = true, value = Validation.NON_EMPTY_STRING) public interface SyntheticInvalidMultiPolygonRelationMembersRemovedTag { @TagKey String KEY = "synthetic_invalid_multipolygon_relation_members_removed"; String MEMBER_DELIMITER = ","; /** * The list of all added member identifiers * * @param taggable * The {@link Taggable} whose members we're interested in * @return Iterable of all the added member identifiers for this item */ static Optional> all(final Taggable taggable) { return taggable.getTag(KEY).map(tagValue -> Arrays.stream(tagValue.split(MEMBER_DELIMITER)) .map(Long::valueOf).filter(Objects::nonNull).collect(Collectors.toList())); } /** * @param taggable * The {@link Taggable} we're looking at * @return {@code true} if this item has removed invalid relation members */ static boolean hasAddedRelationMember(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } static String join(final Collection ids) { return String.join(MEMBER_DELIMITER, ids); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticInvalidWaySectionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag identifying an Atlas Edge that was the remnant of way-sectioning that exceeded the maximum * 999 slices. As a result, this edge contains the rest of the un-sectioned OSM Way. This usually * indicates a data error and is NOT an OSM tag. * * @author mgostintsev */ @Tag(synthetic = true) public enum SyntheticInvalidWaySectionTag { YES; @TagKey public static final String KEY = "synthetic_invalid_way_section"; public static boolean isYes(final Taggable taggable) { return Validators.isOfType(taggable, SyntheticInvalidWaySectionTag.class, YES); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticRelationMemberAdded.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * Tag that tracks any {@link Relation} member identifiers that were added during country-slicing. *

* This is not an OSM tag. * * @author mgostintsev */ @Tag(synthetic = true, value = Validation.NON_EMPTY_STRING) public interface SyntheticRelationMemberAdded { @TagKey String KEY = "synthetic_relation_member_added"; String MEMBER_DELIMITER = ","; /** * The list of all added member identifiers * * @param taggable * The {@link Taggable} whose members we're interested in * @return Iterable of all the added member identifiers for this item */ static Optional> all(final Taggable taggable) { return taggable.getTag(KEY).map(tagValue -> Arrays.stream(tagValue.split(MEMBER_DELIMITER)) .map(Long::valueOf).filter(Objects::nonNull).collect(Collectors.toList())); } /** * @param taggable * The {@link Taggable} we're looking at * @return {@code true} if this item has added relation members */ static boolean hasAddedRelationMember(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } static String join(final Collection ids) { return String.join(MEMBER_DELIMITER, ids); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticRelationRoleUpdated.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Tag that tracks any {@link Relation} member role update. An example is an inner role changed to * an outer role. The format will be {relation-member-identifier}|{RoleChange}. *

* This is not an OSM tag. * * @author mgostintsev */ @Tag(synthetic = true, value = Validation.NON_EMPTY_STRING) public interface SyntheticRelationRoleUpdated { /** * @author mgostintsev */ enum RoleChange { INNER_TO_OUTER, OUTER_TO_INNER; static RoleChange safeValueOf(final String value) { try { return valueOf(value.toUpperCase()); } catch (final IllegalArgumentException ignored) { return null; } } } @TagKey String KEY = "synthetic_relation_role_updated"; String ROLE_CHANGE_IDENTIFIER_DELIMITER = "|"; /** * The optional Tuple of the member identifier and type of {@link RoleChange} * * @param taggable * The {@link Taggable} we're interested in * @return the optional tuple of the member and type of {@link RoleChange} */ static Optional> all(final Taggable taggable) { final Optional possibleUpdatedRole = taggable .getTag(SyntheticRelationRoleUpdated.class, Optional.empty()); if (possibleUpdatedRole.isPresent()) { final List updatedRole = Arrays .asList(possibleUpdatedRole.get().split(ROLE_CHANGE_IDENTIFIER_DELIMITER)); return Optional.of(Tuple.createTuple(Long.valueOf(updatedRole.get(0)), RoleChange.safeValueOf(updatedRole.get(1)))); } return Optional.empty(); } /** * @param taggable * The {@link Taggable} we're looking at * @return {@code true} if this item has added relation members */ static boolean hasUpdatedRelationMemberRole(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/SyntheticSyntheticRelationMemberTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag to indicate an entity is a synthetic relation member added during country slicing. *

* This is not an OSM tag. * * @author samg */ @Tag(synthetic = true) public enum SyntheticSyntheticRelationMemberTag { YES; @TagKey public static final String KEY = "synthetic_relation_member"; public static boolean isSyntheticRelationMember(final Taggable taggable) { return Validators.from(SyntheticSyntheticRelationMemberTag.class, taggable).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/Taggable.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.locale.IsoLanguage; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.osmosis.core.domain.v0_6.Tag; /** * Any class that should expose OSM tags can implement this interface for more complex interactions * with tag validation. * * @author cstaylor * @author matthieun */ public interface Taggable { /** * Options for the localized tag search: *

    *
  • DEFAULT - return the non-localized value of a tag if the localized value for a given * language is not found
    • *
  • LOCALIZED_ONLY - only return the localized value if the tag is a localized tag for a * given language; empty otherwise
    • *
  • FORCE_ALL_LOCALIZED_ONLY - same as LOCALIZED_ONLY but treat non-localized tags as * localizable
    • *
* * @author cstaylor */ enum TagSearchOption { DEFAULT, LOCALIZED_ONLY, FORCE_ALL_LOCALIZED_ONLY } static Taggable with(final Collection tagCollection) { final Map tags = new HashMap<>(); tagCollection.forEach(tag -> tags.put(tag.getKey(), tag.getValue())); return with(tags); } static Taggable with(final Map tags) { return new Taggable() { @Override public Optional getTag(final String key) { return Optional.ofNullable(tags.get(key)); } @Override public Map getTags() { return tags; } @Override public String toString() { return tags.toString(); } }; } static Taggable with(final String... tags) { return with(Maps.hashMap(tags)); } /** * Utility function to test if an entity's tag value starts with some given values. * * @param key * The tag key * @param matches * The matching values * @return True if the tag's value matches at least one of the matching values. */ default boolean containsValue(final String key, final Iterable matches) { final Optional valueOption = getTag(key); if (valueOption.isPresent()) { final String value = valueOption.get(); for (final String candidate : matches) { if (candidate.startsWith("!")) { if (candidate.length() > 1) { final String forbiddenValue = candidate.substring(1); if (!value.equalsIgnoreCase(forbiddenValue)) { return true; } } else { return false; } } if ("*".equals(candidate) || value.equalsIgnoreCase(candidate)) { return true; } if (candidate != null && candidate.startsWith("*") && value.endsWith(candidate.substring(1))) { return true; } if (candidate != null && candidate.endsWith("*") && value.startsWith(candidate.substring(0, candidate.length() - 1))) { return true; } } } else { for (final String candidate : matches) { if (!candidate.startsWith("!")) { return false; } } return true; } return false; } /** * Neat single place where we can get tags by their language if they exist, otherwise return the * default without the localized name * * @param key * the base key we want to find * @param language * the optional language's version of the tag we want to find * @param searchOptions * search options for finding the value of a key. Sometimes we only want to check the * localized level and not bring in the non-localized value * @return the value of the localized version of the tag if it exists, the value of the base key * if the localized one is missing, or an empty optional if neither are available */ default Optional getTag(final Class key, final Optional language, final TagSearchOption... searchOptions) { final EnumSet searchOptionSet = searchOptions.length > 0 ? EnumSet.copyOf(Arrays.asList(searchOptions)) : EnumSet.noneOf(TagSearchOption.class); final Optional localizedKeyName = Validators.localizeKeyName(key, language, searchOptions); if (localizedKeyName.isPresent()) { final Optional localizedValue = getTag(localizedKeyName.get()); if (localizedValue.isPresent() || searchOptionSet.contains(TagSearchOption.LOCALIZED_ONLY) || searchOptionSet.contains(TagSearchOption.FORCE_ALL_LOCALIZED_ONLY)) { return localizedValue; } final Optional optionalKey = Validators.localizeKeyName(key, Optional.empty()); if (optionalKey.isPresent()) { return getTag(optionalKey.get()); } } return Optional.empty(); } Optional getTag(String key); /** * Some taggables support fetching all keys, some don't * * @return all of the tag keys and their values */ default Map getTags() { return new HashMap<>(); } /** * Will retrieve tags based on a filter * * @param filter * The predicate to test each tag by * @return The map of filtered tags. */ default Map getTags(final Predicate filter) { return this.getTags().entrySet().stream().filter(item -> filter.test(item.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } /** * @param tags * A tag map to compare to * @return True if this contains at least one tag specified in the tag map */ default boolean hasAtLeastOneOf(final Map tags) { for (final Map.Entry entry : tags.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); final Optional myValue = getTag(key); if (myValue.isPresent()) { if ("*".equals(value)) { return true; } if (value.equals(myValue.get())) { return true; } } } return false; } /** * Return the set of languages explicitly set on the tag provided. * * @param tag * check this tag on this Taggable for explicitly defined languages * @param searchOptions * optional list of flags that alter the behavior of the underlying tag value search * @return the optional set of explicitly listed languages found */ default Optional> languagesFor(final Class tag, final TagSearchOption... searchOptions) { final TreeSet returnValue = new TreeSet<>(); final EnumSet searchOptionSet = searchOptions.length > 0 ? EnumSet.copyOf(Arrays.asList(searchOptions)) : EnumSet.noneOf(TagSearchOption.class); if (!searchOptionSet.contains(TagSearchOption.FORCE_ALL_LOCALIZED_ONLY) && !Validators.hasLocalizedTagKey(tag)) { throw new CoreException("{} isn't a localizable tag", tag.getName()); } final String prefix = Validators.TagKeySearch.findTagKeyIn(tag) .orElseThrow( () -> new CoreException("Could not find key for tag {}", tag.getName())) .getKeyName(); final int prefixLength = prefix.length(); for (final String key : this.getTags().keySet()) { if (key.startsWith(prefix) && key.length() > prefixLength) { final LocalizedTagNameWithOptionalDate parser = new LocalizedTagNameWithOptionalDate( key); parser.getLanguage().ifPresent(returnValue::add); } } return Optional.of(returnValue); } /** * Get the value for this tag key * * @param key * The tag key * @return The value. null if the value does not exist */ default String tag(final String key) { return getTag(key).orElse(null); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TemporaryDateOnTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM temporary:date_on tag * * @author brianjor */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/temporary:date_on#values", osm = "https://wiki.openstreetmap.org/wiki/Item:Q15233") public interface TemporaryDateOnTag { @TagKey String KEY = "temporary:date_on"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TollTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM toll tag * * @author robert_stack * @author pmi */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/?key=toll#values", osm = "http://wiki.openstreetmap.org/wiki/Key:toll") public enum TollTag { YES, NO, SNOWMOBILE; @TagKey public static final String KEY = "toll"; public static boolean isYes(final Taggable taggable) { final Taggable checkMe = taggable; return Validators.isOfType(checkMe, TollTag.class, TollTag.YES); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TourismTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM tourism tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/tourism#values", osm = "http://wiki.openstreetmap.org/wiki/Key:tourism") public enum TourismTag { INFORMATION, HOTEL, ATTRACTION, VIEWPOINT, PICNIC_SITE, CAMP_SITE, GUEST_HOUSE, MUSEUM, ARTWORK, CHALET, MOTEL, HOSTEL, CARAVAN_SITE, ALPINE_HUT, THEME_PARK, ZOO, YES, APARTMENT, WILDERNESS_HUT, GALLERY, BED_AND_BREAKFAST, WINE_CELLAR, RESORT, AQUARIUM; @TagKey public static final String KEY = "tourism"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TracktypeTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM tracktype tag * * @author robert_stack */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/tracktype#values", osm = "http://wiki.openstreetmap.org/wiki/Key:tracktype") public enum TracktypeTag { GRADE1, GRADE2, GRADE3, GRADE4, GRADE5; @TagKey public static final String KEY = "tracktype"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TrafficCalmingTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM traffic_calming tag * * @author mgostintsev */ @Tag(taginfo = "http://wiki.openstreetmap.org/wiki/Key:traffic_calming#Common_values", osm = "http://wiki.openstreetmap.org/wiki/Key:traffic_calming") public enum TrafficCalmingTag { BUMP, HUMP, TABLE, CUSHION, RUMBLE_STRIP, CHICANE, CHOKER, ISLAND; @TagKey public static final String KEY = "traffic_calming"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TunnelTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM tunnel tag * * @author robert_stack * @author pmi */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/tunnel#values", osm = "http://wiki.openstreetmap.org/wiki/Key:tunnel") public enum TunnelTag { YES, CULVERT, BUILDING_PASSAGE, FLOODED, NO; private static final EnumSet TUNNEL_WAYS = EnumSet.of(YES, CULVERT, BUILDING_PASSAGE, FLOODED); @TagKey public static final String KEY = "tunnel"; public static boolean isTunnel(final Taggable taggable) { final Optional tunnel = Validators.from(TunnelTag.class, taggable); return tunnel.isPresent() && TUNNEL_WAYS.contains(tunnel.get()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TurnLanesBackwardTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM turn:lanes:backward tag indicating the lane types for a two-way * * @author brian_l_davis */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/turn%3Alanes%3Abackward#values", osm = "https://wiki.openstreetmap.org/wiki/Key:turn") public interface TurnLanesBackwardTag extends TurnLanesTag { @TagKey String KEY = "turn:lanes:backward"; /** * The list of turn types specified in the turn:lanes:backward tag * * @param taggable * The taggable object being test * @return The list of turn types when tagged */ static Optional>> getBackwardTurnLanes(final Taggable taggable) { return taggable.getTag(KEY) .map(tagValue -> Arrays.stream(tagValue.split(TURN_LANE_DELIMITER)) .map(lane -> Arrays.stream(lane.split(TURN_TYPE_DELIMITER)) .map(TurnType::safeValueOf).filter(Objects::nonNull) .collect(Collectors.toSet())) .collect(Collectors.toList())); } /** * Checks if the {@link Taggable} has the {@link TurnType} in the {@code turn:lanes:backward} * tags. * * @param taggable * The taggable object being test * @param turnType * A turn type to test for * @return {@code true} if tagged with the turn type, otherwise {@code false} */ static boolean hasBackwardTurnLane(final Taggable taggable, final TurnType turnType) { return getBackwardTurnLanes(taggable) .map(lanes -> lanes.stream().anyMatch(turnLane -> turnLane.contains(turnType))) .orElse(false); } /** * Checks if the {@link Taggable} has a {@code turn:lanes:backward} tag value. * * @param taggable * The taggable object being test * @return {@code true} if tagged, otherwise {@code false} */ static boolean hasBackwardTurnLane(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TurnLanesForwardTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM turn:lanes:forward tag indicating the lane types for a two-way * * @author brian_l_davis */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/turn%3Alanes%3Aforward#values", osm = "https://wiki.openstreetmap.org/wiki/Key:turn") public interface TurnLanesForwardTag extends TurnLanesTag { @TagKey String KEY = "turn:lanes:forward"; /** * The list of turn types specified in the turn:lanes:forward tag * * @param taggable * The taggable object being test * @return The list of turn types when tagged */ static Optional>> getForwardTurnLanes(final Taggable taggable) { return taggable.getTag(KEY) .map(tagValue -> Arrays.stream(tagValue.split(TURN_LANE_DELIMITER)) .map(lane -> Arrays.stream(lane.split(TURN_TYPE_DELIMITER)) .map(TurnType::safeValueOf).filter(Objects::nonNull) .collect(Collectors.toSet())) .collect(Collectors.toList())); } /** * Checks if the {@link Taggable} has the {@link TurnType} in the {@code turn:lanes:forward} * tags. * * @param taggable * The taggable object being test * @param turnType * A turn type to test for * @return {@code true} if tagged with the turn type, otherwise {@code false} */ static boolean hasForwardTurnLane(final Taggable taggable, final TurnType turnType) { return getForwardTurnLanes(taggable) .map(lanes -> lanes.stream().anyMatch(turnLane -> turnLane.contains(turnType))) .orElse(false); } /** * Checks if the {@link Taggable} has a {@code turn:lanes:forward} tag value. * * @param taggable * The taggable object being test * @return {@code true} if tagged, otherwise {@code false} */ static boolean hasForwardTurnLane(final Taggable taggable) { return taggable.getTag(KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TurnLanesTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.OptionalIterable; /** * OSM turn:lanes tag indicating the lane types for a one-way road * * @author brian_l_davis */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/turn%3Alanes#values", osm = "https://wiki.openstreetmap.org/wiki/Key:turn") public interface TurnLanesTag extends TurnTag { @TagKey String KEY = "turn:lanes"; /** * The list of turn types specified in the turn:lanes tag * * @param taggable * The taggable object being test * @return The list of turn types when tagged */ static Optional>> getTurnLanes(final Taggable taggable) { return taggable.getTag(KEY) .map(tagValue -> Arrays.stream(tagValue.split(TURN_LANE_DELIMITER)) .map(lane -> Arrays.stream(lane.split(TURN_TYPE_DELIMITER)) .map(TurnType::safeValueOf).filter(Objects::nonNull) .collect(Collectors.toSet())) .collect(Collectors.toList())); } /** * Checks if the {@link Taggable} has the {@link TurnType} in any of the * {@code turn:lanes[:forward|:backward]} tags. * * @param taggable * The taggable object being test * @param turnType * A turn type to test for * @return {@code true} if tagged with the turn type, otherwise {@code false} */ @SuppressWarnings("unchecked") static boolean hasTurnLane(final Taggable taggable, final TurnType turnType) { final OptionalIterable>> turnLanes = new OptionalIterable<>(Iterables .iterable(getTurnLanes(taggable), TurnLanesForwardTag.getForwardTurnLanes(taggable), TurnLanesBackwardTag.getBackwardTurnLanes(taggable))); return Iterables.asList(turnLanes).stream().anyMatch( lanes -> lanes.stream().anyMatch(turnLane -> turnLane.contains(turnType))); } /** * Checks if the {@link Taggable} has a {@code turn[:lanes[:forward|:backward]]} tag value. * * @param taggable * The taggable object being test * @return {@code true} if tagged, otherwise {@code false} */ static boolean hasTurnLane(final Taggable taggable) { return taggable.getTag(TurnLanesTag.KEY).isPresent() || taggable.getTag(TurnLanesForwardTag.KEY).isPresent() || taggable.getTag(TurnLanesBackwardTag.KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TurnRestrictionTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.EnumSet; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * Tag for a turn restriction relation * * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/restriction#values", osm = "http://wiki.openstreetmap.org/wiki/Relation:restriction") public enum TurnRestrictionTag { NO_RIGHT_TURN, NO_LEFT_TURN, NO_U_TURN, NO_STRAIGHT_ON, ONLY_RIGHT_TURN, ONLY_LEFT_TURN, ONLY_STRAIGHT_ON, NO_ENTRY, NO_EXIT; @TagKey public static final String KEY = "restriction"; private static final EnumSet NO_PATH_RESTRICTIONS = EnumSet .of(NO_RIGHT_TURN, NO_LEFT_TURN, NO_U_TURN, NO_STRAIGHT_ON, NO_ENTRY, NO_EXIT); private static final EnumSet ONLY_PATH_RESTRICTIONS = EnumSet .of(ONLY_RIGHT_TURN, ONLY_LEFT_TURN, ONLY_STRAIGHT_ON); public static boolean isNoPathRestriction(final Taggable taggable) { final Optional turnRestriction = Validators .from(TurnRestrictionTag.class, taggable); return turnRestriction.isPresent() && NO_PATH_RESTRICTIONS.contains(turnRestriction.get()); } public static boolean isOnlyPathRestriction(final Taggable taggable) { final Optional turnRestriction = Validators .from(TurnRestrictionTag.class, taggable); return turnRestriction.isPresent() && ONLY_PATH_RESTRICTIONS.contains(turnRestriction.get()); } public static boolean isRestriction(final Taggable taggable) { final Optional turnRestriction = Validators .from(TurnRestrictionTag.class, taggable); return turnRestriction.isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/TurnTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.OptionalIterable; /** * Base OSM turn tag indicating a turn direction * * @author brian_l_davis */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/turn#values", osm = "https://wiki.openstreetmap.org/wiki/Key:turn") public interface TurnTag { /** * @author brian_l_davis */ enum TurnType { LEFT, SHARP_LEFT, SLIGHT_LEFT, THROUGH, RIGHT, SHARP_RIGHT, SLIGHT_RIGHT, REVERSE, MERGE_TO_LEFT, MERGE_TO_RIGHT, NONE; static final EnumSet leftTurn = EnumSet.of(LEFT, SHARP_LEFT, SLIGHT_LEFT, MERGE_TO_LEFT); static final EnumSet rightTurn = EnumSet.of(RIGHT, SHARP_RIGHT, SLIGHT_RIGHT, MERGE_TO_RIGHT); static TurnType safeValueOf(final String value) { try { return valueOf(value.toUpperCase()); } catch (final IllegalArgumentException ignored) { return null; } } } @TagKey String KEY = "turn"; String TURN_LANE_DELIMITER = "\\|"; String TURN_TYPE_DELIMITER = ";"; /** * The list of turn types specified in the turn tag * * @param taggable * The taggable object being test * @return The list of turn types when tagged */ static Optional>> getTurns(final Taggable taggable) { return taggable.getTag(KEY) .map(tagValue -> Arrays.stream(tagValue.split(TURN_LANE_DELIMITER)) .map(lane -> Arrays.stream(lane.split(TURN_TYPE_DELIMITER)) .map(TurnType::safeValueOf).filter(Objects::nonNull) .collect(Collectors.toSet())) .collect(Collectors.toList())); } /** * Checks if the {@link Taggable} has the {@link TurnType} in any of the * {@code turn[:lanes[:forward|:backward]]} tags. * * @param taggable * The taggable object being test * @param turnType * A turn type to test for * @return {@code true} if tagged with the turn type, otherwise {@code false} */ @SuppressWarnings("unchecked") static boolean hasTurn(final Taggable taggable, final TurnType turnType) { final OptionalIterable>> turnLanes = new OptionalIterable<>( Iterables.iterable(getTurns(taggable), TurnLanesTag.getTurnLanes(taggable), TurnLanesForwardTag.getForwardTurnLanes(taggable), TurnLanesBackwardTag.getBackwardTurnLanes(taggable))); return Iterables.asList(turnLanes).stream().anyMatch( lanes -> lanes.stream().anyMatch(turnLane -> turnLane.contains(turnType))); } /** * Checks if the {@link Taggable} has a {@code turn[:lanes[:forward|:backward]]} tag value. * * @param taggable * The taggable object being test * @return {@code true} if tagged, otherwise {@code false} */ static boolean hasTurn(final Taggable taggable) { return taggable.getTag(TurnTag.KEY).isPresent() || taggable.getTag(TurnLanesTag.KEY).isPresent() || taggable.getTag(TurnLanesForwardTag.KEY).isPresent() || taggable.getTag(TurnLanesBackwardTag.KEY).isPresent(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/URLTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM url tag * * @author cstaylor */ @Tag(value = Validation.URI, taginfo = "http://taginfo.openstreetmap.org/keys/url#values", osm = "http://wiki.openstreetmap.org/wiki/Key:url") public interface URLTag { @TagKey String KEY = "url"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/UsageTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM usage tag * * @author jacker */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/usage#values", osm = "http://wiki.openstreetmap.org/wiki/Key:usage") public enum UsageTag { MAIN, BRANCH, INDUSTRIAL, FREIGHT, TOURISM, MILITARY, YARD, STOCK, DISTRIBUTION, FACILITY, TRANSMISSION, TEST; @TagKey public static final String KEY = "usage"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/VacantTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM vacant tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/vacant#values") public enum VacantTag { YES, SELL, RENT, RESTAURANT, RENTAL, NO, PUB, CONSTRUCTION, SHOP; @TagKey public static final String KEY = "vacant"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/VehicleTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM vehicle tag * * @author mkalender */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/vehicle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:vehicle") public enum VehicleTag { YES, NO, PRIVATE, AGRICULTURAL, PERMISSIVE, DESTINATION, DESIGNATED, DELIVERY, FORESTRY, OFFICIAL, CUSTOMERS, EMERGENCY, BUS, SERVICE, UNKNOWN; @TagKey public static final String KEY = "vehicle"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/VendingTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM's vending tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/vending#values", osm = "http://wiki.openstreetmap.org/wiki/Tag:amenity%3Dvending_machine") public enum VendingTag { PARKING_TICKETS, CIGARETTES, EXCREMENT_BAGS, PUBLIC_TRANSPORT_TICKETS, DRINKS, SWEETS, PARCEL_PICKUP, PARCEL_MAIL_IN, CONDOMS, NEWS_PAPERS, STAMPS, BICYCLE_TUBE, FUEL, GAS; @TagKey public static final String KEY = "vending"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WaterTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM water tag * * @author Sid * @author cstaylor * @author mgostintsev */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/water#values", osm = "http://wiki.openstreetmap.org/wiki/Key:water") public enum WaterTag { INTERMITTENT, LAKE, LAGOON, POND, REFLECTING_POOL, RESERVOIR, BASIN, CANAL, RIVER, FISH_PASS, OXBOW, LOCK, MOAT, WASTEWATER, STREAM_POOL, SEA, TIDAL, SALT_POOL, POOL; @TagKey public static final String KEY = "water"; public static Optional get(final Taggable taggable) { return Validators.from(WaterTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WaterwayTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM waterway tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/waterway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:waterway") public enum WaterwayTag { STREAM, DITCH, RIVER, DRAIN, RIVERBANK, CANAL, DAM, WEIR, RAPIDS, WATERFALL, LOCK_GATE, WADI, DRYSTREAM, DOCK, BOATYARD, DERELICT_CANAL, MILESTONE, BROOK, TURNING_POINT, FUEL, FISH_PASS, WATER_POINT; @TagKey public static final String KEY = "waterway"; public static Optional get(final Taggable taggable) { return Validators.from(WaterwayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WebsiteTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM website tag * * @author cstaylor */ @Tag(value = Validation.URI, taginfo = "http://taginfo.openstreetmap.org/keys/website#values", osm = "http://wiki.openstreetmap.org/wiki/Key:website") public interface WebsiteTag { @TagKey String KEY = "website"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WetlandTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM wetland tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/wetland#values", osm = "http://wiki.openstreetmap.org/wiki/Key:wetland") public enum WetlandTag { BOG, MARSH, SWAMP, REEDBED, TIDALFLAT, MANGROVE, WET_MEADOW, SALTMARSH, STRING_BOG, SALTERN, FEN; @TagKey public static final String KEY = "wetland"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WheelchairDescriptionTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM wheelchair:description tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/wheelchair:description#values") public interface WheelchairDescriptionTag { @TagKey String KEY = "wheelchair:description"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WheelchairTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM wheelchair tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/wheelchair#values", osm = "http://wiki.openstreetmap.org/wiki/Key%3Awheelchair") public enum WheelchairTag { YES, NO, LIMITED; @TagKey public static final String KEY = "wheelchair"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WidthTag.java ================================================ package org.openstreetmap.atlas.tags; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.extraction.LengthExtractor; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * OSM width tag * * @author cstaylor * @author bbreithaupt */ @Tag(value = Validation.DOUBLE, range = @Range(min = 0, max = Integer.MAX_VALUE), taginfo = "http://taginfo.openstreetmap.org/keys/width#values", osm = "http://wiki.openstreetmap.org/wiki/Key:width") public interface WidthTag { @TagKey String KEY = "width"; static Optional get(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return LengthExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WifiTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM wifi tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/wifi#values", osm = "http://wiki.openstreetmap.org/wiki/Key:internet_access") public interface WifiTag { @TagKey String KEY = "wifi"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WikidataTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.TagValue.ValueType; /** * OSM wikidata tag * * @author cstaylor */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/wikidata#values", osm = "http://wiki.openstreetmap.org/wiki/Key:wikidata") public interface WikidataTag { @TagKey(KeyType.LOCALIZED) String KEY = "wikidata"; @TagValue(ValueType.REGEX) String WIKI_DATA = "Q[1-9]\\d*"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WikipediaTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM wikipedia tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/wikipedia#values", osm = "http://wiki.openstreetmap.org/wiki/Key:wikipedia") public interface WikipediaTag { @TagKey(KeyType.LOCALIZED) String KEY = "wikipedia"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/WinterRoadTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM Winter Tag. * * @author sayas01 */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/winter_road#values", osm = "https://wiki.openstreetmap.org/wiki/Key:winter_road") public enum WinterRoadTag { YES, NO; @TagKey public static final String KEY = "winter_road"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/ZooTag.java ================================================ package org.openstreetmap.atlas.tags; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM zoo tag * * @author stephencerqueira */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/zoo#values", osm = "http://wiki.openstreetmap.org/wiki/Key%zoo") public enum ZooTag { AVIARY, BIRDS, ENCLOSURE, FALCONRY, PETTING_ZOO, REPTILE, SAFARI_PARK, WILDLIFE_PARK; @TagKey public static final String KEY = "zoo"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/Tag.java ================================================ package org.openstreetmap.atlas.tags.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * Java annotation for Atlas tags including how we should validate their values at runtime * * @author cstaylor */ @Target(value = ElementType.TYPE_USE) @Retention(value = RetentionPolicy.RUNTIME) public @interface Tag { /** * This is an optional range of values that can be used in numeric validators * * @author cstaylor */ public @interface Range { // These are values that shouldn't be allowed, even if they fall inside the range long[] exclude() default {}; // Values aren't mixed up: these are sentinel values that disable the range long max() default Long.MIN_VALUE; long min() default Long.MAX_VALUE; } /** * Validation rules for the values of a tag: *
    *
  • MATCH - an exact match against a set of values
    • *
  • ORDINAL - a positive integer. Also implies MATCH for any defined TagValues in the Tag *
    • *
  • LONG - a long of any value constrained by the optional range. Also implies MATCH for any * defined TagValues in the Tag
    • *
  • DOUBLE - parse the value as a double. Also implies MATCH for any defined TagValues in the * Tag
    • *
  • TIMESTAMP - parse the value as a long and create a Date from that value. Also implies * MATCH for any defined TagValues in the Tag
    • *
  • NON_EMPTY_STRING - valid if the value has at least one non-whitespace character. Does NOT * imply MATCH
    • *
  • ISO_COUNTRY - valid if the value matches an entry in the ISO country code list. Does NOT * imply MATCH
    • *
  • NONE - no validation performed
    • *
  • URI - check if the value if a well-formed URI
    • *
  • SPEED - check if the value is a well-formed speed
    • *
  • LENGTH - check if the value is a well-formed length
    • *
* * @author cstaylor */ enum Validation { MATCH, ORDINAL, LONG, DOUBLE, TIMESTAMP, NON_EMPTY_STRING, ISO3_COUNTRY, ISO2_COUNTRY, NONE, URI, SPEED, LENGTH; } /** * Optional URL to OSM wiki page for this tag * * @return the URL or an empty string if no tag is defined */ String osm() default ""; Range range() default @Range(); /** * If true, this tag is an artifact of processing and does not exist in OSM itself. * * @return true if this tag has been marked as synthetic, false otherwise */ boolean synthetic() default false; /** * Optional URL to taginfo site for this tag * * @return the URL or an empty string if no tag is defined */ String taginfo() default ""; /** * The validation rule for this particular Atlas Tag * * @return the defined Validation rule: defaults to MATCH */ Validation value() default Validation.MATCH; /** * This lets tags use shared enum values. For example shop and disused:shop * * @return the optional array of enums to copy their names for exact matches */ Class>[] with() default {}; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/TagKey.java ================================================ package org.openstreetmap.atlas.tags.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * Java annotation for an Atlas tag's key field. We can find this value at runtime using java * introspection without resorting to idioms like 'KEY_' or '_TAG' * * @author cstaylor */ @Retention(value = RetentionPolicy.RUNTIME) @Target(value = ElementType.FIELD) public @interface TagKey { /** * Some keys in OSM can be localized (for example, wikipedia:[language ISO2 code]) * * @author cstaylor */ enum KeyType { EXACT, LOCALIZED } KeyType value() default KeyType.EXACT; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/TagValue.java ================================================ package org.openstreetmap.atlas.tags.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * Java annotation for constants in an Atlas tag definition for Tag.Validation.MATCH rules * * @author cstaylor */ @Retention(value = RetentionPolicy.RUNTIME) @Target(value = ElementType.FIELD) public @interface TagValue { /** * We can support exact string match values or regular expression patterns. The default is exact * string matches * * @author cstaylor */ enum ValueType { EXACT, REGEX; } ValueType value() default ValueType.EXACT; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/TagValueAs.java ================================================ package org.openstreetmap.atlas.tags.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * Annotation for changing how Validators will interpret a tag's value. Only applies for enums * constants since their values are usually directly lower-cased. * * @author cstaylor */ @Retention(value = RetentionPolicy.RUNTIME) @Target(value = ElementType.FIELD) public @interface TagValueAs { boolean deprecated() default false; String value(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/TagValueDeprecated.java ================================================ package org.openstreetmap.atlas.tags.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * If a particular TagValue shouldn't be used anymore, we can mark it with this annotation. In the * future the tag integrity check can use this information to flag tag values as obsolete * * @author cstaylor */ @Retention(value = RetentionPolicy.RUNTIME) @Target(value = ElementType.FIELD) public @interface TagValueDeprecated { boolean value() default true; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/AltitudeExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.geography.Altitude; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * Extracts a {@link Altitude} from a value. Can be used on tag values such as * {@link org.openstreetmap.atlas.tags.HeightTag}. * * @author bbreithaupt */ public class AltitudeExtractor implements TagExtractor { /** * Validates and converts a value to a {@link Altitude}. * * @param value * {@link String} value. * @return {@link Optional} of a {@link Altitude} */ public static Optional validateAndExtract(final String value) { if (value.startsWith("-")) { final Optional distance = LengthExtractor .validateAndExtract(value.substring(1)); return distance.map(distance1 -> Altitude.meters(distance1.asMeters() * -1)); } final Optional distance = LengthExtractor.validateAndExtract(value); return distance.map(distance1 -> Altitude.meters(distance1.asMeters())); } @Override public Optional validateAndExtract(final String value, final Tag tag) { return AltitudeExtractor.validateAndExtract(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/IsoCountryExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.utilities.collections.StringList; /** * Extracts ISO2 and ISO3 country code values. * * @author mgostintsev */ public class IsoCountryExtractor implements TagExtractor> { @Override public Optional> validateAndExtract(final String value, final Tag tag) { final List countries = StringList.split(value, ISOCountryTag.COUNTRY_DELIMITER) .stream().map(IsoCountry::forCountryCode).filter(Optional::isPresent) .map(Optional::get).collect(Collectors.toList()); return Optional.of(countries); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/LengthExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.validation.LengthValidator; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Distance; /** * Extracts a {@link Distance} from a value. Can be used on tag values such as * {@link org.openstreetmap.atlas.tags.WidthTag}. * * @author bbreithaupt */ public class LengthExtractor implements TagExtractor { private static final LengthValidator VALIDATOR = new LengthValidator(); private static final String SINGLE_SPACE = " "; private static final String COMMA = ","; private static final String DECIMAL_POINT = "."; /** * Validates and converts a value to a {@link Distance}. * * @param value * {@link String} value. * @return {@link Optional} of a {@link Distance} */ public static Optional validateAndExtract(final String value) { // Some countries use comma as decimal separator // (https://en.wikipedia.org/wiki/Decimal_separator#Hindu-Arabic_numerals). Replace comma // with decimal point for further processing. final String uppercaseValue = value.replace(COMMA, DECIMAL_POINT).toUpperCase(); if (VALIDATOR.isValid(uppercaseValue)) { if (uppercaseValue.endsWith(SINGLE_SPACE + Distance.UnitAbbreviations.M)) { return Optional.of(Distance.meters(Double.valueOf(uppercaseValue.substring(0, uppercaseValue.lastIndexOf(SINGLE_SPACE + Distance.UnitAbbreviations.M))))); } else if (uppercaseValue.endsWith(SINGLE_SPACE + Distance.UnitAbbreviations.KM)) { return Optional.of(Distance .kilometers(Double.valueOf(uppercaseValue.substring(0, uppercaseValue .lastIndexOf(SINGLE_SPACE + Distance.UnitAbbreviations.KM))))); } else if (uppercaseValue.endsWith(SINGLE_SPACE + Distance.UnitAbbreviations.MI)) { return Optional .of(Distance.miles(Double.valueOf(uppercaseValue.substring(0, uppercaseValue .lastIndexOf(SINGLE_SPACE + Distance.UnitAbbreviations.MI))))); } else if (uppercaseValue.endsWith(SINGLE_SPACE + Distance.UnitAbbreviations.NMI)) { return Optional.of(Distance .nauticalMiles(Double.valueOf(uppercaseValue.substring(0, uppercaseValue .lastIndexOf(SINGLE_SPACE + Distance.UnitAbbreviations.NMI))))); } else if (uppercaseValue.contains(Distance.INCHES_NOTATION)) { final StringList split = StringList.split(uppercaseValue, Distance.FEET_NOTATION); if (split.size() == 2) { return Optional.of(Distance.feetAndInches(Double.valueOf(split.get(0)), Double.valueOf(split.get(1).substring(0, split.get(1).lastIndexOf(Distance.INCHES_NOTATION))))); } else if (split.size() == 1) { return Optional.of(Distance.inches(Double.valueOf(split.get(0).substring(0, split.get(0).lastIndexOf(Distance.INCHES_NOTATION))))); } } else if (uppercaseValue.contains(Distance.FEET_NOTATION)) { return Optional.of(Distance.feet(Double.valueOf(uppercaseValue.substring(0, uppercaseValue.lastIndexOf(Distance.FEET_NOTATION))))); } else { return Optional.of(Distance.meters(Double.valueOf(uppercaseValue))); } } return Optional.empty(); } @Override public Optional validateAndExtract(final String value, final Tag tag) { return LengthExtractor.validateAndExtract(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/LongExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.validation.LongValidator; /** * Extracts long {@link Number} values. * * @author mgostintsev */ public class LongExtractor implements TagExtractor { @Override public Optional validateAndExtract(final String value, final Tag tag) { final LongValidator validator = new LongValidator(); final Range range = tag.range(); if (range != null) { validator.setRange(range.min(), range.max()); for (final long exclusion : range.exclude()) { validator.excludeValue(exclusion); } } if (validator.isValid(value)) { return Optional.of(Long.parseLong(value)); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/NonEmptyStringExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.validation.NonEmptyStringValidator; /** * Extracts non-empty String values * * @author mgostintsev */ public final class NonEmptyStringExtractor { private static final NonEmptyStringValidator VALIDATOR = new NonEmptyStringValidator(); public static Optional validateAndExtract(final String value) { if (VALIDATOR.isValid(value)) { return Optional.of(value); } return Optional.empty(); } private NonEmptyStringExtractor() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/OrdinalExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Range; import org.openstreetmap.atlas.tags.annotations.validation.OrdinalValidator; /** * Extracts integer {@link Number} values. * * @author mgostintsev */ public final class OrdinalExtractor implements TagExtractor { @Override public Optional validateAndExtract(final String value, final Tag tag) { final OrdinalValidator validator = new OrdinalValidator(); final Range range = tag.range(); if (range != null) { validator.setRange(range.min(), range.max()); for (final long exclusion : range.exclude()) { validator.excludeValue(exclusion); } } if (validator.isValid(value)) { return Optional.of(Integer.parseInt(value)); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/SpeedExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.io.InputStreamReader; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.validation.SpeedValidator; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.scalars.Speed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.stream.JsonReader; /** * Extracts {@link Speed} values. * * @author mgostintsev */ public final class SpeedExtractor { private static final SpeedValidator VALIDATOR = new SpeedValidator(); private static final String SINGLE_SPACE = " "; private static final Logger logger = LoggerFactory.getLogger(SpeedExtractor.class); private static final JsonObject IMPLICIT_SPEED_MAP = new Gson().fromJson( new JsonReader(new InputStreamReader( Tag.class.getResourceAsStream("implicit-speed-values.json"))), JsonObject.class); public static Optional validateAndExtract(final String value) { if (VALIDATOR.isValid(value)) { try { final String valueOrImplicit = IMPLICIT_SPEED_MAP.has(value.toLowerCase()) ? IMPLICIT_SPEED_MAP.get(value.toLowerCase()).getAsString() : value; if (valueOrImplicit.endsWith(Speed.MILES_PER_HOUR)) { return Optional.of(Speed.milesPerHour(Double.valueOf( StringList.split(valueOrImplicit, SINGLE_SPACE).iterator().next()))); } if (valueOrImplicit.endsWith(Speed.NAUTICAL_MILES_PER_HOUR)) { return Optional.of(Speed.knots(Double.valueOf( StringList.split(valueOrImplicit, SINGLE_SPACE).iterator().next()))); } if (valueOrImplicit.endsWith(Speed.KILOMETERS_PER_HOUR)) { return Optional.of(Speed.kilometersPerHour(Double.valueOf( StringList.split(valueOrImplicit, SINGLE_SPACE).iterator().next()))); } if ("none".equals(valueOrImplicit)) { return Optional.empty(); } return Optional.of(Speed.kilometersPerHour(Double.valueOf(valueOrImplicit))); } catch (final NumberFormatException e) { logger.warn("Unable to read speed from {}", value); return Optional.empty(); } } return Optional.empty(); } private SpeedExtractor() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/extraction/TagExtractor.java ================================================ package org.openstreetmap.atlas.tags.annotations.extraction; import java.util.Optional; import org.openstreetmap.atlas.tags.annotations.Tag; /** * Extracts the tag value from the textual representation and associated {@link Tag}. * * @param * The extracted value type * @author mgostintsev */ public interface TagExtractor { /** * Validates and extracts the value from given textual representation * * @param value * The textual representation of this tag's value * @param tag * The associated {@link Tag} * @return the {@link Optional} containing the extracted value */ Optional validateAndExtract(String value, Tag tag); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/DoubleValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * Checks if the value of a tag is either an exact value or can be coerced into a java double and * within an optional range of accepted values * * @author cstaylor */ public class DoubleValidator extends NumericValidator { @Override protected Number parse(final String value) { return Double.parseDouble(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/ExactMatchValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * Checks if the value of the tag matches a list of known good values * * @author cstaylor */ public class ExactMatchValidator implements TagValidator { private final Set values; private final Set regexes; public ExactMatchValidator() { this.values = new HashSet<>(); this.regexes = new HashSet<>(); } @Override public boolean isValid(final String value) { if (this.values.contains(value)) { return true; } for (final Pattern pattern : this.regexes) { if (pattern.matcher(value).matches()) { return true; } } return false; } /** * Add regular expression patterns to the validator * * @param regexes * regular expression encoded in a String * @return fluent interface means we return this */ public ExactMatchValidator withRegularExpressions(final String... regexes) { this.regexes.addAll( Arrays.asList(regexes).stream().map(Pattern::compile).collect(Collectors.toSet())); return this; } /** * Add exact strings to the validator * * @param values * exact values we want to match * @return fluent interface means we return this */ public ExactMatchValidator withValues(final String... values) { this.values.addAll(Arrays.asList(values)); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/ISO2CountryValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.Arrays; import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; /** * Checks if the value of a tag matches one of the supported ISO3 country names in the Java Locale * classes * * @author cstaylor */ public class ISO2CountryValidator implements TagValidator { private static Set validISOCountries; static { validISOCountries = Arrays.asList(Locale.getAvailableLocales()).stream() .filter(ISO2CountryValidator::hasISO2Country).map(Locale::getCountry) .collect(Collectors.toSet()); } private static boolean hasISO2Country(final Locale locale) { final String country = locale.getCountry(); return StringUtils.isNotBlank(country) && country.length() == 2; } @Override public boolean isValid(final String value) { return validISOCountries.contains(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/ISO3CountryValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.Arrays; import java.util.Locale; import java.util.MissingResourceException; import java.util.Set; import java.util.stream.Collectors; /** * Checks if the value of a tag matches one of the supported ISO3 country names in the Java Locale * classes * * @author cstaylor */ public class ISO3CountryValidator implements TagValidator { private static Set validISOCountries; static { validISOCountries = Arrays.asList(Locale.getAvailableLocales()).stream() .filter(ISO3CountryValidator::hasISO3Country).map(Locale::getISO3Country) .collect(Collectors.toSet()); } private static boolean hasISO3Country(final Locale locale) { try { locale.getISO3Country(); return true; } catch (final MissingResourceException oops) { return false; } } @Override public boolean isValid(final String value) { return validISOCountries.contains(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/ISOCountryValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import org.openstreetmap.atlas.locale.IsoCountry; /** * Checks if the value of a tag matches an ISO2 or ISO3 country code * * @author cstaylor */ public class ISOCountryValidator implements TagValidator { @Override public boolean isValid(final String value) { return IsoCountry.isValidCountryCode(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/LengthValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.Arrays; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.geography.atlas.items.complex.buildings.HeightConverter; import org.openstreetmap.atlas.utilities.scalars.Distance; import org.openstreetmap.atlas.utilities.scalars.Distance.UnitAbbreviations; /** * Based on mnahoum's {@link HeightConverter} class. *

* Note that values like width and height are still considered lengths in the OSM Units definition * http://wiki.openstreetmap.org/wiki/Map_Features/Units * * @author cstaylor * @author gpogulsky */ public class LengthValidator implements TagValidator { private static final DoubleValidator DOUBLE_VALIDATOR; static { DOUBLE_VALIDATOR = new DoubleValidator(); } /** * Validates if the give value is a proper length value. *

* The expected values are "12.5 m", "12.5 km", "12.5 mi", "12.5 nmi", "12.5", * "12'5\"".Incomplete values like "12'"or "5\"" are also recognized. The method will properly * handle malformed values like "Estacion de Servicio \"Los Arrayanes\"" (contains \ ", but is * not a number), "12'err", etc. *

*/ @Override public boolean isValid(final String value) { final boolean result; final Matcher suffixMatcher = Pattern .compile(String.format("(\\d+.?\\d*) (%s)", String.join("|", Arrays.stream(UnitAbbreviations.values()).map(Enum::toString) .collect(Collectors.toList())))) .matcher(value.toUpperCase()); if (suffixMatcher.matches()) { result = DOUBLE_VALIDATOR.isValid(suffixMatcher.group(1)); } else { final int feetIndex = value.indexOf(Distance.FEET_NOTATION); if (feetIndex > -1) { if (DOUBLE_VALIDATOR.isValid(value.substring(0, feetIndex))) { // Tail? if (feetIndex + 1 < value.length()) { final int inchesIndex = value.indexOf(Distance.INCHES_NOTATION, feetIndex + 1); if (inchesIndex > -1) { result = this.validateInchesAndTail(value, feetIndex + 1, inchesIndex); } else { result = StringUtils .isBlank(value.substring(feetIndex + 1, value.length())); } } else { result = true; } } else { result = false; } } else { final int inchesIndex = value.indexOf(Distance.INCHES_NOTATION); if (inchesIndex > -1) { result = this.validateInchesAndTail(value, 0, inchesIndex); } else { result = DOUBLE_VALIDATOR.isValid(value); } } } return result; } private boolean validateInchesAndTail(final String value, final int start, final int index) { return DOUBLE_VALIDATOR.isValid(value.substring(start, index)) && (index + 1 == value.length() || StringUtils.isBlank(value.substring(index + 1, value.length()))); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/LongValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * Checks if the value of a tag is either an exact value or can be coerced into a java long and * within an optional range of accepted values * * @author cstaylor */ public class LongValidator extends NumericValidator { @Override protected Number parse(final String value) { return Long.parseLong(value); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/NonEmptyStringValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * Checks if the value of a tag has at least one non-whitespace character * * @author cstaylor */ public class NonEmptyStringValidator implements TagValidator { @Override public boolean isValid(final String value) { return value.trim().length() > 0; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/NoneValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * Does no checking: only used for permitting any kind of values in a tag * * @author cstaylor */ public class NoneValidator implements TagValidator { @Override public boolean isValid(final String value) { return true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/NumericValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.HashSet; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Numeric validators can check ranges of valid data or even exclude certain values regardless of * the range * * @author cstaylor */ public abstract class NumericValidator extends ExactMatchValidator { private static final Logger logger = LoggerFactory.getLogger(NumericValidator.class); private Long minimum; private Long maximum; private final Set exclusions; public NumericValidator() { this.exclusions = new HashSet<>(); } public void excludeValue(final long value) { this.exclusions.add(new Double(value)); } /** * Since all NumericValidators are ExactMatchValidators, we first check if the value matches any * exact match, and then proceed with the numeric conversion and range check */ @Override public final boolean isValid(final String value) { try { return super.isValid(value) || withinRange(parse(value)); } catch (final NumberFormatException oops) { return false; } } public void setMaximum(final long maximum) { if (this.minimum == null) { this.maximum = maximum; } else { if (maximum > this.minimum) { this.maximum = maximum; } else { logger.debug( "Cannot set maximum less than or equal to minimum {}. Consider using setRange instead.", this.minimum); } } } public void setMinimum(final long minimum) { if (this.maximum == null) { this.minimum = minimum; } else { if (minimum < this.maximum) { this.minimum = minimum; } else { logger.debug( "Cannot set minimum greater than or equal to maximum {}. Consider using setRange instead.", this.maximum); } } } public void setRange(final long minimum, final long maximum) { if (minimum < maximum) { this.minimum = minimum; this.maximum = maximum; } else if (minimum == Long.MAX_VALUE || maximum == Long.MIN_VALUE) { /* * Special case where at least one of the range endpoints is a default value. In this * case, we don't want to send out a log message - instead just do nothing. */ } else { logger.debug("Invalid range supplied, minimum: {} cannot be greater than maximum: {}.", minimum, maximum); } } protected boolean checkExclusions(final Number checkMe) { return this.exclusions.contains(checkMe.doubleValue()); } protected abstract Number parse(String value); protected boolean withinRange(final Number checkMe) { if (checkExclusions(checkMe)) { return false; } final double theValue = checkMe.doubleValue(); if (this.minimum != null && theValue < this.minimum) { return false; } if (this.maximum != null && theValue > this.maximum) { return false; } return true; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/OrdinalValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * Ordinals must be positive integers * * @author cstaylor */ public class OrdinalValidator extends NumericValidator { @Override protected Number parse(final String value) { return Long.parseLong(value); } @Override protected boolean withinRange(final Number checkMe) { return super.withinRange(checkMe) && checkMe.longValue() > 0L; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/SpeedValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.io.InputStreamReader; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.utilities.scalars.Speed; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.stream.JsonReader; /** * Checks if the value of the tag matches the OSM conventions for denoting speed. See more on the * speed limits wiki and * max speed wiki. Note: the * validation has been loosened slightly from OSM standards to handle more valid limits. * Specifically, OSM dictates if km/h are the unit, no unit should be specified. The validation * below identifies 'kph' as a valid suffix. * * @author mgostintsev */ public class SpeedValidator implements TagValidator { private static final DoubleValidator DOUBLE_VALIDATOR; private static final JsonObject IMPLICIT_SPEED_MAP = new Gson().fromJson( new JsonReader(new InputStreamReader( Tag.class.getResourceAsStream("implicit-speed-values.json"))), JsonObject.class); static { DOUBLE_VALIDATOR = new DoubleValidator(); DOUBLE_VALIDATOR.setMinimum(0); } @Override public boolean isValid(final String value) { if (value.endsWith(Speed.MILES_PER_HOUR)) { return DOUBLE_VALIDATOR.isValid( value.substring(0, value.length() - Speed.MILES_PER_HOUR.length()).trim()); } else if (value.endsWith(Speed.NAUTICAL_MILES_PER_HOUR)) { return DOUBLE_VALIDATOR.isValid(value .substring(0, value.length() - Speed.NAUTICAL_MILES_PER_HOUR.length()).trim()); } else if (value.endsWith(Speed.KILOMETERS_PER_HOUR)) { return DOUBLE_VALIDATOR.isValid( value.substring(0, value.length() - Speed.KILOMETERS_PER_HOUR.length()).trim()); } else if ("none".equals(value)) { return true; } else if (IMPLICIT_SPEED_MAP.has(value.toLowerCase())) { return true; } else { return DOUBLE_VALIDATOR.isValid(value); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/TagDocumenter.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.net.URI; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators.TagKeySearch; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfoList; import io.github.classgraph.ScanResult; /** * Class that walks across all Tags and generates metadata about them that can be converted into * HTML or any other desired documentation format as needed. * * @author cstaylor */ public class TagDocumenter { /** * Callback that receives metadata about a Tag * * @author cstaylor */ public interface Callback { void tagFound(CallbackData data); } /** * Metadata container class for Tag information * * @author cstaylor */ public static final class CallbackData implements Comparable { private String tagClassName; private String tagKey; private final SortedSet validTagValues; private URI tagInfoLink; private URI osmWikiLink; private String validationType; private boolean localized; private boolean synthetic; CallbackData() { this.validTagValues = new TreeSet<>(); } @Override public int compareTo(final CallbackData other) { return this.tagKey.compareTo(other.tagKey); } @Override public boolean equals(final Object obj) { if (obj == this) { return true; } if (obj instanceof CallbackData) { final CallbackData other = (CallbackData) obj; boolean returnValue = Objects.equals(this.tagClassName, other.tagClassName); returnValue = returnValue && Objects.equals(this.tagKey, other.tagKey); returnValue = returnValue && Objects.equals(this.validTagValues, other.validTagValues); returnValue = returnValue && Objects.equals(this.tagInfoLink, other.tagInfoLink); returnValue = returnValue && Objects.equals(this.osmWikiLink, other.osmWikiLink); returnValue = returnValue && Objects.equals(this.validationType, other.validationType); returnValue = returnValue && Objects.equals(this.localized, other.localized); returnValue = returnValue && Objects.equals(this.synthetic, other.synthetic); return returnValue; } return false; } public Optional getOsmWikiLink() { return Optional.ofNullable(this.osmWikiLink); } public String getTagClassName() { return this.tagClassName; } public Optional getTagInfoLink() { return Optional.ofNullable(this.tagInfoLink); } public String getTagKey() { return this.tagKey; } public Iterable getValidTagValues() { return this.validTagValues; } public String getValidationType() { return this.validationType; } @Override public int hashCode() { return Objects.hash(this.tagKey, this.tagClassName); } public boolean isLocalized() { return this.localized; } public boolean isSynthetic() { return this.synthetic; } } private final Set tagData; /** * Calls the other constructor with the default package name */ public TagDocumenter() { this("org.openstreetmap.atlas"); } /** * Find all of the tags in packageName and create metadata for all of them * * @param packageName * the base package to search the current classloader for Tags */ public TagDocumenter(final String packageName) { this.tagData = new TreeSet<>(); /* * We definitely don't want tags in classes named TestCase. When running this from the * command line we shouldn't get any TestCase tags anyways, but when running this in * development mode under Eclipse with core in the classpath they will be picked up. */ // Scan the given package try (ScanResult scanResult = new ClassGraph().enableAllInfo().whitelistPackages(packageName) .scan()) { // Look at annotated classes final ClassInfoList tagClassInfoList = scanResult .getClassesWithAnnotation("org.openstreetmap.atlas.tags.annotations.Tag"); // Ignore any TestCase classes tagClassInfoList.loadClasses().forEach(klass -> { if (!klass.getName().contains("TestCase")) { this.tagData.add(createCallbackDataFromClass(klass)); } }); } } public void walk(final Callback callback) { this.tagData.stream().forEach(callback::tagFound); } private CallbackData createCallbackDataFromClass(final Class tagClass) { final CallbackData returnValue = new CallbackData(); TagKeySearch.findTagKeyIn(tagClass).ifPresent(results -> { final String tagName = results.getKeyName(); final TagKey tagKey = results.getTagKey(); returnValue.tagKey = tagName; returnValue.tagClassName = tagClass.getName(); returnValue.osmWikiLink = results.getTag().osm().length() > 0 ? URI.create(results.getTag().osm()) : null; returnValue.tagInfoLink = results.getTag().taginfo().length() > 0 ? URI.create(results.getTag().taginfo()) : null; returnValue.localized = tagKey.value() == TagKey.KeyType.LOCALIZED; returnValue.validationType = results.getTag().value().name(); returnValue.synthetic = results.getTag().synthetic(); }); return returnValue; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/TagValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; /** * TagValidators verify a Tag's value. Some may be complex checks like data conversions, while * others may compare against a simple set of values * * @author cstaylor */ public interface TagValidator { /** * Checks if value is valid for this kind of tag * * @param value * the textual representation of this tag's value * @return true if the value is valid, false otherwise */ boolean isValid(String value); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/TimestampValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.util.Date; /** * Checks if the value of a tag is either an exact value or can be coerced into a java Date object * * @author cstaylor */ public class TimestampValidator extends ExactMatchValidator { @Override public boolean isValid(final String value) { if (super.isValid(value)) { return true; } try { new Date(Long.parseLong(value)); return true; } catch (final NumberFormatException oops) { return false; } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/URIValidator.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.net.URI; import java.net.URISyntaxException; /** * Validator for verifying a String follows the proper URI syntax * * @author cstaylor */ public class URIValidator extends ExactMatchValidator { @Override public boolean isValid(final String value) { if (super.isValid(value)) { return true; } try { new URI(value); return true; } catch (final URISyntaxException oops) { return false; } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/annotations/validation/Validators.java ================================================ package org.openstreetmap.atlas.tags.annotations.validation; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.locale.IsoLanguage; import org.openstreetmap.atlas.tags.LocalizedTagNameWithOptionalDate; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.Taggable.TagSearchOption; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; import org.openstreetmap.atlas.tags.annotations.TagValue; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.cache.CachingValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfoList; import io.github.classgraph.ScanResult; /** * Builds a table of {@link TagValidator}s using Java annotations and introspection. * * @author cstaylor */ public class Validators { /** * Extracted the code for finding the tag key into its own public inner class * * @author cstaylor */ public static final class TagKeySearch { public static Optional findTagKeyIn(final Class tagClass) { final Tag tag = tagClass.getDeclaredAnnotation(Tag.class); for (final Field field : tagClass.getDeclaredFields()) { final TagKey tagKey = field.getAnnotation(TagKey.class); if (tagKey != null && field.getType().isAssignableFrom(String.class)) { try { final String returnValue = (String) field.get(null); if (returnValue == null || returnValue.trim().length() == 0) { throw new IllegalArgumentException( String.format("%s is missing a key", tagClass.getName())); } return Optional.of(new TagKeySearchResults(tag, tagKey, returnValue)); } catch (final IllegalAccessException oops) { throw new IllegalArgumentException(String.format( "Check the source code for %s: the @TagKey is probably not a public static final String constant", tagClass.getName()), oops); } } } return Optional.empty(); } } /** * Immutable object capturing the results of a tag key search * * @author cstaylor */ public static final class TagKeySearchResults { private final Tag tag; private final TagKey key; private final String keyName; private TagKeySearchResults(final Tag tag, final TagKey key, final String keyName) { this.tag = tag; this.key = key; this.keyName = keyName; } public String getKeyName() { return this.keyName; } public Tag getTag() { return this.tag; } public TagKey getTagKey() { return this.key; } } /** * Single point for handling both standard and localizable tags, simplifying the main Validators * source code below * * @author cstaylor */ private static final class ValidatorMap { private final Map validators; private final Map localizedValidators; private final Map> origins; ValidatorMap() { this.validators = new HashMap<>(); this.localizedValidators = new HashMap<>(); this.origins = new HashMap<>(); } boolean canValidate(final String name) { return validatorFor(name) != null; } Class classFor(final String name) { return this.origins.get(name); } void put(final Class tagClass, final String name, final TagKey tagKey, final TagValidator validator) { if (tagKey.value() == KeyType.EXACT) { this.validators.put(name, validator); } else { // This is for localized tags this.localizedValidators.put(name, validator); } this.origins.put(name, tagClass); } TagValidator validatorFor(final String name) { // Step 1: Check standard exact name match validators TagValidator validator = this.validators.get(name); if (validator == null) { final LocalizedTagNameWithOptionalDate localizedName = new LocalizedTagNameWithOptionalDate( name); validator = this.localizedValidators.get(localizedName.getName()); } return validator; } } private static final Logger logger = LoggerFactory.getLogger(Validators.class); private final ValidatorMap validators; private final EnumMap> validatorTypes; public static String findTagNameIn(final Class tagClass) { final Optional tagName = TagKeySearch.findTagKeyIn(tagClass); if (tagName.isPresent()) { return tagName.get().getKeyName(); } throw new IllegalArgumentException( String.format("key must be declared in class: %s", tagClass.getName())); } /** * Helpful method for swizzling an interface Tag that includes the contents of an enum tag * through the [with] annotation feature into its possible value if that value is found in the * passed in Taggable parameter and the enumType provided is actually listed in the [with] * attribute. *

* See the FromEnumTestCase class for an example of how to use this method * * @param * the type of enum tag we're parsing * @param tagType * the interface tag with a [with] attribute that we want a possible value from * @param enumType * the return value type we want a value from * @param taggable * the source of tags and their values * @return an empty optional if the tagType isn't a tag, doesn't have a key, enumType is not * included in tagType's [with] list, the value isn't found in taggable, or no enum * value in enumType matches (ignoring case) the tag's value */ public static > Optional from(final Class tagType, final Class enumType, final Taggable taggable) { final Tag tag = tagType.getDeclaredAnnotation(Tag.class); if (tag != null && Stream.of(tag.with()).anyMatch(possible -> possible == enumType)) { return fromHelper(findTagNameIn(tagType), enumType, taggable); } return Optional.empty(); } /** * Caching version - use in generic applications. *

* Helpful method for swizzling an Enum Tag into its possible value if that value is found in * the passed in Taggable parameter. This cuts down on a lot of duplicate code that we had in * each enum-type Tag. *

* See the FromEnumTestCase class for an example of how to use this method * * @param * the type of enum tag we're parsing * @param tagType * the enum style tag that we want a possible value from * @param taggable * the source of tags and their values * @return an empty optional if the enum isn't a tag, doesn't have a key, the value isn't found * in taggable, or no enum value matches (ignoring case) the tag's value */ public static > Optional from(final Class tagType, final Taggable taggable) { return CachingValidator.getInstance().from(tagType, taggable); } /** * Reflection version - use when you need to get a few tags, and no caching is necessary. This * method is used by the caching version to populate the cache. *

* Helpful method for swizzling an Enum Tag into its possible value if that value is found in * the passed in Taggable parameter. This cuts down on a lot of duplicate code that we had in * each enum-type Tag. *

* * @param * the type of enum tag we're parsing * @param tagType * the enum style tag that we want a possible value from * @param taggable * the source of tags and their values * @return an empty optional if the enum isn't a tag, doesn't have a key, the value isn't found * in taggable, or no enum value matches (ignoring case) the tag's value */ public static > Optional fromAnnotation(final Class tagType, final Taggable taggable) { if (tagType.getDeclaredAnnotation(Tag.class) != null) { return fromHelper(findTagNameIn(tagType), tagType, taggable); } return Optional.empty(); } /** * Convenience method for checking if a class is actually a tag and that its key is localizable * * @param tagType * the tag class we want to check * @return true if tagType is actually a tag and its key is localizable */ public static boolean hasLocalizedTagKey(final Class tagType) { /* * First, sanity check the key */ if (tagType == null) { throw new IllegalArgumentException("tagType can't be null"); } /* * Next, check if the key is actually a key */ final Optional tagKey = Validators.TagKeySearch.findTagKeyIn(tagType); if (!tagKey.isPresent()) { throw new IllegalArgumentException( String.format("%s isn't a known key", tagType.getName())); } return tagKey.get().getTagKey().value() == KeyType.LOCALIZED; } /** * Convenience method that returns a filter for a {@link Taggable} that evaluates if all passed * in tag types are present * * @param tagTypes * the type of tags to check * @return a filter for a Taggable entity that evaluates if if contains all specified tag types */ public static Predicate hasValuesFor(final Class... tagTypes) { if (tagTypes.length == 0) { return taggable -> false; } return taggable -> hasValuesFor(taggable, tagTypes); } /** * Convenience method for checking if we have defined values for all tags passed into the method * * @param taggable * where we look up the tags * @param tagTypes * the type of tags to check * @return true if all of the tags have values, false if at least one is missing */ public static boolean hasValuesFor(final Taggable taggable, final Class... tagTypes) { for (final Class tagType : tagTypes) { if (!taggable.getTag(findTagNameIn(tagType)).isPresent()) { return false; } } return true; } /** * Use this method to check if a tag exists in a Taggable object _and_ is _not_ one of several * provided values. While we could do the same with an EnumSet, this makes calling code cleaner * since they only need a single line to check for the existence of at least one item in a set * of tag values * * @param * the enum-type tag's class object * @param taggable * where the tags should be read from * @param type * the class of the enum-type tag we are looking for? * @param values * which values do we want to check against? * @return true if the tag exists in taggable and if the value is not any of the specified * values (like an enumset) */ public static > boolean isNotOfType(final Taggable taggable, final Class type, @SuppressWarnings("unchecked") final T... values) { final Optional possibleRealValue = Validators.from(type, taggable); if (!possibleRealValue.isPresent()) { return false; } final T realValue = possibleRealValue.get(); for (final T searching : values) { if (realValue == searching) { return false; } } return true; } /** * Use this method to check if a tag exists in two {@link Taggable} objects, and those values * are the same. * * @param * the enum-type tag's class object * @param firstTaggable * one taggable we are comparing * @param secondTaggable * the other taggable we are comparing against * @param type * the class of the enum-type tag we are looking for * @return true if the tag exists in firstTaggable AND secondTaggable, AND the value of * firstTaggable is equal to the value of secondTaggable. */ public static boolean isOfSameType(final Taggable firstTaggable, final Taggable secondTaggable, final Class type) { final String key = findTagNameIn(type); return firstTaggable.getTag(key) .flatMap(oneTag -> secondTaggable.getTag(key).map(oneTag::equals)).orElse(false); } /** * Use this method to check if a tag exists in a {@link Taggable} object _and_ is one of several * expected values. While we could do the same with an EnumSet, this makes calling code cleaner * since they only need a single line to check for the existence of at least one item in a set * of tag values * * @param * the enum-type tag's class object * @param taggable * where the tags should be read from * @param type * the class of the enum-type tag we are looking for? * @param values * which values do we want to check against? * @return true if the tag exists in taggable and if the value matches any of the specified * values (like an enumset) */ public static > boolean isOfType(final Taggable taggable, final Class type, @SuppressWarnings("unchecked") final T... values) { final Optional possibleRealValue = Validators.from(type, taggable); if (!possibleRealValue.isPresent()) { return false; } final T realValue = possibleRealValue.get(); for (final T searching : values) { if (realValue == searching) { return true; } } return false; } /** * Convenience method for creating the localized name of a tag if and only if tagType is a Tag * and the TagKey is localizable * * @param tagType * check this class if it's a localizable tag and return the localized name * @param language * the optional language to localize * @param searchOptions * optional arguments that change how we interpret tags * @return an optional string of the localized tag name */ public static Optional localizeKeyName(final Class tagType, final Optional language, final TagSearchOption... searchOptions) { /* * First, sanity check the key */ if (tagType == null) { throw new IllegalArgumentException("tagType can't be null"); } /* * Next, check if the key is actually a key */ final Optional tagKey = Validators.TagKeySearch.findTagKeyIn(tagType); if (!tagKey.isPresent()) { throw new IllegalArgumentException( String.format("%s isn't a known key", tagType.getName())); } final EnumSet searchOptionSet = searchOptions.length > 0 ? EnumSet.copyOf(Arrays.asList(searchOptions)) : EnumSet.noneOf(TagSearchOption.class); final TagKeySearchResults data = tagKey.get(); Optional value = Optional.empty(); if (language.isPresent() && (data.getTagKey().value() == KeyType.LOCALIZED || searchOptionSet.contains(TagSearchOption.FORCE_ALL_LOCALIZED_ONLY))) { value = Optional.of( String.format("%s:%s", data.getKeyName(), language.get().getLanguageCode())); } if (!value.isPresent()) { value = Optional.of(data.getKeyName()); } return value; } /** * Simple utility method for creating a hashmap out of a set of enum-type tags. *

* The key portion of each map entry is extracted from the tag type itself. * * @param values * the enum-type tags we want to convert into a hashmap * @return the completed hashmap */ public static Map toMap(final Enum... values) { return Arrays.asList(values).stream().collect(Collectors.toMap( value -> findTagNameIn(value.getClass()), value -> value.name().toLowerCase())); } /** * Shamelessly taken from SO: * http://stackoverflow.com/questions/7254126/get-annotations-for-enum-type-variable * * @param constant * the particular enum constant we're interested in converting * @return the value of the constant or the overriden value from the TagValueAs annotation */ private static String enumConstantToValue(final Enum constant) { try { final Field field = constant.getDeclaringClass().getField(constant.name()); final TagValueAs substitutedValue = field.getAnnotation(TagValueAs.class); return substitutedValue == null ? ((Enum) field.get(null)).name().toLowerCase() : substitutedValue.value(); } catch (final IllegalAccessException | NoSuchFieldException oops) { throw new CoreException("{} can't access field value", constant, oops); } } private static > Optional fromHelper(final String tagName, final Class enumType, final Taggable taggable) { // First try simple match // If it fails, try matching based on annotations if (tagName != null) { final Optional tagValue = taggable.getTag(tagName); if (tagValue.isPresent()) { final String internedValue = tagValue.get().toUpperCase().intern(); try { final T enumValue = Enum.valueOf(enumType, internedValue); return Optional.of(enumValue); } catch (final IllegalArgumentException badArgument) { // There is no direct name match return fromMatchingHelper(tagName, enumType, internedValue); } } } return Optional.empty(); } @SuppressWarnings("unchecked") private static > Optional fromMatchingHelper(final String tagName, final Class enumType, final String internedValue) { try { for (final T enumValue : (T[]) enumType.getMethod("values").invoke(null)) { if (matches(enumValue, internedValue)) { return Optional.of(enumValue); } } } catch (final NoSuchMethodException | IllegalAccessException | InvocationTargetException anImpossibleError) { logger.error("{} doesn't have a values method, or it couldn't be called: impossible", tagName, anImpossibleError); } return Optional.empty(); } private static boolean matches(final Enum constant, final String value) { try { final Field field = constant.getDeclaringClass().getField(constant.name()); final TagValueAs substitutedValue = field.getAnnotation(TagValueAs.class); final String comparisonString = substitutedValue != null ? substitutedValue.value() : constant.name(); return comparisonString.toUpperCase().intern() == value.intern(); } catch (final NoSuchFieldException oops) { throw new CoreException("{} can't access field value", constant, oops); } } public Validators(final Class childrenOf) { this(childrenOf.getPackage().getName()); } public Validators(final String packageName) { this.validatorTypes = new EnumMap<>(Validation.class); this.validators = new ValidatorMap(); fillValidatorTypes(this.validatorTypes); final List> klasses = new ArrayList<>(); // Scan all classes in the given package with the Tag annotation try (ScanResult scanResult = new ClassGraph().enableAllInfo().whitelistPackages(packageName) .scan()) { final ClassInfoList tagClassInfoList = scanResult .getClassesWithAnnotation("org.openstreetmap.atlas.tags.annotations.Tag"); tagClassInfoList.loadClasses().forEach(klasses::add); } klasses.stream().forEach(this::processClass); } /** * Is key a known tag? * * @param key * the type of tag we want to verify * @return true if we can handle [key] tags, false otherwise */ public boolean canValidate(final String key) { return this.validators.canValidate(key); } /** * Sometimes we want to know where a tag found by Validators was defined. * * @param tag * the name of the tag we're searching for * @return an Optional containing the class reference if it exists, an empty optional otherwise */ public Optional> findClassDefining(final String tag) { return Optional.ofNullable(this.validators.classFor(tag)); } public Optional getTagInfo(final String tagName) { final Class tagClass = this.validators.classFor(tagName); if (tagClass != null) { final Tag tag = tagClass.getAnnotation(Tag.class); if (tag.taginfo().equals("")) { return Optional.empty(); } else { return Optional.of(tag.taginfo()); } } return Optional.empty(); } /** * Get the validator used for verifying tags named tagName * * @param key * the name of the tag we want to verify * @return the validator if we support it, null otherwise */ public TagValidator getValidatorFor(final String key) { return this.validators.validatorFor(key); } /** * Convenience method for checking and verifying a value without having to chain calls * * @param key * the key we want to verify * @param value * the value we want to verify against the allowed values for key * @return true if the value is valid for key, false otherwise */ public boolean isValidFor(final String key, final String value) { return canValidate(key) && getValidatorFor(key).isValid(value); } protected void fillValidatorTypes( final EnumMap> validatorTypes) { validatorTypes.put(Validation.DOUBLE, DoubleValidator.class); validatorTypes.put(Validation.LONG, LongValidator.class); validatorTypes.put(Validation.MATCH, ExactMatchValidator.class); validatorTypes.put(Validation.TIMESTAMP, TimestampValidator.class); validatorTypes.put(Validation.NON_EMPTY_STRING, NonEmptyStringValidator.class); validatorTypes.put(Validation.NONE, NoneValidator.class); validatorTypes.put(Validation.ISO3_COUNTRY, ISO3CountryValidator.class); validatorTypes.put(Validation.ISO2_COUNTRY, ISO2CountryValidator.class); validatorTypes.put(Validation.ORDINAL, OrdinalValidator.class); validatorTypes.put(Validation.URI, URIValidator.class); validatorTypes.put(Validation.SPEED, SpeedValidator.class); validatorTypes.put(Validation.LENGTH, LengthValidator.class); } private TagValidator createValidatorFor(final Validation validation) { final Class validatorClass = this.validatorTypes.get(validation); if (validatorClass == null) { throw new IllegalArgumentException( String.format("%s is an unsupported validator", validation)); } try { return validatorClass.newInstance(); } catch (final IllegalAccessException | InstantiationException oops) { throw new IllegalArgumentException( String.format("%s is an unsupported validator", validation), oops); } } private TagValidator fillEnumerationValues(final ExactMatchValidator validator, final Class tagClass) { for (final Enum enumValue : (Enum[]) tagClass.getEnumConstants()) { validator.withValues(enumConstantToValue(enumValue)); } return validator; } private void fillExactMatches(final ExactMatchValidator validator, final Field[] fields) { for (final Field field : fields) { final TagValue tagValue = field.getAnnotation(TagValue.class); if (tagValue != null) { if (field.getType().isAssignableFrom(String.class)) { try { final String returnValue = (String) field.get(null); if (returnValue == null || returnValue.trim().length() == 0) { throw new IllegalArgumentException("key can't be empty"); } switch (tagValue.value()) { case REGEX: validator.withRegularExpressions(returnValue); break; case EXACT: validator.withValues(returnValue); break; default: throw new IllegalStateException(String.format( "%s is an unsupported value type", tagValue.value())); } } catch (final IllegalAccessException oops) { throw new IllegalArgumentException(oops); } } } } } private void processClass(final Class tagClass) { final Tag tag = tagClass.getAnnotation(Tag.class); if (tag != null) { // I've only seen tags being null under eclipse when they mark // enums. I think this is an Eclipse bug. // This shouldn't be affected by our standard build final TagValidator validator = createValidatorFor(tag.value()); TagKeySearch.findTagKeyIn(tagClass).ifPresent(results -> { if (validator instanceof ExactMatchValidator) { final ExactMatchValidator exactMatch = (ExactMatchValidator) validator; /** * If our validator also supports direct lookup of valid values, fill them here */ fillExactMatches(exactMatch, tagClass.getDeclaredFields()); if (tagClass.isEnum()) { fillEnumerationValues(exactMatch, tagClass); } /** * With classes let us share enum values */ for (final Class> withClass : tag.with()) { fillEnumerationValues(exactMatch, withClass); } } if (validator instanceof NumericValidator) { final NumericValidator numeric = (NumericValidator) validator; numeric.setRange(tag.range().min(), tag.range().max()); for (final long excludeMe : tag.range().exclude()) { numeric.excludeValue(excludeMe); } } this.validators.put(tagClass, results.getKeyName(), results.getTagKey(), validator); }); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/cache/CachingValidator.java ================================================ package org.openstreetmap.atlas.tags.cache; import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.tags.Taggable; /** * Implementation of cache for Validators. *

* Used by {@link org.openstreetmap.atlas.tags.annotations.validation.Validators Validators} * implicitly. Can be used on its own. *

* CachingValidator uses {@link Tagger} for caching actual values of given Tag. * * @author gpogulsky */ public class CachingValidator { private static CachingValidator INSTANCE = new CachingValidator(); @SuppressWarnings("rawtypes") private final Map map; public static CachingValidator getInstance() { return INSTANCE; } public CachingValidator() { this.map = new HashMap<>(); } /** * Provides Enum value associated with the given Tag type for an object, if it exists. *

* {@link org.openstreetmap.atlas.tags.annotations.validation.Validators#from(Class, Taggable) * Validators.from} is using this method implicitly. This method could be used on its own in * place of Validators.from. * * @param * the type of enum tag we're parsing * @param tagType * the enum style tag that we want a possible value from * @param taggable * the source of tags and their values * @return an empty optional if the enum isn't a tag, doesn't have a key, the value isn't found * in taggable, or no enum value matches (ignoring case) the tag's value */ public > Optional from(final Class tagType, final Taggable taggable) { final Tagger tagger = this.getTagger(tagType); return tagger.getTag(taggable); } private synchronized > Tagger addTagger(final Class tagType) { @SuppressWarnings("unchecked") Tagger tagger = this.map.get(tagType); if (tagger == null) { tagger = new Tagger(tagType); this.map.put(tagType, tagger); } return tagger; } private > Tagger getTagger(final Class tagType) { @SuppressWarnings("unchecked") Tagger tagger = this.map.get(tagType); if (tagger == null) { tagger = this.addTagger(tagType); } return tagger; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/cache/Tagger.java ================================================ package org.openstreetmap.atlas.tags.cache; import java.io.Serializable; import java.util.Optional; import java.util.concurrent.ExecutionException; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; /** * Cache for Tags of certain type. For applications that check tags on big numbers of objects, it * would save time to cache associations between tag names and their representative Enum values. * * @author gpogulsky * @author sbhalekar * @param * - type of tag Enum class */ public class Tagger> implements Serializable { private static final long serialVersionUID = -9170158494924659179L; private final Class type; private final String tagName; private final Cache> cache; public Tagger(final Class type) { // This would not work properly with localized Tags. // So far we don't have any Enum-based tags that are localized. // But if they appear, this code should prevent those (throw). this.type = type; this.tagName = Validators.findTagNameIn(type); this.cache = CacheBuilder.newBuilder().build(); } public Optional getTag(final Taggable taggable) { final Optional possibleTagValue = taggable.getTag(this.tagName); if (possibleTagValue.isPresent()) { final String tagValue = possibleTagValue.get(); try { // Referenced from // https://github.com/google/guava/wiki/CachesExplained#from-a-callable // If tagValue is present in the cache then return the value; otherwise execute // function add the value to cache and return it return this.cache.get(tagValue, () -> Validators.fromAnnotation(Tagger.this.type, taggable)); } // this exception is thrown by the Callable in the get method of the cache. Ideally // we should never hit this exception catch (final ExecutionException e) { throw new CoreException("Error getting tag value from the cache", e); } } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/ConfiguredTaggableFilter.java ================================================ package org.openstreetmap.atlas.tags.filters; import java.io.Serializable; import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.configuration.Configuration; /** * List of filters defined in a configuration object. * * @author matthieun */ public class ConfiguredTaggableFilter implements Predicate, Serializable { private static final long serialVersionUID = -3849791821180104953L; public static final String FILTERS_CONFIGURATION_NAME = "filters"; private final List filters; @SuppressWarnings("unchecked") public ConfiguredTaggableFilter(final Configuration configuration) { this.filters = ((List) configuration.get(FILTERS_CONFIGURATION_NAME).valueOption() .orElseThrow(() -> new CoreException("No filters defined in configuration {}", configuration))) .stream().map(TaggableFilter::forDefinition).collect(Collectors.toList()); } @Override public boolean test(final Taggable taggable) { for (final TaggableFilter filter : this.filters) { if (!filter.test(taggable)) { return false; } } return true; } @Override public String toString() { return this.filters.toString(); } protected List getFilters() { return this.filters; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/LineFilterConverter.java ================================================ package org.openstreetmap.atlas.tags.filters; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.TaggableFilter.TreeBoolean; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Parses a {@link TaggableFilter}'s line. * * @author matthieun */ public class LineFilterConverter implements TwoWayConverter { private static final String VALUES_SEPARATOR = ","; private static final String KEY_VALUE_SEPARATOR = "->"; private static final Predicate ALL_VALID = (Predicate & Serializable) taggable -> true; @Override public String backwardConvert(final TaggableFilter object) { return backwardConvert(object, 0); } @Override public TaggableFilter convert(final String object) { return convert(object, TreeBoolean.OR, 0); } private String backwardConvert(final TaggableFilter object, final int numberOfOccurrences) { final Optional definition = object.getDefinition(); if (definition.isPresent()) { return definition.get(); } else { final int numberOfSeparators = numberOfSeparators(numberOfOccurrences); final String separatorCharacter = object.getTreeBoolean().separator(); final StringList separatorList = new StringList(); for (int i = 0; i < numberOfSeparators; i++) { separatorList.add(separatorCharacter); } final String separator = separatorList.join(""); final StringList result = new StringList(); object.getChildren() .forEach(child -> result.add(backwardConvert(child, numberOfOccurrences + 1))); return result.join(separator); } } private TaggableFilter convert(final String object, final TreeBoolean treeBoolean, final int numberOfOccurrences) { final String regex = regex(treeBoolean.separator(), numberOfSeparators(numberOfOccurrences)); final StringList split = StringList.splitByRegex(object, regex); final List children = new ArrayList<>(); if (split.size() > 1) { // There is a split on the initial operation for (final String value : split) { children.add(convert(value, treeBoolean.other(), numberOfOccurrences + 1)); } return new TaggableFilter(children, treeBoolean); } else if (object.contains(treeBoolean.other().separator())) { // We need to keep splitting children.add(convert(object, treeBoolean.other(), numberOfOccurrences + 1)); return new TaggableFilter(children, treeBoolean); } else { final String definition = split.get(0); final Predicate simple = simple(definition); return new TaggableFilter(simple, definition); } } private String escaped(final String value) { if (TreeBoolean.OR.separator().equals(value)) { return "\\" + TreeBoolean.OR.separator(); } return value; } private int numberOfSeparators(final int numberOfOccurrences) { return numberOfOccurrences / 2 + 1; } private String regex(final String character, final int numberOfOccurrences) { if (numberOfOccurrences < 1) { throw new CoreException("Invalid number of occurences for pattern with {}: {}", character, numberOfOccurrences); } // For || an example is "(? simple(final String simple) { if ("".equals(simple)) { return ALL_VALID; } final StringList split = StringList.split(simple, KEY_VALUE_SEPARATOR); if (split.size() != 2) { throw new CoreException("Taggable filter definition \"{}\" is invalid.", simple); } final String key = split.get(0); final StringList values = StringList.split(split.get(1), VALUES_SEPARATOR); return (Serializable & Predicate) taggable -> taggable.containsValue(key, values); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/README.md ================================================ # Tag filtering ## TaggableFilter `TaggableFilter` is an extension of `Predicate` that allows for easy string definitions. With a String containing a `TaggableFilter` definition, one can be created easily: ```java String definition; Predicate filter = TaggableFilter.forDefinition(definition); ``` ## String definition `TaggableFilter` is built with a decision tree that alternates the OR and AND operands. ``` OR / \ / \ AND Test / \ / \ Test Test ``` Each leaf node is a "simple" filter, based on tags. The taggable filter is a collection of simple filters with AND and OR operands. ### Simple filters #### One tag ``` water->pond ``` Only features with water=pond in their tags will pass this test. #### Multiple tag values ``` water->pond,lake,canal ``` Only features with water=pond or water=lake or water=canal in their tags will pass this test. #### Negation ``` water->!pond ``` Anything without water=pond ``` water->!pond,!lake ``` Anything without water=pond or without water=lake. That one is not very useful as it is always true because one of the two will always be true (The water tag can have only one value). ``` water->!pond&water->!lake ``` Anything without water=pond and without water=lake. ``` water->! ``` Anything without water tag. #### Wildcard ``` water->* ``` Anything with a water tag. ### Tree logic AND and OR operands are represented by `&` and `|` respectively. The same operands one level down are `&&` and `||`. Two levels down are `&&&` and `|||` and so on. **All trees always start with OR!** This means that `a&b|c` translates to `(a AND b) OR c`. To achieve `a AND (b OR c)` one needs to write `a&b||c` For example: ``` OR / \ / \ AND highway=motorway / \ / \ highway=service service=parking_aisle ``` Is defined by: ``` highway->service&service->parking_aisle|highway->motorway ``` And ``` OR / \ / \ AND highway=motorway / \ / \ highway=service OR / \ / \ cycleway=lane AND / \ / \ cycleway:lane=* No cycleway tag ``` Is defined by: ``` highway->service&cycleway->lane||cycleway:lane->*&&cycleway->!|highway->motorway ``` #### Backwards compatibility An older version of `TaggableFilter` had only two OR levels possible, and the lower OR level was represented by `^`. This is still allowed and is interpreted by the parser as `||`. ## ConfiguredTaggableFilter `ConfiguredTaggableFilter` is a `TaggableFilter` that can be created from a JSON file that contains multiple of those above filters: ```javascript { "filters": [ "access->!no|motor_vehicle->yes|motorcar->yes|vehicle->yes", "oneway->!reversible", "route->ferry|man_made->pier|junction->roundabout|highway->motorway" ] } ``` Each line in that filter must pass for the `TaggableFilter` to pass a `Taggable`. It could be translated to an AND of each filter. Here it is equivalent to: ```javascript { "filters": [ "access->!no||motor_vehicle->yes||motorcar->yes||vehicle->yes&oneway->!reversible&route->ferry||man_made->pier||junction->roundabout||highway->motorway" ] } ``` ## RegexTaggableFilter `RegexTaggableFilter` is an extension of `Predicate` that allows filtering certain tag values based on regex patterns. The filter also accepts a map of values that are excepted from the regex patterns. In order to create `RegexTaggableFilter` one must provide: - a set of String values representing the tag names that need to be checked - a set of String values representing the regex used to match the values - optionally, a map of tag names and sets of excepted values. For example: ```java final Set tagNames = new HashSet<>(Arrays.asList("source", "highway")); final Set regex = new HashSet<>(Arrays.asList(".*(?i)\\bmap\\b.*", ".*(?i)\\bsecondary\\b.*")); final HashMap> exceptions = new HashMap<>(Map.of( "source", Set.of("personal map", "public map") )); ``` For a simple filter there is the option to create a `RegexTaggableFilter` from a `String`. This constructor does not support passing the exception map which defaults in an empty `HashMap`. Example of `String` for this case: ``` "source,highway|.*illegal.*,.*secondary.*" ``` The first part before the `|` character represents the list of tag names separated by commas. The second part after the `|` represents the list of regex patterns, also separated by commas. Do take into account that commas in the regex pattern might cause an incorrect construction since the creation of the list of regex from string uses commas for splitting. If the `Taggable` object contains a key that is part of the tagNames set, matches at least one of the given regex patterns and the tag-value combination is not part of the exceptions map, the test method will return a positive response. Also, if the list of tag names is empty, the test method will always return true. The `RegexTaggableFilter` also offers a `getMatchedTags` method that returns a joined String of all the tag names that passed the test. ##ConfiguredFilter The `ConfiguredFilter` reads a json configuration files that follows a certain schema and creates filters based on it, including `TaggableFilter` and `RegexTaggableFilter`. Example of configuration json: ``` { "my": { "conf": { "filter": { "predicate": "....", "geometry.wkb": [ "...", "..." ], "taggableFilter": "...", "regexTaggableFilter": "..." } } } } ``` The filter can be accessed using "my.conf" as root, and "filter" as name. The `ConfiguredFilter` predicate returns true if all the composing filter predicates are true. ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/RegexTaggableFilter.java ================================================ package org.openstreetmap.atlas.tags.filters; import java.io.Serializable; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.tags.Taggable; /** * This {@link Taggable} filter relies on regex patterns to verify specific tag values. * * @author mm-ciub on 11/09/2020. */ public class RegexTaggableFilter implements Predicate, Serializable { private static final String FILTER_DELIMITER = "\\|"; private static final String COMMA = ","; private final Set tagNames; private final Set regexPatterns; private final Map> exceptions; /** * @param tagNames * - the set of tag names who's value will be tested * @param regex * - a set of regex strings that will validate the value for each tag * @param exceptions * - a map of tag names and values that are valid regardless if they match the regex * pattern */ public RegexTaggableFilter(final Set tagNames, final Set regex, final Map> exceptions) { this.tagNames = tagNames; this.regexPatterns = regex.stream().map(Pattern::compile).collect(Collectors.toSet()); this.exceptions = exceptions != null ? exceptions : new HashMap<>(); } /** * Useful constructor for inline configuration. This option does not support passing exceptions. * * @param definition * - The {@link String} definition of the filter example: * "tagName1,tagName2|regex1,regex2,regex3" */ public RegexTaggableFilter(final String definition) { this.exceptions = new HashMap<>(); final String[] filter = definition.split(FILTER_DELIMITER); if (filter.length == 2) { this.tagNames = Set.of(filter[0].split(COMMA)); this.regexPatterns = Stream.of(filter[1].split(COMMA)).map(Pattern::compile) .collect(Collectors.toSet()); } else { this.tagNames = new HashSet<>(); this.regexPatterns = new HashSet<>(); } } /** * Returns a joined String containing the names of the tags that match at least one of the regex * patterns and are not an exception * * @param taggable * - the element containing the tags to be checked * @return String - example: source,barrier,boundary */ public String getMatchedTags(final Taggable taggable) { final Set matchedTags = findMatches(taggable); return String.join(",", matchedTags); } @Override public boolean test(final Taggable taggable) { if (this.tagNames.isEmpty()) { return true; } final Set matchedTags = findMatches(taggable); return !matchedTags.isEmpty(); } private Set findMatches(final Taggable taggable) { final Set matchedTags = new HashSet<>(); for (final String tagName : this.tagNames) { final Optional tagValue = taggable.getTag(tagName); if (tagValue.isPresent()) { final Optional match = this.regexPatterns.stream() .map(pattern -> pattern.matcher(tagValue.get())).filter(Matcher::find) .findAny(); if (match.isPresent() && !(this.exceptions.containsKey(tagName) && this.exceptions.get(tagName).contains(tagValue.get()))) { matchedTags.add(tagName); } } } return matchedTags; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/TaggableFilter.java ================================================ package org.openstreetmap.atlas.tags.filters; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; /** * {@link Taggable} filter that relies on a String definition *

* Examples of String definition: *

* highway=motorway AND name=[not empty]
* highway->motorway&name->* *

* highway=motorway OR oneway=[not]yes
* highway->motorway|oneway->!yes *

* highway=motorway OR [no "name" tag]
* highway->motorway|name->! *

* amenity=bus_station OR highway=bus_stop OR ( (bus=* OR trolleybus=*) AND * public_transport=[stop_position OR platform OR station] )
* amenity->bus_station|highway->bus_stop|bus->*||trolleybus->*&public_transport-> * stop_position, platform,station * * @author matthieun */ public class TaggableFilter implements Predicate, Serializable { /** * @author matthieun */ protected enum TreeBoolean { AND, OR; public TreeBoolean other() { switch (this) { case AND: return OR; case OR: return AND; default: throw new CoreException(ERROR_MESSAGE, this); } } public String separator() { switch (this) { case AND: return "&"; case OR: return "|"; default: throw new CoreException(ERROR_MESSAGE, this); } } } private static final long serialVersionUID = 5697377487014951158L; private static final String ERROR_MESSAGE = "Unknown TreeBoolean {}"; private final List children; private final TreeBoolean treeBoolean; private final Predicate simple; private final String definition; public static TaggableFilter forDefinition(final String definition) { return new LineFilterConverter().convert(definition); } /** * @param definition * The {@link String} definition of the filter * @deprecated Use {@code TaggableFilter.forDefinition(definition)} instead. */ @Deprecated() public TaggableFilter(final String definition) { this(TaggableFilter.forDefinition(definition)); } protected TaggableFilter(final List children, final TreeBoolean treeBoolean) { this.children = children; this.treeBoolean = treeBoolean; this.simple = null; this.definition = null; } protected TaggableFilter(final Predicate simple, final String definition) { this.children = new ArrayList<>(); this.treeBoolean = TreeBoolean.OR; this.simple = simple; this.definition = definition; } private TaggableFilter(final TaggableFilter other) { this.children = other.children; this.treeBoolean = other.treeBoolean; this.simple = other.simple; this.definition = other.definition; } public TaggableMatcher convertToTaggableMatcher() { return new TaggableFilterToMatcherConverter().convert(this); } @Override public boolean test(final Taggable taggable) { if (this.simple != null) { return this.simple.test(taggable); } if (this.children.isEmpty()) { throw new CoreException("Malformed predicate {}", this); } switch (this.treeBoolean) { case AND: return this.children.stream().allMatch(tree -> tree.test(taggable)); case OR: return this.children.stream().anyMatch(tree -> tree.test(taggable)); default: throw new CoreException(ERROR_MESSAGE, this); } } @Override public String toString() { return new LineFilterConverter().backwardConvert(this); } protected List getChildren() { return this.children; } protected Optional getDefinition() { return Optional.ofNullable(this.definition); } protected Predicate getSimple() { return this.simple; } protected TreeBoolean getTreeBoolean() { return this.treeBoolean; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/TaggableFilterToMatcherConverter.java ================================================ package org.openstreetmap.atlas.tags.filters; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Token; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.Converter; /** * @author lcram */ public class TaggableFilterToMatcherConverter implements Converter { protected static String toTaggableMatcherDefinition(final TaggableFilter filter) { if (filter.getSimple() != null) { final String definition = filter.getDefinition().orElseThrow( () -> new CoreException("Simple filter definition was not present")); final String[] split = definition.split("->"); if (split.length != 2) { throw new CoreException("Array length was not 2 for split('->') on definition `{}'", definition); } final String key = split[0]; final String value = split[1]; final String[] commaSeparatedValues = value.split(","); if (commaSeparatedValues.length == 1) { return getKeyValueForSingleValue(key, value); } return getKeyValueForMultiValue(key, commaSeparatedValues); } return Token.TokenType.PAREN_OPEN.getLiteralValue() + new StringList(filter.getChildren().stream() .map(TaggableFilterToMatcherConverter::toTaggableMatcherDefinition) .collect(Collectors.toList())) .join(" " + filter.getTreeBoolean().separator() + " ") + Token.TokenType.PAREN_CLOSE.getLiteralValue(); } private static String getKeyValueForMultiValue(final String key, final String[] commaSeparatedValues) { final StringBuilder keyValue = new StringBuilder(); keyValue.append(key); keyValue.append(Token.TokenType.EQUAL.getLiteralValue()); keyValue.append(Token.TokenType.PAREN_OPEN.getLiteralValue()); // check for illegal values final String commaSeparatedValuesString = new StringList(commaSeparatedValues).join(","); for (final String value : commaSeparatedValues) { if (Token.TokenType.BANG.getLiteralValue().equals(value)) { throw new CoreException( "Cannot transpile `{}->{}' since composite value `{}' contains a lone `{}' operator." + "\n" + "Expression `{}->{}' is ambiguous and order dependent, please rewrite your TaggableFilter to remove it.", key, commaSeparatedValuesString, commaSeparatedValuesString, Token.TokenType.BANG.getLiteralValue(), key, commaSeparatedValuesString); } if ("*".equals(value)) { throw new CoreException( "Cannot transpile `{}->{}' since composite value `{}' contains a lone `*' operator." + "\n" + "Expression `{}->{}' is ambiguous and order dependent, please rewrite your TaggableFilter to remove it.", key, commaSeparatedValuesString, commaSeparatedValuesString, key, commaSeparatedValuesString); } } keyValue.append(new StringList(commaSeparatedValues) .join(" " + Token.TokenType.OR.getLiteralValue() + " ")); keyValue.append(Token.TokenType.PAREN_CLOSE.getLiteralValue()); return keyValue.toString(); } private static String getKeyValueForSingleValue(final String key, final String value) { if (value.charAt(0) == Token.TokenType.BANG.getLiteralValue().charAt(0) && value.length() == 1) { // case foo->! // return !foo return Token.TokenType.BANG.getLiteralValue() + key; } else if (value.charAt(0) == Token.TokenType.BANG.getLiteralValue().charAt(0) && value.length() > 1) { // case foo->!bar final String valueWithNoLeadingBang = value.substring(1); // return (foo!=bar | !foo) return Token.TokenType.PAREN_OPEN.getLiteralValue() + key + Token.TokenType.BANG_EQUAL.getLiteralValue() + valueWithNoLeadingBang + " " + Token.TokenType.OR.getLiteralValue() + " " + Token.TokenType.BANG.getLiteralValue() + key + Token.TokenType.PAREN_CLOSE.getLiteralValue(); } else if (value.charAt(0) == '*' && value.length() == 1) { // case foo->* // return foo return key; } else if (value.charAt(0) == '*' && value.length() > 1) { // case foo->*bar final String newValue = value.substring(1); if (hasRegexCharacter(newValue)) { throw new CoreException( "Cannot transpile `{}->{}' since new value `{}' contains a regex control character.", key, value, newValue); } // return foo=/.*bar/ return key + Token.TokenType.EQUAL.getLiteralValue() + Token.TokenType.REGEX.getLiteralValue() + ".*" + newValue + Token.TokenType.REGEX.getLiteralValue(); } else if (value.charAt(value.length() - 1) == '*' && value.length() > 1) { // case foo->bar* final String newValue = value.substring(0, value.length() - 1); if (hasRegexCharacter(newValue)) { throw new CoreException( "Cannot transpile `{}->{}' since new value `{}' contains a regex control character.", key, value, newValue); } // return foo=/bar.*/ return key + Token.TokenType.EQUAL.getLiteralValue() + Token.TokenType.REGEX.getLiteralValue() + newValue + ".*" + Token.TokenType.REGEX.getLiteralValue(); } else { // case foo->bar // return foo=bar return key + Token.TokenType.EQUAL.getLiteralValue() + value; } } private static boolean hasRegexCharacter(final String string) { return string.matches("^.*[^a-zA-Z0-9_ ].*$"); } @Override public TaggableMatcher convert(final TaggableFilter filter) { String taggableMatcherDefinition = toTaggableMatcherDefinition(filter); /* * Remove leading `(' and trailing `)' if present, since they are redundant. For compound * expressions, the converter always adds a redundant pair of parentheses (due to the * implementation), so we know it is safe to remove them. There may be additional redundant * parentheses that we miss. For example, `foo->bar&baz->bat' will become `((foo=bar & * baz=bat))', so we'll remove the first pair of redundant parentheses but miss the inner * pair. Ultimately we should find a better way to handle this. */ if (taggableMatcherDefinition.charAt(0) == Token.TokenType.PAREN_OPEN.getLiteralValue() .charAt(0) && taggableMatcherDefinition.charAt(taggableMatcherDefinition.length() - 1) == Token.TokenType.PAREN_CLOSE.getLiteralValue().charAt(0)) { taggableMatcherDefinition = taggableMatcherDefinition.substring(1, taggableMatcherDefinition.length() - 1); } return TaggableMatcher.from(taggableMatcherDefinition); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/ConfiguredTaggableMatcher.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher; import java.io.Serializable; import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.utilities.configuration.Configuration; /** * List of {@link TaggableMatcher}s defined in a configuration object. * * @author lcram */ public class ConfiguredTaggableMatcher implements Predicate, Serializable { public static final String MATCHERS_CONFIGURATION_NAME = "matchers"; private static final long serialVersionUID = -1870768831799297979L; private final List matchers; @SuppressWarnings("unchecked") public ConfiguredTaggableMatcher(final Configuration configuration) { this.matchers = ((List) configuration.get(MATCHERS_CONFIGURATION_NAME).valueOption() .orElseThrow(() -> new CoreException("No matchers defined in configuration {}", configuration))) .stream().map(TaggableMatcher::from).collect(Collectors.toList()); } @Override public boolean test(final Taggable taggable) { for (final TaggableMatcher matcher : this.matchers) { if (!matcher.test(taggable)) { return false; } } return true; } @Override public String toString() { return this.matchers.toString(); } protected List getMatchers() { return this.matchers; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/README.md ================================================ # TaggableMatcher #### Table of Contents 1. [Quick Intro and Examples](#quick-intro-and-examples) * [Some sample matchers with explanations](#some-sample-matchers-with-explanations) 2. [Basic Semantics](#basic-semantics) 3. [Syntax Rules](#syntax-rules) * [Table of Operators](#table-of-operators) * [Precedence](#precedence) * [Escaping and Whitespace](#escaping-and-whitespace) * [More On Quoting](#more-on-quoting) * [Regex](#regex) * [Tree Representation](#tree-representation) 4. [Converting Your Old TaggableFilters](#converting-your-old-taggableFilters) ## Quick Intro and Examples `TaggableMatcher` is an extension of `Predicate` that supports intuitive string definitions. You can create a new `TaggableMatcher` like: ```java // Create a simple predicate: String definition = "highway=primary"; Predicate predicate = TaggableMatcher.from(definition); // You could also do this to access the extra TaggableMatcher methods: TaggableMatcher matcher = TaggableMatcher.from(definition + " & name=I280"); ``` ### Some sample matchers with explanations Match any `Taggable` containing a "name" tag with value "John's Coffee Shop": ``` name="John's Coffee Shop" ``` Match any `Taggable` that is a "water=pond" or is a "water=lake" that is *not* named "Lake Michigan": ``` water = pond | water = lake & name != Lake\ Michigan ``` Match any tertiary or residential highway whose "name" tag contains the word "street" or "Street": ``` (highway=tertiary | highway=residential) & name=/.*\b[s|S]treet\b.*/ ``` Match all non-highway features, but also include primary and secondary highways: ``` !highway | highway=(primary | secondary) ``` Match any `Taggable` that is a "natural=lake" or "water=lake": ``` (natural | water) = lake ``` And that's really all there is to it! Enough to get you started. Read on to get more details about the syntax rules and various features. ## Basic Semantics Consider the following definition: ``` foo = bar ``` This will create a `TaggableMatcher` with a single `key=value` pair constraint, "foo=bar". `TaggableMatcher` constraints are case sensitive, but they are also *inclusive*, meaning that they automatically match anything containing *at least* the specified constraint. So the above `TaggableMatcher` would match both `Taggable(foo=bar)` as well as `Taggable(baz=bat, foo=bar)`. If we wanted to exclude the `Taggable` containing the "baz=bat" `key=value` pair, we would need to explicitly specify that in a compound constraint. One way to do this is by combining the original `key=value` constraint and a negated `key-only` constraint with an `&` (AND) operator, like: ``` foo=bar & !baz ``` In the above example, `!baz` is the `key-only` constraint. Any constraint that does not include an `=` or `!=` operator will become a `key-only` constraint and will match against the *key only*, as the name suggests. So something like: ``` water & !name ``` is equivalent to the old `TaggableFilter` syntax: ``` water->*&name->! ``` which will match any `Taggable` that both *has* a "water" key and does *not have* a "name" key, with no constraint on the associated values. ## Syntax Rules `TaggableMatcher` syntax follows basic boolean expression syntax, with the standard boolean `==`/`!=` operators replaced by `=`/`!=` to denote `key=value` pair constraints. Additionally, like boolean expressions, chained `=`/`!=` operators are forbidden by the semantic checker since these would be nonsense in the context of tag matching (more on that in the `Tree Representation` section). For example, this `TaggableMatcher` would generate the following error: ``` foo = (bar = baz) org.openstreetmap.atlas.exception.CoreException: semantic error: invalid nested equality operators ``` ### Table of Operators | Operator | Description | | -------- | ----------- | | `( .. )` | Group a subexpression to increase its precedence | | `!` | Negate a subexpression | | `=` | Specify a `key=value` pair constraint that must be **included** in a given `Taggable` | | `!=` | Specify `key=value` pair constraint that must be **excluded** from a given `Taggable` | | `&` | Specify an AND relationship between constraints or between specific keys/values within a constraint | | `^` | Specify an XOR relationship between constraints or between specific keys/values within a constraint | | `\|` | Specify an OR relationship between constraints or between specific keys/values within a constraint | ### Precedence `TaggableMatcher` operator precedence follows that of standard boolean expressions, but again with the `=`/`!=` operators taking the place of the standard boolean `==`/`!=` operators. The following snippet lists operators in descending order, from highest precedence to lowest precedence. `TaggableMatcher` will evaluate higher precedence operators first and will fall back on left-to-right evaluation. ``` ( .. ) ! =, != (these have equivalent precedence) & ^ | ``` ### Escaping and Whitespace For `TaggableMatcher` definitions, whitespace is not meaningful by default - the lexer will simply ignore it. This means that, for example: ``` foo=bar|baz=bat ``` and ``` foo = bar | baz = bat ``` are semantically equivalent. In order to include significant whitespace in a constraint, you must either escape the whitespace or wrap the whitespace-containing literal in quotes (`TaggableMatcher` supports single ' or double " quoted literals). For example, the following matcher will fail with a syntax error: ``` name = Lake Michigan & water = lake org.openstreetmap.atlas.exception.CoreException: syntax error: unexpected token LITERAL(Michigan) name = Lake Michigan & water = lake ~~~~~~~~~~~~^ ``` Instead, you must do either: ``` name = Lake\ Michigan & water = lake ``` or ``` name = "Lake Michigan" & water = lake ``` or ``` name = 'Lake Michigan' & water = lake ``` You may also use `\`, `"`, `'` to escape operator characters. For example, to match a tag that contains a literal "=" character you could do: ``` math = 2+2\=4 ``` or ``` math="2+2=4" ``` ### More On Quoting As shown above, `TaggableMatcher` supports both single and double quoting. There are not many differences between the two, other than their escaping rules. Specifically, a single quoted string may contain unescaped double quote characters but must escape all inner single quote characters. And vice versa for double quoted strings. A few examples of this: ``` // The ' does not need escaping, but the " do name="John's \"Coffee\" Shop" ``` ``` // Here we must escape the ', but we can skip escaping the " name='John\'s "Coffee" Shop' ``` Finally note that unlike many shell languages, a quoted string constitutes a complete literal, and the lexer will **not** coalesce multiple consecutive literals together. So something like: ``` foo = bar" and baz" ``` will generate the following syntax error: ``` org.openstreetmap.atlas.exception.CoreException: syntax error: unexpected token LITERAL( and baz) foo = bar" and baz" ~~~~~~~~~^ ``` ### Regex `TaggableMatcher` supports regex matching for keys and values with the following syntax: ``` name = /[l|L]ake.*/ ``` Anything between the `/` symbols will be treated as a regex operand. Regexes are evaluated using Java's `String#matches(String)` method, which means that in order to get a match, the regex must match the **entire** key or value string. So for example, the above regex would match `Taggable(name=lake michigan)`, but it would **not** match `Taggable(name=Arrow Lake)`. In order to match this second `Taggable` as well as the first, the regex would need to be: ``` name = /.*[l|L]ake.*/ ``` You can escape a closing `/` symbol using the escape symbol `\`, like so: `\/`. This will pass the `\/` directly into the regex. For example: ``` name=/foo\/bar/ ``` would result in a matcher regex `foo\/bar`, which would match the string "foo/bar". https://regex101.com is a nice tool for testing regex and matching. Just note that unlike `TaggableMatcher` regex, it will match substrings of the input. ### Tree Representation It can be helpful to think about `TaggableMatchers` as syntax trees with the various operators as internal nodes. You can print out pretty Unicode trees for your `TaggableMatchers` using the `TaggableMatcherPrinterCommand`, which you can call from the command line as `print-matcher` using [Atlas Shell Tools](https://github.com/osmlab/atlas/tree/dev/atlas-shell-tools). For example, we could print the following `TaggableMatcher` as this tree: ``` a = b & c = d | e != f ``` ``` | ┌───────────┴───────────┐ & != ┌─────┴─────┐ ┌─────┴─────┐ = = e f ┌──┴──┐ ┌──┴──┐ a b c d ``` The `TaggableMatcher` is evaluated by walking the tree in post-order (LRN). As mentioned in an earlier section, chained "="/"!=" operators are forbidden since expressions containing them are nonsensical in the context of tag matching. Expressions with chained equality operators become trees in which an equality operator is present in the subtree of another equality operator. For example, the matcher definition: ``` a = b = c ``` becomes this tree: ``` = ┌─────┴─────┐ a = ┌──┴──┐ b c ``` The `TaggableMatcher` semantic checker is able to detect subtrees like this and reject the matcher definition as invalid. In fact, the `TaggableMatcherPrinterCommand` will not even allow you to print this tree since it fails the semantic check. ## Converting Your Old TaggableFilters The `TaggableMatcherPrinterCommand` mentioned earlier can also help you convert old `TaggableFilter` definitions into the new `TaggableMatcher` syntax. All you need to do is specify the `--reverse` option at the command line, and then pass as many `TaggableFilter` definitions as you would like to convert. For example: ``` $ atlas print-matcher --reverse 'foo->bar|baz->bat' 'cat->mat&hat->zat,hello' foo=bar | baz=bat (cat=mat & hat=(zat | hello)) ``` Note that occasionally, `print-matcher` will include unnecessary parentheses its generated `TaggableMatcher` definitions (like the outermost parentheses in the above example). You can safely remove those. Also note that `print-matcher` will fail to convert certain kinds of valid `TaggableFilters`, specifically those that make use of ambiguous combinations of operators. For example: ``` $ atlas print-matcher --reverse 'foo->bar,!,bat' print-matcher: error: Cannot transpile `foo->bar,!,bat' since composite value `bar,!,bat' contains a lone `!' operator. Expression `foo->bar,!,bat' is ambiguous and order dependent, please rewrite your TaggableFilter to remove it. ``` You can also use the `print-matcher` command to easily print a tree for an old-style `TaggableFilter`, should you want help debugging one. Make use of your shell's command substitution feature like so: ``` atlas print-matcher "$(atlas print-matcher --reverse 'foo->bar||baz->bat&cat->hat')" & ┌───────────────┴───────────────┐ | = ┌───────┴───────┐ ┌───────┴───────┐ = = cat hat ┌───┴───┐ ┌───┴───┐ foo bar baz bat ``` Finally, as mentioned above, you can obtain `print-matcher` by installing [Atlas Shell Tools](https://github.com/osmlab/atlas/tree/dev/atlas-shell-tools). ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/TaggableMatcher.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.Predicate; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Lexer; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Parser; import org.openstreetmap.atlas.tags.filters.matcher.parsing.SemanticChecker; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Token; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.ASTNode; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.TreePrinter; /** * @author lcram */ public final class TaggableMatcher implements Predicate, Serializable { private static final long serialVersionUID = -3505184622005535575L; private final ASTNode rootNode; private final String definition; public static TaggableMatcher from(final String definition) { if (definition.isEmpty()) { return new TaggableMatcher(null, definition); } final List tokens = new Lexer().lex(definition); final ASTNode rootNode = new Parser(tokens, definition).parse(); new SemanticChecker().check(rootNode); return new TaggableMatcher(rootNode, definition); } private TaggableMatcher(final ASTNode rootNode, final String definition) { this.rootNode = rootNode; this.definition = definition; } public String getDefinition() { return this.definition; } /** * Get the length of the longest line for the printed tree returned by * {@link TaggableMatcher#prettyPrintTree()}. * * @return the length of the longest line of the printed tree */ public long lengthOfLongestLineForPrintedTree() { if (this.rootNode == null) { return 0L; } return TreePrinter.lengthOfLongestLineForTree(this.rootNode); } /** * Print this {@link TaggableMatcher} in syntax tree form. * * @return this {@link TaggableMatcher} as a tree */ public String prettyPrintTree() { if (this.rootNode == null) { return ""; } return TreePrinter.print(this.rootNode); } @Override public boolean test(final Taggable taggable) { if (this.rootNode == null) { return true; } final Map tags = taggable.getTags(); final List keys = new ArrayList<>(); final List values = new ArrayList<>(); for (final Map.Entry entry : tags.entrySet()) { keys.add(entry.getKey()); values.add(entry.getValue()); } return this.rootNode.match(keys, values); } @Override public String toString() { return this.getClass().getSimpleName() + "(" + this.definition + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/Lexer.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; /** * This class can transform an input {@link String} into a sequence of {@link Token}s recognizable * by the {@link Parser}. * * @author lcram */ public class Lexer { /** * @author lcram */ private static class InputBuffer { static final int EOF = -1; private final String string; private int position; InputBuffer(final String string) { this.string = string; this.position = 0; } int consumeCharacter() { if (this.position >= this.string.length()) { return EOF; } return this.string.charAt(this.position++); } int peek() { if (this.position >= this.string.length()) { return EOF; } return this.string.charAt(this.position); } void unconsume() { if (this.position > 0) { this.position--; } } } /** * @author lcram */ private static class LexemeBuffer { private final List characters; LexemeBuffer() { this.characters = new ArrayList<>(); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); for (final Character character : this.characters) { builder.append(character); } return builder.toString(); } void addCharacter(final char character) { this.characters.add(character); } void clear() { this.characters.clear(); } LexemeBuffer stripLeading() { this.characters.remove(0); return this; } LexemeBuffer stripTrailing() { this.characters.remove(this.characters.size() - 1); return this; } } public static String debugString(final List lexedTokens) { final StringBuilder builder = new StringBuilder(); for (final Token token : lexedTokens) { builder.append(token.toString()); builder.append(", "); } return builder.toString(); } /** * Lex a given input line. * * @param inputLine * the input line * @return a {@link List} of the processed {@link Token}s */ public List lex(final String inputLine) // NOSONAR { final List lexedTokens = new ArrayList<>(); final LexemeBuffer lexemeBuffer = new LexemeBuffer(); final InputBuffer inputBuffer = new InputBuffer(inputLine); while (inputBuffer.peek() != InputBuffer.EOF) { if (isKeyValueCharacter(inputBuffer.peek()) || inputBuffer.peek() == Token.TokenType.ESCAPE.getLiteralValue().charAt(0)) { literal(inputBuffer, lexemeBuffer, lexedTokens); } else if (isWhitespaceCharacter(inputBuffer.peek())) { whitespace(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.EQUAL.getLiteralValue().charAt(0)) { equal(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.AND.getLiteralValue().charAt(0)) { and(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.OR.getLiteralValue().charAt(0)) { or(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.XOR.getLiteralValue().charAt(0)) { xor(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.PAREN_OPEN.getLiteralValue().charAt(0)) { parenOpen(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.PAREN_CLOSE.getLiteralValue().charAt(0)) { parenClose(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.BANG.getLiteralValue().charAt(0)) { bangOrBangEqual(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.REGEX.getLiteralValue().charAt(0)) { regex(inputBuffer, lexemeBuffer, lexedTokens); } else if (inputBuffer.peek() == Token.TokenType.DOUBLE_QUOTE.getLiteralValue().charAt(0)) { quote(inputBuffer, lexemeBuffer, lexedTokens, Token.TokenType.DOUBLE_QUOTE.getLiteralValue().charAt(0)); } else if (inputBuffer.peek() == Token.TokenType.SINGLE_QUOTE.getLiteralValue().charAt(0)) { quote(inputBuffer, lexemeBuffer, lexedTokens, Token.TokenType.SINGLE_QUOTE.getLiteralValue().charAt(0)); } else { throw new CoreException("unknown char {}", (char) inputBuffer.peek()); } lexemeBuffer.clear(); } // Remove all whitespace from token stream return lexedTokens.stream().filter(token -> token.getType() != Token.TokenType.WHITESPACE) .collect(Collectors.toList()); } private void and(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens .add(new Token(Token.TokenType.AND, lexemeBuffer.toString(), inputBuffer.position)); } private void bangOrBangEqual(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); if (inputBuffer.peek() == Token.TokenType.EQUAL.getLiteralValue().charAt(0)) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens.add(new Token(Token.TokenType.BANG_EQUAL, lexemeBuffer.toString(), inputBuffer.position)); } else { lexedTokens.add( new Token(Token.TokenType.BANG, lexemeBuffer.toString(), inputBuffer.position)); } } private void equal(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, // NOSONAR final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens.add( new Token(Token.TokenType.EQUAL, lexemeBuffer.toString(), inputBuffer.position)); } private boolean isKeyValueCharacter(final int character) { /* * Anything not in this list counts as a key/value character. To use characters on this list * in a key/value literal, users must use escapes '\' or double quotes '"'. */ return ((char) character) != Token.TokenType.AND.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.OR.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.XOR.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.EQUAL.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.PAREN_OPEN.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.PAREN_CLOSE.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.REGEX.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.BANG.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.DOUBLE_QUOTE.getLiteralValue().charAt(0) && ((char) character) != Token.TokenType.SINGLE_QUOTE.getLiteralValue().charAt(0) && !isWhitespaceCharacter((char) character); } private boolean isWhitespaceCharacter(final int character) { return Character.isWhitespace((char) character); } private void literal(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { int character; do { character = inputBuffer.consumeCharacter(); if (character == InputBuffer.EOF) { break; } if (character == Token.TokenType.ESCAPE.getLiteralValue().charAt(0)) { /* * If we see an ESCAPE, consume the immediate next character and place it in the * lexeme buffer. We throw the escape character '\' out. If the escape character * comes just before the EOF, fail. */ final int escaped = inputBuffer.consumeCharacter(); if (escaped == InputBuffer.EOF) { throwSyntaxError("EOF after '\\'", inputBuffer, inputBuffer.string); } lexemeBuffer.addCharacter((char) escaped); } else { lexemeBuffer.addCharacter((char) character); } } while (isKeyValueCharacter(character)); if (character != InputBuffer.EOF) { /* * We reached the end of the putative LITERAL token, so give back the non-literal * character to the input buffer for the main loop to re-process. */ inputBuffer.unconsume(); lexemeBuffer.stripTrailing(); } lexedTokens.add( new Token(Token.TokenType.LITERAL, lexemeBuffer.toString(), inputBuffer.position)); } private void or(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens .add(new Token(Token.TokenType.OR, lexemeBuffer.toString(), inputBuffer.position)); } private void parenClose(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens.add(new Token(Token.TokenType.PAREN_CLOSE, lexemeBuffer.toString(), inputBuffer.position)); } private void parenOpen(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens.add(new Token(Token.TokenType.PAREN_OPEN, lexemeBuffer.toString(), inputBuffer.position)); } private void quote(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens, final char quoteType) { int character; do { character = inputBuffer.consumeCharacter(); if (character == InputBuffer.EOF) { throwSyntaxError("EOF after `" + quoteType + "'", inputBuffer, inputBuffer.string); } if (character == Token.TokenType.ESCAPE.getLiteralValue().charAt(0)) { final int escaped = inputBuffer.consumeCharacter(); lexemeBuffer.addCharacter((char) escaped); } else { lexemeBuffer.addCharacter((char) character); } } while (inputBuffer.peek() != quoteType); // consume the trailing "/' inputBuffer.consumeCharacter(); // Strip leading "/' character final String lexeme = lexemeBuffer.stripLeading().toString(); if (quoteType == Token.TokenType.DOUBLE_QUOTE.getLiteralValue().charAt(0)) { lexedTokens.add(new Token(Token.TokenType.DOUBLE_QUOTE, lexeme, inputBuffer.position)); } else if (quoteType == Token.TokenType.SINGLE_QUOTE.getLiteralValue().charAt(0)) { lexedTokens.add(new Token(Token.TokenType.SINGLE_QUOTE, lexeme, inputBuffer.position)); } else { throw new CoreException("Unknown quote type `{}'", quoteType); } } private void regex(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { int character; do { character = inputBuffer.consumeCharacter(); if (character == InputBuffer.EOF) { throwSyntaxError("EOF after '/'", inputBuffer, inputBuffer.string); } if (character == Token.TokenType.ESCAPE.getLiteralValue().charAt(0)) { /* * If the user is specifically escaping a '/', we need to make sure that '\/' gets * into the regex and the '/' will not be interpreted as an end to the regex. */ if (inputBuffer.peek() == Token.TokenType.REGEX.getLiteralValue().charAt(0)) { lexemeBuffer.addCharacter((char) character); final int escapedForwardSlash = inputBuffer.consumeCharacter(); lexemeBuffer.addCharacter((char) escapedForwardSlash); } // Otherwise, pass the '\' forward into the regex normally else { lexemeBuffer.addCharacter((char) character); } } else { lexemeBuffer.addCharacter((char) character); } } while (inputBuffer.peek() != Token.TokenType.REGEX.getLiteralValue().charAt(0)); // consume the trailing '/' inputBuffer.consumeCharacter(); // Strip leftover leading '/' character final String lexeme = lexemeBuffer.stripLeading().toString(); lexedTokens.add(new Token(Token.TokenType.REGEX, lexeme, inputBuffer.position)); } private void throwSyntaxError(final String unexpected, final InputBuffer inputBuffer, final String inputLine) { final String arrow = "~".repeat(Math.max(0, inputBuffer.position)) + "^"; if (unexpected != null) { throw new CoreException("syntax error: unexpected {}\n{}\n{}", unexpected, inputLine, arrow); } throw new CoreException("syntax error: unexpected input\n{}\n{}", inputLine, arrow); } private void whitespace(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens.add(new Token(Token.TokenType.WHITESPACE, lexemeBuffer.toString(), inputBuffer.position)); } private void xor(final InputBuffer inputBuffer, final LexemeBuffer lexemeBuffer, final List lexedTokens) { lexemeBuffer.addCharacter((char) inputBuffer.consumeCharacter()); lexedTokens .add(new Token(Token.TokenType.XOR, lexemeBuffer.toString(), inputBuffer.position)); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/Parser.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.ASTNode; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.AndOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.BangOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.BinaryOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.EqualsOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.LiteralOperand; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.Operand; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.OrOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.RegexOperand; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.UnaryOperator; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.XorOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class can transform a sequence of {@link Token}s into a syntactically valid abstract syntax * tree (AST) that can then be checked by the {@link SemanticChecker}. Finally, a * {@link TaggableMatcher} may walk this AST to determine if a {@link Taggable}'s tag map * corresponds to the matcher. This {@link Parser} implements an LL(1) {@link TaggableMatcher} * expression grammar using recursive descent. The grammar can be found below in comment form. * * @author lcram */ public class Parser { /* * The Grammar. Operator precedence is handled in the standard way. '=' and '!=' are treated as * extremely "sticky" (i.e. high precedence) operators. The grammar is not capable of detecting * "nested" '=' and '!=' operators (e.g. foo=(bar=baz)), which are syntactically valid but * semantically invalid. Syntax trees containing nested equality operators must be dealt with at * a later stage. */ // OR -> XOR OR' // OR' -> | XOR OR' // OR' -> '' // XOR -> AND XOR' // XOR' -> | AND XOR' // XOR' -> '' // AND -> EQ AND' // AND' -> & EQ AND' // AND' -> '' // EQ -> VALUE EQ' // EQ' -> = VALUE EQ' // EQ' -> != VALUE EQ' // EQ' -> '' // VALUE -> ( OR ) // VALUE -> ! VALUE // VALUE -> literal // VALUE -> /regex/ /** * @author lcram */ private static class TokenBuffer { private final List tokens; private final String inputLine; private int position; TokenBuffer(final List tokens, final String inputLine) { this.tokens = tokens; this.inputLine = inputLine; this.position = 0; } void nextToken() { if (this.position < this.tokens.size()) { this.position++; } } Token peek() { if (this.position >= this.tokens.size()) { return new Token(Token.TokenType.EOF, null, this.inputLine.length()); } return this.tokens.get(this.position); } } private static final Logger logger = LoggerFactory.getLogger(Parser.class); private final TokenBuffer tokenBuffer; private final String inputLine; public Parser(final List tokens, final String inputLine) { this.tokenBuffer = new TokenBuffer(tokens, inputLine); this.inputLine = inputLine; } public ASTNode parse() { BinaryOperator.clearIdentifierCounter(); UnaryOperator.clearIdentifierCounter(); Operand.clearIdentifierCounter(); return or(); } private void accept(final Token.TokenType tokenType) { if (this.tokenBuffer.peek().getType() == tokenType) { logger.debug("ACCEPT: accepted {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); this.tokenBuffer.nextToken(); } else { throwSyntaxError(tokenType, this.tokenBuffer.peek(), this.inputLine); } } // AND -> EQ AND' private ASTNode and() { ASTNode node = null; logger.debug("AND: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG || this.tokenBuffer.peek().getType() == Token.TokenType.LITERAL || this.tokenBuffer.peek().getType() == Token.TokenType.REGEX || this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_OPEN) { node = eq(); final ASTNode rightResult = andPrime(); if (rightResult != null) { node = new AndOperator(node, rightResult); } } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } // AND' -> & EQ AND' // AND' -> '' private ASTNode andPrime() { ASTNode node; logger.debug("AND_PRIME: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.AND) { logger.debug("AND_PRIME: try accepting: {}", Token.TokenType.AND); accept(Token.TokenType.AND); node = eq(); final ASTNode rightResult = andPrime(); if (rightResult != null) { node = new AndOperator(node, rightResult); } } else { // epsilon transition logger.debug("AND_PRIME: taking epsilon"); return null; } return node; } // EQ -> VALUE EQ' private ASTNode eq() { ASTNode node = null; logger.debug("EQ: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG || this.tokenBuffer.peek().getType() == Token.TokenType.LITERAL || this.tokenBuffer.peek().getType() == Token.TokenType.REGEX || this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_OPEN) { node = value(); if (this.tokenBuffer.peek().getType() == Token.TokenType.EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, false); } } else if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG_EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, true); } } } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } // EQ' -> = VALUE EQ' // EQ' -> != VALUE EQ' // EQ' -> '' private ASTNode eqPrime() // NOSONAR { ASTNode node; logger.debug("EQ_PRIME: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.EQUAL) { logger.debug("EQ_PRIME: try accepting: {}", Token.TokenType.EQUAL); accept(Token.TokenType.EQUAL); node = value(); if (this.tokenBuffer.peek().getType() == Token.TokenType.EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, false); } } else if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG_EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, true); } } } else if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG_EQUAL) { logger.debug("EQ_PRIME: try accepting: {}", Token.TokenType.BANG_EQUAL); accept(Token.TokenType.BANG_EQUAL); node = value(); if (this.tokenBuffer.peek().getType() == Token.TokenType.EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, false); } } else if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG_EQUAL) { final ASTNode rightResult = eqPrime(); if (rightResult != null) { node = new EqualsOperator(node, rightResult, true); } } } else { // epsilon transition logger.error("EQ_PRIME: taking epsilon"); return null; } return node; } // OR -> XOR OR' private ASTNode or() { ASTNode node = null; logger.debug("OR: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG || this.tokenBuffer.peek().getType() == Token.TokenType.LITERAL || this.tokenBuffer.peek().getType() == Token.TokenType.REGEX || this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_OPEN) { node = xor(); final ASTNode rightResult = orPrime(); if (rightResult != null) { node = new OrOperator(node, rightResult); } } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } // OR' -> | XOR OR' // OR' -> '' private ASTNode orPrime() { ASTNode node = null; logger.debug("OR_PRIME: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.OR) { logger.debug("OR_PRIME: try accepting: {}", Token.TokenType.OR); accept(Token.TokenType.OR); node = xor(); final ASTNode rightResult = orPrime(); if (rightResult != null) { node = new OrOperator(node, rightResult); } } else if (this.tokenBuffer.peek().getType() == Token.TokenType.EOF) { // epsilon transition logger.debug("OR_PRIME: taking epsilon"); return null; } else if (this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_CLOSE) { // epsilon transition logger.debug("OR_PRIME: taking epsilon due to FOLLOW )"); return null; } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } private void throwSyntaxError(final Token.TokenType expectedTokenType, final Token currentToken, final String inputLine) { final String arrow = "~".repeat(Math.max(0, currentToken.getIndexInLine())) + "^"; if (expectedTokenType == null) { throw new CoreException("syntax error: unexpected token {}({})\n{}\n{}", currentToken.getType(), currentToken.getLexeme(), inputLine, arrow); } throw new CoreException("syntax error: expected {}, but saw {}({})\n{}\n{}", expectedTokenType, currentToken.getType(), currentToken.getLexeme(), inputLine, arrow); } // VALUE -> ( OR ) // VALUE -> ! VALUE // VALUE -> literal // VALUE -> /regex/ private ASTNode value() { final String valueAcceptMessage = "VALUE: try accepting: {}"; ASTNode node = null; logger.debug("VALUE: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_OPEN) { logger.debug(valueAcceptMessage, Token.TokenType.PAREN_OPEN); accept(Token.TokenType.PAREN_OPEN); node = or(); logger.debug(valueAcceptMessage, Token.TokenType.PAREN_CLOSE); accept(Token.TokenType.PAREN_CLOSE); } else if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG) { logger.debug(valueAcceptMessage, Token.TokenType.BANG); // accept the BANG first, and then parse the remaining token buffer accept(Token.TokenType.BANG); node = new BangOperator(value()); } else if (this.tokenBuffer.peek().getType() == Token.TokenType.LITERAL) { logger.debug(valueAcceptMessage, Token.TokenType.LITERAL); // Create the AST node first, since accepting will advance the token buffer node = new LiteralOperand(this.tokenBuffer.peek()); accept(Token.TokenType.LITERAL); } else if (this.tokenBuffer.peek().getType() == Token.TokenType.REGEX) { logger.debug(valueAcceptMessage, Token.TokenType.REGEX); // Create the AST node first, since accepting will advance the token buffer node = new RegexOperand(this.tokenBuffer.peek()); accept(Token.TokenType.REGEX); } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } // XOR -> AND XOR' private ASTNode xor() { ASTNode node = null; logger.debug("XOR: peek: {}({})", this.tokenBuffer.peek().getType(), this.tokenBuffer.peek().getLexeme()); if (this.tokenBuffer.peek().getType() == Token.TokenType.BANG || this.tokenBuffer.peek().getType() == Token.TokenType.LITERAL || this.tokenBuffer.peek().getType() == Token.TokenType.REGEX || this.tokenBuffer.peek().getType() == Token.TokenType.PAREN_OPEN) { node = and(); final ASTNode rightResult = xorPrime(); if (rightResult != null) { node = new XorOperator(node, rightResult); } } else { throwSyntaxError(null, this.tokenBuffer.peek(), this.inputLine); } return node; } // XOR' -> | AND XOR' // XOR' -> '' private ASTNode xorPrime() { ASTNode node; if (this.tokenBuffer.peek().getType() == Token.TokenType.XOR) { logger.debug("XOR_PRIME: try accepting: {}", Token.TokenType.XOR); accept(Token.TokenType.XOR); node = and(); final ASTNode rightResult = xorPrime(); if (rightResult != null) { node = new XorOperator(node, rightResult); } } else { // epsilon transition logger.debug("XOR_PRIME: taking epsilon"); return null; } return node; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/SemanticChecker.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.ASTNode; import org.openstreetmap.atlas.tags.filters.matcher.parsing.tree.EqualsOperator; /** * Semantic checker for a {@link TaggableMatcher} abstract syntax tree (AST). This checker will make * sure the supplied matcher does not contain invalid "=" or "!=" semantics. For e.g. "foo = (bar != * baz)" is valid syntactically per the {@link TaggableMatcher} expression grammar, but not * semantically since it makes no sense for the purposes of tag matching. We will need to catch that * with this checker. Basically, we can walk the AST, and if the left or right subtree of a "="/"!=" * operator contains another "="/"!=" operator, then we fail. * * @author lcram */ public class SemanticChecker { public void check(final ASTNode root) { if (root == null) { return; } if (root instanceof EqualsOperator && this.subtreeContainsEquals(root)) { throw new CoreException("semantic error: invalid nested equality operators"); } check(root.getLeftChild()); check(root.getRightChild()); check(root.getCenterChild()); } private boolean subtreeContainsEquals(final ASTNode root) { if (root == null) { return false; } final ASTNode leftRoot = root.getLeftChild(); final ASTNode rightRoot = root.getRightChild(); final ASTNode centerRoot = root.getCenterChild(); if (leftRoot instanceof EqualsOperator || rightRoot instanceof EqualsOperator || centerRoot instanceof EqualsOperator) { return true; } return subtreeContainsEquals(leftRoot) || subtreeContainsEquals(rightRoot) || subtreeContainsEquals(centerRoot); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/Token.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing; import java.io.Serializable; import java.util.Objects; /** * @author lcram */ public class Token implements Serializable { /** * @author lcram */ public enum TokenType { AND("&"), BANG("!"), BANG_EQUAL("!="), DOUBLE_QUOTE("\""), ESCAPE("\\"), EOF(null), EQUAL("="), LITERAL(null), OR("|"), PAREN_OPEN("("), PAREN_CLOSE(")"), REGEX("/"), SINGLE_QUOTE("'"), WHITESPACE(null), XOR("^"); private final String literalValue; TokenType(final String literalValue) { this.literalValue = literalValue; } public String getLiteralValue() { return this.literalValue; } } private static final long serialVersionUID = -8498419139066512731L; private final TokenType type; private final String lexeme; private final int indexInLine; public Token(final TokenType type, final String lexeme, final int indexInLine) { this.lexeme = lexeme; if (type == TokenType.DOUBLE_QUOTE || type == TokenType.SINGLE_QUOTE) { /* * Override DOUBLE_QUOTE/SINGLE_QUOTE with regular LITERAL, since after lexing no other * component cares about this distinction. Using LITERAL everywhere will simplify * following code. */ this.type = TokenType.LITERAL; /* * We need to add 2 back to the lexeme length to account for the "/' characters we * removed. */ final int addBack = 2; this.indexInLine = indexInLine - (lexeme != null ? lexeme.length() + addBack : 0); } else { this.type = type; this.indexInLine = indexInLine - (lexeme != null ? lexeme.length() : 0); } } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || getClass() != other.getClass()) { return false; } final Token token = (Token) other; return this.type == token.type && Objects.equals(this.getLexeme(), token.getLexeme()) && this.indexInLine == token.indexInLine; } public int getIndexInLine() { return this.indexInLine; } public String getLexeme() { return this.lexeme; } public TokenType getType() { return this.type; } @Override public int hashCode() { return Objects.hash(this.type, this.getLexeme(), this.indexInLine); } @Override public String toString() { return "(" + this.type + ", " + this.lexeme + ")"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/ASTNode.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.io.Serializable; import java.util.List; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; /** * A generic abstract syntax tree (AST) node. Any node must be able to print itself and its subtree, * as well as have some kind of name for debug purposes. * * @author lcram */ public abstract class ASTNode implements Serializable { private static final long serialVersionUID = -2085619833468362629L; /** * Construct a debug printout of the entire tree. For a prettified version of this tree, try * {@link ASTNode#getPrettyPrintText()}. * * @return the debug printout */ public abstract String debugPrintTree(); /** * Get the center child of this {@link ASTNode}. {@link UnaryOperator}s are the only type of * node that have center children. * * @return the center child {@link ASTNode} */ public abstract ASTNode getCenterChild(); public abstract int getIdentifier(); /** * Get the left child of this {@link ASTNode}. {@link BinaryOperator}s are the only type of node * that have left children. * * @return the left child {@link ASTNode} */ public abstract ASTNode getLeftChild(); public abstract String getName(); /** * Get the representation of this node for the purposes of {@link TaggableMatcher}'s pretty tree * print functionality. * * @return the pretty tree */ public abstract String getPrettyPrintText(); /** * Get the right child of this {@link ASTNode}. {@link BinaryOperator}s are the only type of * node that have right children. * * @return the right child {@link ASTNode} */ public abstract ASTNode getRightChild(); public abstract boolean match(List keys, List values); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/AndOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; /** * @author lcram */ public class AndOperator extends BinaryOperator { public AndOperator(final ASTNode left, final ASTNode right) { super(left, right); } @Override public String getName() { return "AND_" + getIdentifier(); } @Override public String getPrettyPrintText() { return "&"; } @Override public boolean match(final List keys, final List values) { return getLeftChild().match(keys, values) && getRightChild().match(keys, values); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/BangOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; /** * @author lcram */ public class BangOperator extends UnaryOperator { public BangOperator(final ASTNode child) { super(child); } @Override public String getName() { return "BANG_" + getIdentifier(); } @Override public String getPrettyPrintText() { return "!"; } @Override public boolean match(final List keys, final List values) { return !getCenterChild().match(keys, values); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/BinaryOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; /** * @author lcram */ public abstract class BinaryOperator extends ASTNode { private static int counter = 0; private static final long serialVersionUID = -8367179322128266687L; private final ASTNode left; private final ASTNode right; private final int identifier; public static void clearIdentifierCounter() { counter = 0; } protected BinaryOperator(final ASTNode left, final ASTNode right) { this.left = left; this.right = right; this.identifier = counter++; } @Override public String debugPrintTree() { final StringBuilder builder = new StringBuilder(); builder.append(this.getName() + "\n"); builder.append(this.getName() + " left: " + this.left.getName() + "\n"); builder.append(this.getName() + " right: " + this.right.getName() + "\n"); builder.append(this.left.debugPrintTree()); builder.append(this.right.debugPrintTree()); return builder.toString(); } @Override public ASTNode getCenterChild() { return null; } @Override public int getIdentifier() { return this.identifier; } @Override public ASTNode getLeftChild() { return this.left; } @Override public ASTNode getRightChild() { return this.right; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/EqualsOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.Collections; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; /** * @author lcram */ public class EqualsOperator extends BinaryOperator { private static final long serialVersionUID = 4853555543879394794L; private final boolean bang; public EqualsOperator(final ASTNode left, final ASTNode right, final boolean bang) { super(left, right); this.bang = bang; } @Override public String getName() { if (this.bang) { return "BANGEQ_" + getIdentifier(); } else { return "EQ_" + getIdentifier(); } } @Override public String getPrettyPrintText() { if (this.bang) { return "!="; } else { return "="; } } @Override public boolean match(final List keys, final List values) { if (keys.size() != values.size()) { throw new CoreException("`keys' and `values' sizes did not match, {} vs {}", keys.size(), values.size()); } for (int i = 0; i < keys.size(); i++) { final boolean leftSide = getLeftChild().match(Collections.singletonList(keys.get(i)), null); boolean rightSide = getRightChild().match(null, Collections.singletonList(values.get(i))); /* * For BANG_EQUALS, flip the boolean value of the right side to mimic the logic of `!='. */ if (this.bang) { rightSide = !rightSide; } if (leftSide && rightSide) { return true; } } return false; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/LiteralOperand.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; import java.util.Objects; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Token; /** * @author lcram */ public class LiteralOperand extends Operand { public LiteralOperand(final Token token) { super(token); } @Override public String getName() { return getToken().getLexeme() + "_" + getIdentifier(); } @Override public String getPrettyPrintText() { return getToken().getLexeme(); } @Override public boolean match(final List keys, final List values) { if (keys == null && values == null) { throw new CoreException("keys and values were null"); } return Objects.requireNonNullElse(keys, values).contains(getToken().getLexeme()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/Operand.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Token; /** * @author lcram */ public abstract class Operand extends ASTNode { private static int counter = 0; private static final long serialVersionUID = 4045177960157269200L; private final Token token; private final int identifier; public static void clearIdentifierCounter() { counter = 0; } protected Operand(final Token token) { this.token = token; this.identifier = counter++; } @Override public String debugPrintTree() { final StringBuilder builder = new StringBuilder(); builder.append(this.getName() + "\n"); return builder.toString(); } @Override public ASTNode getCenterChild() { return null; } @Override public int getIdentifier() { return this.identifier; } @Override public ASTNode getLeftChild() { return null; } @Override public ASTNode getRightChild() { return null; } public Token getToken() { return this.token; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/OrOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; /** * @author lcram */ public class OrOperator extends BinaryOperator { public OrOperator(final ASTNode left, final ASTNode right) { super(left, right); } @Override public String getName() { return "OR_" + getIdentifier(); } @Override public String getPrettyPrintText() { return "|"; } @Override public boolean match(final List keys, final List values) { return getLeftChild().match(keys, values) || getRightChild().match(keys, values); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/RegexOperand.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; import java.util.Objects; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.matcher.parsing.Token; /** * @author lcram */ public class RegexOperand extends Operand { public RegexOperand(final Token token) { super(token); } @Override public String getName() { return getToken().getLexeme() + "_" + getIdentifier(); } @Override public String getPrettyPrintText() { return "/" + getToken().getLexeme() + "/"; } @Override public boolean match(final List keys, final List values) { if (keys == null && values == null) { throw new CoreException("keys and values were null"); } return Objects.requireNonNullElse(keys, values).stream() .anyMatch(string -> string.matches(getToken().getLexeme())); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/TreePrinter.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Inspired by MightyPork's algorithm here: https://stackoverflow.com/a/29704252 * * @author lcram */ public final class TreePrinter { public static long lengthOfLongestLineForTree(final ASTNode root) { return Arrays.stream(print(root).split("\n")).mapToLong(String::length).max() .orElseThrow(() -> new CoreException("Failed to caluclate longest line length!")); } /** * Get a {@link TaggableMatcher} tree as a string. * * @param root * the root {@link ASTNode} of the tree * @return the string tree */ public static String print(final ASTNode root) // NOSONAR { final StringBuilder treeString = new StringBuilder(); final Tuple>, Integer> tuple = discoverAllTreeNodes(root); final List> lines = tuple.getFirst(); final int widestNodeWidth = tuple.getSecond(); /* * How are we calculating this? The last line returned by the breadth first search will be * the longest, since it will contain a lot of nulls for every dead branch in the tree. So * we use the last line as a baseline for line length. Then, we multiply by the widest width * of any node plus 4 (4 gives some nice padding for readability). As the loop walks down * the tree, this value will be continually halved, since each level there are approx. twice * as many tree pieces. */ final int widthPadding = 4; int lengthOfTreePiece = lines.get(lines.size() - 1).size() * (widestNodeWidth + widthPadding); boolean firstIteration = true; for (final List line : lines) { final int nodeLeftRightPadding = (int) Math.floor(lengthOfTreePiece / 2f) - 1; /* * This section prints the Unicode box-drawing characters above each line of actual * elements. It does not run on the first iteration of the loop, since there is no line * containing Unicode box-drawing characters to print above the root node. */ if (!firstIteration) { for (int lineElementIndex = 0; lineElementIndex < line.size(); lineElementIndex++) { /* * Decide which Unicode box-drawing character to print below the nodes *ABOVE* * the current line. Only print for odd elements within the line. Since the tree * is binary, there is only one node "between" each of the nodes in the current * line. */ char boxCharacter = ' '; if (isOdd(lineElementIndex)) { if (line.get(lineElementIndex - 1) != null) { boxCharacter = (line.get(lineElementIndex) != null) ? '┴' : '┘'; } else if (line.get(lineElementIndex) != null) { boxCharacter = '└'; } } treeString.append(boxCharacter); /* * Print whitespace above null line elements, since nothing is there. */ if (line.get(lineElementIndex) == null) { treeString.append(" ".repeat(Math.max(0, lengthOfTreePiece - 1))); } /* * Here we decide which box-drawing character to print above the nodes *BELOW* * the current line. */ else { treeString.append((isEven(lineElementIndex) ? " " : "─") .repeat(Math.max(0, nodeLeftRightPadding))); treeString.append(isEven(lineElementIndex) ? "┌" : "┐"); treeString.append((isEven(lineElementIndex) ? "─" : " ") .repeat(Math.max(0, nodeLeftRightPadding))); } } treeString.append("\n"); } /* * This section prints the actual line of elements. */ for (final String element : line) { String element2 = element; if (element2 == null) { element2 = ""; } final double padding = (lengthOfTreePiece / 2f) - (element2.length() / 2f); final int paddingLeft = (int) Math.ceil(padding); final int paddingRight = (int) Math.floor(padding); treeString.append(" ".repeat(Math.max(0, paddingLeft))); treeString.append(element2); treeString.append(" ".repeat(Math.max(0, paddingRight))); } treeString.append("\n"); lengthOfTreePiece /= 2; firstIteration = false; } return treeString.toString(); } private static Tuple>, Integer> discoverAllTreeNodes(final ASTNode root) // NOSONAR { final List> lines = new ArrayList<>(); List nodesThisLevel = new ArrayList<>(); List nodesNextLevel = new ArrayList<>(); nodesThisLevel.add(root); int numberOfNodesRemaining = 1; int widestNodeWidth = 0; while (numberOfNodesRemaining != 0) { final List line = new ArrayList<>(); numberOfNodesRemaining = 0; for (final ASTNode node : nodesThisLevel) { if (node == null) { line.add(null); nodesNextLevel.add(null); nodesNextLevel.add(null); } else { final String nodeText = node.getPrettyPrintText(); line.add(nodeText); if (nodeText.length() > widestNodeWidth) { widestNodeWidth = nodeText.length(); } if (node.getCenterChild() != null) { nodesNextLevel.add(node.getCenterChild()); } else { nodesNextLevel.add(node.getLeftChild()); } nodesNextLevel.add(node.getRightChild()); if (node.getCenterChild() != null) { numberOfNodesRemaining++; } if (node.getLeftChild() != null) { numberOfNodesRemaining++; } if (node.getRightChild() != null) { numberOfNodesRemaining++; } } } if (isOdd(widestNodeWidth)) { widestNodeWidth++; } lines.add(line); final List tmp = nodesThisLevel; nodesThisLevel = nodesNextLevel; nodesNextLevel = tmp; nodesNextLevel.clear(); } return new Tuple<>(lines, widestNodeWidth); } private static boolean isEven(final int integer) { return integer % 2 == 0; } private static boolean isOdd(final int integer) { return integer % 2 != 0; } private TreePrinter() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/UnaryOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; /** * @author lcram */ public abstract class UnaryOperator extends ASTNode { private static int counter = 0; private static final long serialVersionUID = -6551792893893585221L; private final ASTNode child; private final int identifier; public static void clearIdentifierCounter() { counter = 0; } protected UnaryOperator(final ASTNode child) { this.child = child; this.identifier = counter++; } @Override public String debugPrintTree() { final StringBuilder builder = new StringBuilder(); builder.append(this.getName() + "\n"); builder.append(this.getName() + " child: " + this.child.getName() + "\n"); builder.append(this.child.debugPrintTree()); return builder.toString(); } @Override public ASTNode getCenterChild() { return this.child; } @Override public int getIdentifier() { return this.identifier; } @Override public ASTNode getLeftChild() { return null; } @Override public ASTNode getRightChild() { return null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/filters/matcher/parsing/tree/XorOperator.java ================================================ package org.openstreetmap.atlas.tags.filters.matcher.parsing.tree; import java.util.List; /** * @author lcram */ public class XorOperator extends BinaryOperator { public XorOperator(final ASTNode left, final ASTNode right) { super(left, right); } @Override public String getName() { return "XOR_" + getIdentifier(); } @Override public String getPrettyPrintText() { return "^"; } @Override public boolean match(final List keys, final List values) { final boolean left = getLeftChild().match(keys, values); final boolean right = getRightChild().match(keys, values); return (left || right) && !(left && right); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/AlternativeNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM alt_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/alt_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:alt_name") public interface AlternativeNameTag { @TagKey(KeyType.LOCALIZED) String KEY = "alt_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/BridgeNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM bridge:name tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/search?q=bridge%3Aname", osm = "http://wiki.openstreetmap.org/wiki/Key:bridge:name") public interface BridgeNameTag { @TagKey(KeyType.LOCALIZED) String KEY = "bridge:name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/BulkNameFinder.java ================================================ package org.openstreetmap.atlas.tags.names; import java.io.Serializable; import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.stream.Collectors; import org.openstreetmap.atlas.locale.IsoLanguage; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.Taggable.TagSearchOption; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import com.google.common.collect.ImmutableMap; /** * When we need results across multiple languages at the same time this hides a lot of the * boilerplate code that would be required if we used the NameFinder class directly. * * @author ihillberg * @author cstaylor */ public class BulkNameFinder implements Serializable { /** * BulkFind's findIn method will return an implementation of this interface. * * @author cstaylor */ public interface BulkFindResults { /** * Returns an optional map of name/value pairs for the requested name tags in the supplied * language. If the language was not requested during the initial search, the Optional will * be empty. * * @param language * we want the tags for this language only * @return a map of the name/value pairs for the name tags */ Optional, String>> allValuesFor(Optional language); /** * This method will create a map of basekeyname:language to value. This is good for bulk * editing a set all at once. Note: The keys have been transformed into strings * * @return the flattened list of name/value pairs */ Map flatten(); /** * Returns an iterable containing all of the languages encountered when creating these * results * * @return an iterable list of IsoLanguages */ Iterable languagesFound(); /** * Returns a single value for the language and name tag. If the request didn't find a tag by * the type, Optional.empty() will be returned. * * @param language * we want the tag value for this language * @param tagClass * the name tag we want * @return an optional containing the value if it exists or empty if it doesn't */ Optional valueFor(Optional language, Class tagClass); } /** * Internal implementation of the BulkFindResults interface. * * @author cstaylor */ private static final class DefaultBulkFindResults implements BulkFindResults { private final Map, String>> localizedResults = new HashMap<>(); private final Map, String> nonLocalizedResults = new LinkedHashMap<>(); private Map flattenedMap = new HashMap<>(); @Override public Optional, String>> allValuesFor(final Optional language) { return Optional .ofNullable(language.isPresent() ? this.localizedResults.get(language.get()) : this.nonLocalizedResults); } @Override public Map flatten() { return this.flattenedMap; } @Override public Iterable languagesFound() { return this.localizedResults.keySet(); } @Override public Optional valueFor(final Optional language, final Class tagClass) { final Map, String> results = language.isPresent() ? this.localizedResults.get(language.get()) : this.nonLocalizedResults; return Optional.ofNullable(results == null ? null : results.get(tagClass)); } private void completed() { final Map temporaryMap = new HashMap<>(); for (final Entry, String>> entry : this.localizedResults .entrySet()) { final Optional currentLanguage = Optional.of(entry.getKey()); for (final Entry, String> itemEntry : entry.getValue().entrySet()) { Validators.localizeKeyName(itemEntry.getKey(), currentLanguage) .ifPresent(localizedKeyName -> { temporaryMap.put(localizedKeyName, itemEntry.getValue()); }); } } for (final Entry, String> entry : this.nonLocalizedResults.entrySet()) { Validators.localizeKeyName(entry.getKey(), Optional.empty()) .ifPresent(nonLocalizedKeyName -> { temporaryMap.put(nonLocalizedKeyName, entry.getValue()); }); } this.flattenedMap = new ImmutableMap.Builder().putAll(temporaryMap) .build(); } private void put(final IsoLanguage language, final Class tag, final String value) { Map, String> mapping = this.localizedResults.get(language); if (mapping == null) { mapping = new HashMap<>(); this.localizedResults.put(language, mapping); } mapping.put(tag, value); } private void put(final IsoLanguage language, final Map, String> results) { this.localizedResults.put(language, results); } private void put(final Map, String> results) { this.nonLocalizedResults.putAll(results); } } private static final long serialVersionUID = -7709121230794406053L; private final LinkedHashSet requestedLanguages = new LinkedHashSet<>(); private final NameFinder finder = new NameFinder(); private boolean forceLocalized; /** * Convenience method that configures the underlying NameFinder for its standard set of tags to * search * * @return fluent interface returns this */ public static BulkNameFinder createStandardSet() { final BulkNameFinder returnValue = new BulkNameFinder(); returnValue.finder.withTags(NameFinder.STANDARD_TAGS); return returnValue; } /** * Convenience method that adds all languages core is configured to use * * @return fluent interface returns this */ public BulkNameFinder allLanguages() { this.requestedLanguages.addAll(IsoLanguage.allLanguageCodes().stream() .map(languageCode -> IsoLanguage.forLanguageCode(languageCode).get()) .collect(Collectors.toSet())); return this; } /** * Based on the current settings of BulkNameFinder, search taggable for the localized values of * the keys in question and return a BulkFindResults with the resulting tags and values. * * @param taggable * what we're searching for * @return the results of the search */ public BulkFindResults findIn(final Taggable taggable) { /* * We don't want the name finder to outsmart us and pull out the non-localized value if the * localized one doesn't exist */ if (this.forceLocalized) { this.finder.forceLocalized(); } else { this.finder.localizedOnly(); } final DefaultBulkFindResults results = new DefaultBulkFindResults(); for (final IsoLanguage language : this.requestedLanguages) { results.put(language, this.finder.inLanguage(language).all(taggable)); } /* * And the non-localized values */ results.put(this.finder.inLanguage(null).all(taggable)); /* * When we call completed, DefaultBulkFindResults will run through all of the data and build * an immutable flattened map of the name/value pairs so calls to flatten will return the * same map object. This is just an optimization so we don't recalculate the same results on * multiple calls to flatten on the same BulkFindResults object */ results.completed(); return results; } /** * Based on the current tag class settings of BulkNameFinder, search taggable for the localized * values of the keys in question using the language of the values present and return a * BulkFindResults with the resulting tags and values. * * @param taggable * what we're searching for * @param searchOptions * optional list of flags that alter the behavior of the underlying tag value search * @return the results of the search */ public BulkFindResults findInWithMyLanguages(final Taggable taggable, final TagSearchOption... searchOptions) { final EnumSet searchOptionSet = searchOptions.length > 0 ? EnumSet.copyOf(Arrays.asList(searchOptions)) : EnumSet.noneOf(TagSearchOption.class); final TagSearchOption localizationOption = searchOptionSet.contains( TagSearchOption.FORCE_ALL_LOCALIZED_ONLY) ? TagSearchOption.FORCE_ALL_LOCALIZED_ONLY : TagSearchOption.LOCALIZED_ONLY; /* * We don't want the name finder to outsmart us and pull out the non-localized value if the * localized one doesn't exist */ final DefaultBulkFindResults results = new DefaultBulkFindResults(); for (final Class currentTag : this.finder.getTagNames()) { if (Validators.hasLocalizedTagKey(currentTag) || localizationOption == TagSearchOption.FORCE_ALL_LOCALIZED_ONLY) { taggable.languagesFor(currentTag, searchOptions).ifPresent(languages -> { for (final IsoLanguage language : languages) { taggable.getTag(currentTag, Optional.of(language), localizationOption) .ifPresent(value -> { results.put(language, currentTag, value); }); } }); } } /* * And the non-localized values */ results.put(this.finder.inLanguage(null).all(taggable)); /* * When we call completed, DefaultBulkFindResults will run through all of the data and build * an immutable flattened map of the name/value pairs so calls to flatten will return the * same map object. This is just an optimization so we don't recalculate the same results on * multiple calls to flatten on the same BulkFindResults object */ results.completed(); return results; } public BulkNameFinder forceLocalized() { this.forceLocalized = true; return this; } /** * Which language of localized tags are we looking for in the name finder? * * @param languages * we'll want the values of localizable tags in these languages * @return fluent interface returns this */ public BulkNameFinder withLanguage(final IsoLanguage... languages) { this.requestedLanguages.addAll(Arrays.asList(languages)); return this; } /** * Which tags are we looking for in the underlying name finder? * * @param tags * the list of tags we'd like to search entities for * @return fluent interface returns this */ public BulkNameFinder withTags(final Class... tags) { this.finder.withTags(tags); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/HistoricallyKnownAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM old_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/old_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface HistoricallyKnownAsTag { @TagKey(KeyType.LOCALIZED) String KEY = "old_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/HistoricallyReferencedAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM old_ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/old_ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ref") public interface HistoricallyReferencedAsTag { @TagKey String KEY = "old_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/InternationallyKnownAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM int_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/int_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface InternationallyKnownAsTag { @TagKey String KEY = "int_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/InternationallyReferencedAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM int_ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/int_ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:int_ref") public interface InternationallyReferencedAsTag { @TagKey String KEY = "int_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/LocallyKnownAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM loc_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/loc_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface LocallyKnownAsTag { @TagKey String KEY = "loc_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/LocallyReferencedAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM loc_ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/loc_ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ref") public interface LocallyReferencedAsTag { @TagKey String KEY = "loc_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/Name1Tag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM name_1 tag, not official per OSM tagging wiki, but is very prevalent especially in the USA * * @author brian_l_davis */ @Tag(value = Tag.Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/name_1#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name_1") public interface Name1Tag { @TagKey String KEY = "name_1"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NameFinder.java ================================================ package org.openstreetmap.atlas.tags.names; import static org.openstreetmap.atlas.geography.atlas.items.Relation.RELATION_ID_COMPARATOR; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.locale.IsoLanguage; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.Taggable.TagSearchOption; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.utilities.collections.EnhancedCollectors; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; /** * Responsible for finding names in AtlasEntities * * @author cstaylor * @author Sid */ public class NameFinder implements Serializable { /** * Standard set of name tags excluding the reference tags */ public static final ImmutableList> STANDARD_TAGS_NON_REFERENCE = new ImmutableList.Builder>() .add(NameTag.class, InternationallyKnownAsTag.class, NationallyKnownAsTag.class, RegionallyKnownAsTag.class, LocallyKnownAsTag.class, HistoricallyKnownAsTag.class, AlternativeNameTag.class, ShortNameTag.class, OfficialNameTag.class) .build(); /** * Standard set of reference tags */ public static final ImmutableList> STANDARD_TAGS_REFERENCE = new ImmutableList.Builder>() .add(ReferenceTag.class, InternationallyReferencedAsTag.class, NationallyReferencedAsTag.class, RegionallyReferencedAsTag.class, LocallyReferencedAsTag.class, HistoricallyReferencedAsTag.class) .build(); /** * Standard set of name tags in order of priority per mcuthbert's NameTag class */ public static final ImmutableList> STANDARD_TAGS = new ImmutableList.Builder>() .addAll(STANDARD_TAGS_NON_REFERENCE).addAll(STANDARD_TAGS_REFERENCE).build(); public static final ImmutableList STANDARD_TAG_KEYS; private static final long serialVersionUID = -7268140468931884651L; static { STANDARD_TAG_KEYS = STANDARD_TAGS.stream().map(Validators::findTagNameIn) .collect(EnhancedCollectors.toImmutableList()); } private transient IsoLanguage language; private final LinkedHashSet> priorityOrderOfTagNames; private TagSearchOption searchOption = TagSearchOption.DEFAULT; /** * Returns a new NameFinder initialized with the following tags in priority order: *

    *
  1. NameTag
    2. *
  2. InternationallyKnownAsTag
    3. *
  3. NationallyKnownAsTag
    4. *
  4. RegionallyKnownAsTag
    5. *
  5. LocallyKnownAsTag
    6. *
  6. HistoricallyKnownAsTag
    7. *
  7. AlternativeNameTag
    8. *
  8. ShortNameTag
    9. *
  9. OfficialNameTag
    10. *
  10. ReferenceTag
    11. *
  11. InternationallyReferencedAsTag
    12. *
  12. NationallyReferencedAsTag
    13. *
  13. RegionallyReferencedAsTag
    14. *
  14. LocallyReferencedAsTag
    15. *
  15. HistoricallyReferencedAsTag
    16. *
*

* Note: This order was originally written by mcuthbert for his NameTag class. * * @param language * the language we should use for localizable tags when finding their values * @return the initialized NameFinder instance */ public static NameFinder createStandardSet(final IsoLanguage language) { return new NameFinder().withTags(STANDARD_TAGS).inLanguage(language); } private static List children(final AtlasEntity entity) { final List taggables = new ArrayList<>(); taggables.add(entity); final List relations = new ArrayList<>(entity.relations()); relations.sort(RELATION_ID_COMPARATOR); Iterables.addAll(taggables, relations); return taggables; } public NameFinder() { this.priorityOrderOfTagNames = new LinkedHashSet<>(); this.language = null; } public Map, String> all(final Taggable taggable) { final Map, String> returnValue = new HashMap<>(); for (final Class tagClass : this.priorityOrderOfTagNames) { taggable.getTag(tagClass, Optional.ofNullable(this.language), this.searchOption) .ifPresent(tagValue -> { returnValue.put(tagClass, tagValue); }); } return returnValue; } public Optional best(final AtlasEntity entity) { return children(entity).stream().map(this::best).filter(Optional::isPresent) .map(Optional::get).findFirst(); } public Optional best(final Taggable taggable) { return this.priorityOrderOfTagNames.stream() .map(tagClass -> taggable.getTag(tagClass, Optional.ofNullable(this.language), this.searchOption)) .filter(Optional::isPresent).map(Optional::get).findFirst(); } public NameFinder forceLocalized() { this.searchOption = TagSearchOption.FORCE_ALL_LOCALIZED_ONLY; return this; } public ImmutableCollection> getTagNames() { return ImmutableList.> builder().addAll(this.priorityOrderOfTagNames).build(); } public NameFinder inLanguage(final IsoLanguage language) { this.language = language; return this; } public NameFinder localizedOnly() { this.searchOption = TagSearchOption.LOCALIZED_ONLY; return this; } public NameFinder withTags(final Class... tagClasses) { for (final Class tagClass : tagClasses) { this.priorityOrderOfTagNames.add(tagClass); } return this; } public NameFinder withTags(final Iterable> tagClasses) { for (final Class tagClass : tagClasses) { this.priorityOrderOfTagNames.add(tagClass); } return this; } private void readObject(final ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); final String iso2 = (String) stream.readObject(); if (iso2 != null) { this.language = IsoLanguage.forLanguageCode(iso2).orElse(null); } } private void writeObject(final ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); final String iso2 = this.language == null ? null : this.language.getLanguageCode(); stream.writeObject(iso2); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NameLeftTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM name:left tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/name%3Aleft#values", osm = "http://wiki.openstreetmap.org/wiki/Names#Left_and_right_names") public interface NameLeftTag { @TagKey(KeyType.LOCALIZED) String KEY = "name:left"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NameRightTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM name:right tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/name%3Aright#values", osm = "http://wiki.openstreetmap.org/wiki/Names#Left_and_right_names") public interface NameRightTag { @TagKey(KeyType.LOCALIZED) String KEY = "name:right"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import java.util.Optional; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; import org.openstreetmap.atlas.tags.annotations.extraction.NonEmptyStringExtractor; /** * OSM name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface NameTag { @TagKey(KeyType.LOCALIZED) String KEY = "name"; static Optional getNameOf(final Taggable taggable) { final Optional tagValue = taggable.getTag(KEY); if (tagValue.isPresent()) { return NonEmptyStringExtractor.validateAndExtract(tagValue.get()); } return Optional.empty(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NationallyKnownAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM nat_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/nat_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface NationallyKnownAsTag { @TagKey(KeyType.LOCALIZED) String KEY = "nat_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NationallyReferencedAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM nat_ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/nat_ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ref#Examples_on_ways") public interface NationallyReferencedAsTag { @TagKey String KEY = "nat_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/NoNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import java.util.Optional; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * OSM No Name Tag. * * @author matthieun */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/noname#values", osm = "http://wiki.openstreetmap.org/wiki/Key:noname") public enum NoNameTag { YES, NO; @TagKey public static final String KEY = "noname"; public static boolean isNoName(final Taggable taggable) { final Optional noName = Validators.from(NoNameTag.class, taggable); return noName.isPresent() && YES == noName.get(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/OfficialNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM official_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/official_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface OfficialNameTag { @TagKey(KeyType.LOCALIZED) String KEY = "official_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/OldReferenceTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM old_ref tag * * @author kkonishi2 */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/keys/old_ref", osm = "http://wiki.openstreetmap.org/wiki/Key:ref") public interface OldReferenceTag { @TagKey String KEY = "old_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/ReferenceTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ref") public interface ReferenceTag { @TagKey String KEY = "ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/RegionallyKnownAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM reg_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/reg_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface RegionallyKnownAsTag { @TagKey String KEY = "reg_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/RegionallyReferencedAsTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM ref_ref tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/reg_ref#values", osm = "http://wiki.openstreetmap.org/wiki/Key:ref") public interface RegionallyReferencedAsTag { @TagKey String KEY = "reg_ref"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/ShortNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM short_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/short_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:name") public interface ShortNameTag { @TagKey(KeyType.LOCALIZED) String KEY = "short_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/SortingNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; /** * OSM sorting_name tag * * @author cstaylor */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "http://taginfo.openstreetmap.org/keys/sorting_name#values", osm = "http://wiki.openstreetmap.org/wiki/Key:sorting_name") public interface SortingNameTag { @TagKey String KEY = "sorting_name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/names/TunnelNameTag.java ================================================ package org.openstreetmap.atlas.tags.names; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.Tag.Validation; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagKey.KeyType; /** * OSM tunnel:name tag * * @author alexhsieh */ @Tag(value = Validation.NON_EMPTY_STRING, taginfo = "https://taginfo.openstreetmap.org/search?q=tunnel%3Aname", osm = "http://wiki.openstreetmap.org/wiki/Key:tunnel") public interface TunnelNameTag { @TagKey(KeyType.LOCALIZED) String KEY = "tunnel:name"; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/OneWayTag.java ================================================ package org.openstreetmap.atlas.tags.oneway; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.TagValueDeprecated; import org.openstreetmap.atlas.tags.annotations.validation.Validators; import org.openstreetmap.atlas.tags.oneway.bicycle.BicycleOneWayTag; import org.openstreetmap.atlas.tags.oneway.bicycle.CyclewayLeftOneWayTag; import org.openstreetmap.atlas.tags.oneway.bicycle.CyclewayOneWayTag; import org.openstreetmap.atlas.tags.oneway.bicycle.CyclewayRightOneWayTag; import org.openstreetmap.atlas.tags.oneway.bicycle.OneWayBicycleTag; import org.openstreetmap.atlas.tags.oneway.motor.OneWayMotorVehicleTag; import org.openstreetmap.atlas.tags.oneway.motor.OneWayMotorcarTag; import org.openstreetmap.atlas.tags.oneway.motor.OneWayVehicleTag; /** * OSM's oneway tag * * @author cstaylor * @author matthieun */ @Tag(taginfo = "http://taginfo.openstreetmap.org/keys/oneway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum OneWayTag { YES, NO, REVERSIBLE, @TagValueDeprecated TRUE, @TagValueDeprecated FALSE, @TagValueDeprecated @TagValueAs("1") ONE, @TagValueDeprecated @TagValueAs(value = "0") ZERO, @TagValueAs("-1") MINUS_1, @TagValueDeprecated REVERSE; @TagKey public static final String KEY = "oneway"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES, TRUE, ONE); // Note here that REVERSIBLE is not reversed. protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1, REVERSE); protected static final Set TWO_WAYS = EnumSet.of(NO, FALSE, ZERO); /** * @param taggable * The taggable * @return True if the feature is oneway forward, OR bicycle oneway forward. */ public static boolean isBicycleOneWayForward(final Taggable taggable) { return isOneWayForward(taggable) || isBicycleTagSpecificallyOneWayForward(taggable); } /** * @param taggable * The taggable * @return True if the feature is oneway reverse, OR bicycle oneway reverse. */ public static boolean isBicycleOneWayReversed(final Taggable taggable) { return isOneWayReversed(taggable) || isBicycleTagSpecificallyOneWayReversed(taggable); } /** * @param taggable * The taggable * @return True if the feature is two way, AND bicycle two way. */ public static boolean isBicycleTwoWay(final Taggable taggable) { return OneWayTag.isTwoWay(taggable) && isBicycleTagSpecificallyTwoWay(taggable); } /** * This is a subset of two way roads, in which the two way status has been tagged with oneway=no * and not just assumed because of the absence of a oneway tag. * * @param taggable * The object to test * @return True if the object is explicitly two way */ public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isMotorVehicleOneWayForward(final Taggable taggable) { return isOneWayForward(taggable) || OneWayMotorcarTag.isOneWayForward(taggable) || OneWayMotorVehicleTag.isOneWayForward(taggable) || OneWayVehicleTag.isOneWayForward(taggable); } public static boolean isMotorVehicleOneWayReversed(final Taggable taggable) { return isOneWayReversed(taggable) || OneWayMotorcarTag.isOneWayReversed(taggable) || OneWayMotorVehicleTag.isOneWayReversed(taggable) || OneWayVehicleTag.isOneWayReversed(taggable); } public static boolean isMotorVehicleTwoWay(final Taggable taggable) { // All the motor related tags need to be two way (including not set) return OneWayTag.isTwoWay(taggable) && OneWayMotorcarTag.isTwoWay(taggable) && OneWayMotorVehicleTag.isTwoWay(taggable) && OneWayVehicleTag.isTwoWay(taggable); } public static boolean isOneWayForward(final OneWayTag tag) { return ONE_WAYS_FORWARD.contains(tag); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final OneWayTag tag) { return ONE_WAYS_REVERSED.contains(tag); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isOneWayReversible(final OneWayTag tag) { return REVERSIBLE == tag; } public static boolean isOneWayReversible(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && REVERSIBLE == oneWay.get(); } public static boolean isTwoWay(final OneWayTag tag) { return TWO_WAYS.contains(tag); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return !oneWay.isPresent() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(OneWayTag.class, taggable); } private static boolean isBicycleTagSpecificallyOneWayForward(final Taggable taggable) { return BicycleOneWayTag.isOneWayForward(taggable) || CyclewayOneWayTag.isOneWayForward(taggable) || OneWayBicycleTag.isOneWayForward(taggable) || CyclewayRightOneWayTag.isOneWayForward(taggable) || CyclewayLeftOneWayTag.isOneWayForward(taggable); } private static boolean isBicycleTagSpecificallyOneWayReversed(final Taggable taggable) { return BicycleOneWayTag.isOneWayReversed(taggable) || CyclewayOneWayTag.isOneWayReversed(taggable) || OneWayBicycleTag.isOneWayReversed(taggable) || CyclewayRightOneWayTag.isOneWayReversed(taggable) || CyclewayLeftOneWayTag.isOneWayReversed(taggable); } private static boolean isBicycleTagSpecificallyTwoWay(final Taggable taggable) { return BicycleOneWayTag.isTwoWay(taggable) && CyclewayOneWayTag.isTwoWay(taggable) && OneWayBicycleTag.isTwoWay(taggable) && CyclewayRightOneWayTag.isTwoWay(taggable) && CyclewayLeftOneWayTag.isTwoWay(taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/bicycle/BicycleOneWayTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.bicycle; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/bicycle%3Aoneway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum BicycleOneWayTag { YES, NO, @TagValueAs("-1") MINUS_1; @TagKey public static final String KEY = "bicycle:oneway"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(BicycleOneWayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/bicycle/CyclewayLeftOneWayTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.bicycle; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Aleft%3Aoneway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum CyclewayLeftOneWayTag { YES, NO, @TagValueAs("1") ONE, @TagValueAs("-1") MINUS_1, LANE, OPPOSITE, FALSE; @TagKey public static final String KEY = "cycleway:left:oneway"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES, ONE, LANE); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1, OPPOSITE); protected static final Set TWO_WAYS = EnumSet.of(NO, FALSE); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(CyclewayLeftOneWayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/bicycle/CyclewayOneWayTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.bicycle; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Aoneway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum CyclewayOneWayTag { YES, NO, @TagValueAs("-1") MINUS_1; @TagKey public static final String KEY = "cycleway:oneway"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(CyclewayOneWayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/bicycle/CyclewayRightOneWayTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.bicycle; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/cycleway%3Aright%3Aoneway#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum CyclewayRightOneWayTag { YES, NO, @TagValueAs("1") ONE, @TagValueAs("-1") MINUS_1, LANE, DESIGNATED; @TagKey public static final String KEY = "cycleway:right:oneway"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES, ONE, LANE, DESIGNATED); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(CyclewayRightOneWayTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/bicycle/OneWayBicycleTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.bicycle; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/oneway%3Abicycle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum OneWayBicycleTag { YES, NO, @TagValueAs("1") ONE, @TagValueAs("-1") MINUS_1, OPPOSITE; @TagKey public static final String KEY = "oneway:bicycle"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES, ONE); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(OPPOSITE, MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(OneWayBicycleTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/motor/OneWayMotorVehicleTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.motor; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/oneway%3Amotor_vehicle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum OneWayMotorVehicleTag { YES, NO, @TagValueAs("-1") MINUS_1; @TagKey public static final String KEY = "oneway:motor_vehicle"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(OneWayMotorVehicleTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/motor/OneWayMotorcarTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.motor; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/oneway%3Amotorcar#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum OneWayMotorcarTag { YES, NO, @TagValueAs("-1") MINUS_1; @TagKey public static final String KEY = "oneway:motorcar"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(OneWayMotorcarTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/tags/oneway/motor/OneWayVehicleTag.java ================================================ package org.openstreetmap.atlas.tags.oneway.motor; import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.annotations.Tag; import org.openstreetmap.atlas.tags.annotations.TagKey; import org.openstreetmap.atlas.tags.annotations.TagValueAs; import org.openstreetmap.atlas.tags.annotations.validation.Validators; /** * @author matthieun */ @Tag(taginfo = "https://taginfo.openstreetmap.org/keys/oneway%3Avehicle#values", osm = "http://wiki.openstreetmap.org/wiki/Key:oneway") public enum OneWayVehicleTag { YES, NO, @TagValueAs("-1") MINUS_1; @TagKey public static final String KEY = "oneway:vehicle"; protected static final Set ONE_WAYS_FORWARD = EnumSet.of(YES); protected static final Set ONE_WAYS_REVERSED = EnumSet.of(MINUS_1); protected static final Set TWO_WAYS = EnumSet.of(NO); public static boolean isExplicitlyTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && TWO_WAYS.contains(oneWay.get()); } public static boolean isOneWayForward(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_FORWARD.contains(oneWay.get()); } public static boolean isOneWayReversed(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isPresent() && ONE_WAYS_REVERSED.contains(oneWay.get()); } public static boolean isTwoWay(final Taggable taggable) { final Optional oneWay = tag(taggable); return oneWay.isEmpty() || TWO_WAYS.contains(oneWay.get()); } public static Optional tag(final Taggable taggable) { return Validators.from(OneWayVehicleTag.class, taggable); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/README.md ================================================ # Utilities package This package contains utilities that help with collections, [scalars](scalars), threading, running java commands, converting, counting, and many others. ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/AbstractArchiverOrExtractor.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.concurrent.CopyOnWriteArraySet; /** * Abstract superclass for the Archiver and Extractor classes; handles all of the event listener * subscriptions and provides helper methods for sending events * * @author cstaylor * @param * either the Archiver or Extractor class */ abstract class AbstractArchiverOrExtractor { private final Collection> listeners; private final Class klass; private ArchiveVetoDelegate delegate; protected AbstractArchiverOrExtractor(final Class klass) { this.klass = klass; this.listeners = new CopyOnWriteArraySet<>(); } public AbstractArchiverOrExtractor addArchiverEventListener( final ArchiverEventListener listener) { this.listeners.add(listener); return this; } public AbstractArchiverOrExtractor removeArchiverEventListener( final ArchiverEventListener listener) { this.listeners.remove(listener); return this; } public AbstractArchiverOrExtractor setVetoDelegate(final ArchiveVetoDelegate delegate) { this.delegate = delegate; return this; } protected void fireArchiveCompleted() { for (final ArchiverEventListener listener : this.listeners) { listener.archiveCompleted(this.klass.cast(this)); } } protected void fireArchiveFailed() { for (final ArchiverEventListener listener : this.listeners) { listener.archiveFailed(this.klass.cast(this)); } } protected void fireArchiveStarted() { for (final ArchiverEventListener listener : this.listeners) { listener.archiveStarted(this.klass.cast(this)); } } protected void fireItemCompleted(final File file) { for (final ArchiverEventListener listener : this.listeners) { listener.itemCompleted(this.klass.cast(this), file); } } protected void fireItemFailed(final File file, final IOException oops) { for (final ArchiverEventListener listener : this.listeners) { listener.itemFailed(this.klass.cast(this), file, oops); } } protected void fireItemInProgress(final File file, final long count, final long length) { for (final ArchiverEventListener listener : this.listeners) { listener.itemInProgress(this.klass.cast(this), file, count, length); } } protected void fireItemSkipped(final File file) { for (final ArchiverEventListener listener : this.listeners) { listener.itemSkipped(this.klass.cast(this), file); } } protected void fireItemStarted(final File file) { for (final ArchiverEventListener listener : this.listeners) { listener.itemStarted(this.klass.cast(this), file); } } protected boolean shouldSkip(final File file) { return this.delegate != null && this.delegate.shouldSkip(this.klass.cast(this), file); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/ArchiveStorageProfileDelegate.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.File; /** * Callback that informs an archiver for a given file if it should be compressed or not. Some file * formats are already compressed, so compressing them again is pointless and may use extra storage. * * @author cstaylor */ public interface ArchiveStorageProfileDelegate { /** * Determines if a given file should be compressed by the archiver * * @param item * the file in question * @return true if we should compress, false otherwise */ boolean shouldCompress(File item); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/ArchiveVetoDelegate.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.File; /** * Callback that informs an object of type T for a given file if it should be processed or not. Some * files should be skipped (for example, .DS_Store on the OS X), so this lets us configure the * Archiver to skip certain files. * * @param * the owner of item (in this case, an Archiver or Extractor) * @author cstaylor */ public interface ArchiveVetoDelegate { /** * For a given file item, should we skip it? * * @param source * the potential owner of the file * @param item * the file to possibly be skipped * @return true if this item should be skipped, false otherwise */ boolean shouldSkip(T source, File item); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/Archiver.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import org.apache.commons.compress.archivers.ArchiveException; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.DirectoryWalker; import org.apache.commons.io.IOUtils; import org.openstreetmap.atlas.streaming.NotifyingIOUtils; import org.openstreetmap.atlas.streaming.NotifyingIOUtils.IOProgressListener; /** * Class for archiving the contents of a directory into a ZIP archive * * @author cstaylor */ public final class Archiver extends AbstractArchiverOrExtractor { /** * Sets all files to the default mode of compression: no inspection of the type of file is done * * @author cstaylor */ public class DefaultArchiveStorageProfileDelegate implements ArchiveStorageProfileDelegate { private final boolean defaultMode; public DefaultArchiveStorageProfileDelegate(final boolean mode) { this.defaultMode = mode; } @Override public boolean shouldCompress(final File item) { return this.defaultMode; } } /** * Responsible for tracking the process of a file being archived * * @author cstaylor */ class Progress implements IOProgressListener { private final File file; private final long length; Progress(final File file) { this.file = file; this.length = file.length(); } @Override public void completed() { fireItemCompleted(this.file); } @Override public void failed(final IOException oops) { Archiver.this.errorCount++; fireItemFailed(this.file, oops); } @Override public void started() { fireItemStarted(this.file); } @Override public void statusUpdate(final long count) { fireItemInProgress(this.file, count, this.length); } } /** * The guts of the archival algorithm: - On start, initialize the base path to the file's * absolute path and save the length of the name - On handleFile, copy the file contents into * the zip archive - on end, close the zip archive stream * * @author cstaylor */ private class MyDirectoryWalker extends DirectoryWalker { private String path; private int length; public void walk(final File file) throws IOException, ArchiveException { walk(file, new ArrayList()); } @Override protected void handleEnd(final Collection items) throws IOException { Archiver.this.archiveOutputStream.finish(); Archiver.this.archiveOutputStream.close(); super.handleEnd(items); } @Override protected void handleFile(final File file, final int depth, final Collection items) throws IOException { if (shouldSkip(file)) { fireItemSkipped(file); } else { try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(file))) { final String path = file.getAbsolutePath().substring(this.length); final ZipArchiveEntry entry = new ZipArchiveEntry(path); if (Archiver.this.storageDelegate != null) { final int level = Archiver.this.storageDelegate.shouldCompress(file) ? ZipArchiveEntry.DEFLATED : ZipArchiveEntry.STORED; entry.setMethod(level); } Archiver.this.archiveOutputStream.putArchiveEntry(entry); NotifyingIOUtils.copy(input, Archiver.this.archiveOutputStream, new Progress(file)); IOUtils.closeQuietly(input); Archiver.this.archiveOutputStream.closeArchiveEntry(); } catch (final FileNotFoundException oops) { // This can happen on Linux if the filename is corrupt // and we try to open the file fireItemFailed(file, oops); } } super.handleFile(file, depth, items); } @Override protected void handleStart(final File file, final Collection items) throws IOException { this.path = file.getAbsolutePath(); this.length = this.path.length() + 1; super.handleStart(file, items); } } /** * Where we are compressing to */ private final ArchiveOutputStream archiveOutputStream; /** * The number of errors that occurred during archival */ private int errorCount; private ArchiveStorageProfileDelegate storageDelegate; /** * Static factory method for creating a new Archiver with the ZIP archive format; currently the * only format supported * * @param outputFile * the destination zip file * @return the Archiver instance; useful for chaining method calls * @throws ArchiveException * if something compression related failed * @throws IOException * if something I/O related failed */ public static Archiver createZipArchiver(final File outputFile) throws ArchiveException, IOException { return Archiver.createZipArchiver(outputFile, false); } /** * Static factory method for creating a new Archiver with the ZIP archive format; currently the * only format supported * * @param outputFile * the destination zip file * @param compress * true if we should compress all entries by default, false will just stored them * uncompressed * @return the Archiver instance; useful for chaining method calls * @throws ArchiveException * if something compression related failed * @throws IOException * if something I/O related failed */ public static Archiver createZipArchiver(final File outputFile, final boolean compress) throws ArchiveException, IOException { final ZipArchiveOutputStream zout = new ZipArchiveOutputStream(outputFile); zout.setEncoding("UTF-8"); zout.setFallbackToUTF8(true); zout.setUseLanguageEncodingFlag(true); if (compress) { zout.setMethod(ZipArchiveOutputStream.DEFLATED); } else { zout.setMethod(ZipArchiveOutputStream.STORED); } zout.setCreateUnicodeExtraFields( ZipArchiveOutputStream.UnicodeExtraFieldPolicy.NOT_ENCODEABLE); return new Archiver(zout); } public static Archiver createZipArchiver(final Path outputPath) throws ArchiveException, IOException { if (outputPath == null) { throw new IllegalArgumentException("outputPath can't be null"); } return createZipArchiver(outputPath.toFile()); } public static Archiver createZipArchiver(final Path outputPath, final boolean compress) throws ArchiveException, IOException { if (outputPath == null) { throw new IllegalArgumentException("outputPath can't be null"); } return createZipArchiver(outputPath.toFile(), compress); } /** * Creates a new Archiver specifying where the directory contents will be archived to. * * @param archiveOutputStream * the destination archive output stream (ZIP) */ private Archiver(final ArchiveOutputStream archiveOutputStream) throws IOException { super(Archiver.class); this.archiveOutputStream = archiveOutputStream; setVetoDelegate(new DefaultZipVetoDelegate()); } @Override public Archiver addArchiverEventListener(final ArchiverEventListener listener) { super.addArchiverEventListener(listener); return this; } /** * Archives the contents of inputFile to the previously set zip archive stream. Algorithm: - * Sanity check the current state of the Extractor and the inputFile argument - Fire the archive * started event - Walk the contents of the inputFile; this will delegate all compression calls * - If at least one error occurred, fire the archive failed event - Otherwise, fire the archive * completed event * * @param inputFile * the directory we want to extract; must not be null * @return the Archiver instance; useful for chaining method calls * @throws ArchiveException * if something compression related failed * @throws IOException * if something I/O related failed */ public Archiver compress(final File inputFile) throws ArchiveException, IOException { if (inputFile == null) { throw new IllegalArgumentException("inputFile can't be null"); } if (this.archiveOutputStream == null) { throw new IllegalStateException("os can't be null"); } fireArchiveStarted(); new MyDirectoryWalker().walk(inputFile); if (this.errorCount > 0) { fireArchiveFailed(); } else { fireArchiveCompleted(); } return this; } public Archiver compress(final Path inputFile) throws ArchiveException, IOException { if (inputFile == null) { throw new IllegalArgumentException("inputFile can't be null"); } return compress(inputFile.toFile()); } @Override public Archiver removeArchiverEventListener(final ArchiverEventListener listener) { super.removeArchiverEventListener(listener); return this; } public Archiver setStorageDelegate(final ArchiveStorageProfileDelegate storageDelegate) { this.storageDelegate = storageDelegate; return this; } @Override public Archiver setVetoDelegate(final ArchiveVetoDelegate delegate) { super.setVetoDelegate(delegate); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/ArchiverEventListener.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.File; import java.io.IOException; /** * Callback methods for status updates during an extraction (Extractor) or archival (Archiver) * operation. * * @author cstaylor * @param * either the Archiver or Extractor class */ public interface ArchiverEventListener { /** * Called once after the operation completes successfully * * @param source * the source Archiver or Extractor object */ void archiveCompleted(T source); /** * Called once after the operation completes with errors * * @param source * the source Archiver or Extractor object */ void archiveFailed(T source); /** * Called once before the operation begins * * @param source * the source Archiver or Extractor object */ void archiveStarted(T source); /** * Called once for every item that has been extracted or archived successfully * * @param source * the source Archiver or Extractor object * @param item * the item that will be extracted or archived */ void itemCompleted(T source, File item); /** * Called once for every item that has been extracted or archived with errors * * @param source * the source Archiver or Extractor object * @param item * the item that will be extracted or archived * @param oops * the IOException that caused the extraction or archival to fail */ void itemFailed(T source, File item, IOException oops); /** * Called after each partial copy of an item with the number of bytes read and the total number * of bytes that will be read * * @param source * the source Archiver or Extractor object * @param item * the item that is being extracted or archived * @param bytesRead * the number of bytes that have already been read * @param bytesTotal * the total number of bytes in item */ void itemInProgress(T source, File item, long bytesRead, long bytesTotal); /** * Called once for every item that was not processed because the implementation decided to skip * it * * @param source * the source Archiver or Extractor object * @param item * the item that was skipped */ void itemSkipped(T source, File item); /** * Called once for every item that will be extracted or archived * * @param source * the source Archiver or Extractor object * @param item * the item that will be extracted or archived */ void itemStarted(T source, File item); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/DefaultZipVetoDelegate.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.File; /** * Sample veto delegate that skips files we don't want on OS X * * @author cstaylor * @param * the archiver or extractor for this file */ public class DefaultZipVetoDelegate implements ArchiveVetoDelegate { @Override public boolean shouldSkip(final T source, final File item) { final String name = item.getAbsolutePath(); return name.contains(".DS_Store") || name.contains("__MACOSX"); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/Extractor.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Collections; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.io.FileUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.NotifyingIOUtils; import org.openstreetmap.atlas.streaming.NotifyingIOUtils.IOProgressListener; /** * Class for extracting the contents of a ZIP archive to a specified directory * * @author cstaylor */ public final class Extractor extends AbstractArchiverOrExtractor { /** * Responsible for tracking the process of a file being extracted from an archive * * @author cstaylor */ class Progress implements IOProgressListener { private final File file; private final long length; Progress(final File file, final long length) { this.file = file; this.length = length; } @Override public void completed() { fireItemCompleted(this.file); } @Override public void failed(final IOException oops) { Extractor.this.errorCount++; fireItemFailed(this.file, oops); } @Override public void started() { fireItemStarted(this.file); } @Override public void statusUpdate(final long count) { fireItemInProgress(this.file, count, this.length); } } /** * Where we are extracting to */ private final File outputDirectory; /** * The number of errors that occurred during extraction */ private int errorCount; /** * Skip existing? */ private boolean skipExisting; /** * Static factory method for creating a new Extractor with the ZIP archive format; currently the * only format supported * * @param outputDirectory * the destination directory for any extraction operations * @return the Extractor instance; useful for chaining method calls */ public static Extractor extractZipArchive(final File outputDirectory) { if (outputDirectory == null) { throw new IllegalArgumentException("outputDirectory is null"); } return new Extractor(outputDirectory); } public static Extractor extractZipArchive(final Path outputDirectory) { if (outputDirectory == null) { throw new IllegalArgumentException("outputDirectory is null"); } return new Extractor(outputDirectory.toFile()); } /** * Creates a new Extractor specifying where the contents of the zip file will be extracted to. * * @param output * the destination directory */ private Extractor(final File output) { super(Extractor.class); this.outputDirectory = output; setVetoDelegate(new DefaultZipVetoDelegate<>()); } @Override public Extractor addArchiverEventListener(final ArchiverEventListener listener) { super.addArchiverEventListener(listener); return this; } /** * Extracts the contents of inputFile to the previously set outputDirectory. *

* Algorithm:
*

    *
  • Sanity check the current state of the Extractor and the inputFile argument
    • *
  • Fire the archive started event
    • *
  • Walk the contents of the inputFile *
      *
    • Check if the output file's parent directory is created; if not, create it
      • *
    • Copy the contents of the archive entry to the output file
      • *
    • Close both the current archive entry input stream and the output file's output stream *
      • *
    *
    • *
  • If at least one error occurred, fire the archive failed event
    • *
  • Otherwise, fire the archive completed event
    • *
  • Close the zip file
    • *
* * @param inputFile * the zip file we want to extract; must not be null * @return the Extractor instance; useful for chaining method calls * @throws IOException * if something I/O related failed */ public Extractor extract(final File inputFile) throws IOException { if (inputFile == null) { throw new IllegalArgumentException("inputFile can't be null"); } if (this.outputDirectory == null) { throw new IllegalStateException("outputDirectory can't be null"); } if (this.outputDirectory.exists()) { FileUtils.deleteQuietly(this.outputDirectory); } if (!this.outputDirectory.mkdirs()) { throw new IOException( String.format("%s can't be created", this.outputDirectory.getAbsolutePath())); } fireArchiveStarted(); try (ZipFile file = new ZipFile(inputFile)) { for (final ZipArchiveEntry current : Collections.list(file.getEntries())) { final File outputFile = new File(this.outputDirectory, current.getName()); if (current.getName().contains("..")) { throw new CoreException("Using parent directory not allowed: {}", current.getName()); } if (shouldSkip(outputFile)) { fireItemSkipped(outputFile); } else if (outputFile.exists() && this.skipExisting) { fireItemSkipped(outputFile); } else if (current.isDirectory()) { // Continue } else { try (BufferedOutputStream bos = new BufferedOutputStream( new FileOutputStream(outputFile)); InputStream inputStream = file.getInputStream(current)) { outputFile.getParentFile().mkdirs(); NotifyingIOUtils.copy(inputStream, bos, new Progress(outputFile, current.getSize())); } } } if (this.errorCount > 0) { fireArchiveFailed(); } else { fireArchiveCompleted(); } } return this; } public Extractor extract(final Path inputPath) throws IOException { if (inputPath == null) { throw new IllegalArgumentException("inputFile can't be null"); } return extract(inputPath.toFile()); } public Extractor overwriteExisting() { this.skipExisting = false; return this; } @Override public Extractor removeArchiverEventListener(final ArchiverEventListener listener) { super.removeArchiverEventListener(listener); return this; } @Override public Extractor setVetoDelegate(final ArchiveVetoDelegate delegate) { super.setVetoDelegate(delegate); return this; } public Extractor skipExisting() { this.skipExisting = true; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/UnzipperCommand.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Demonstrates the Extractor class * * @author cstaylor */ public class UnzipperCommand extends Command { private static final Switch OUTPUT_FILE_PARAMETER = new Switch<>("output", "Output directory to store zip file entries", Paths::get, Optionality.REQUIRED); private static final Switch INPUT_ZIP_FILE_PARAMETER = new Switch<>("zip", "Zip file to extract all of the data", Paths::get, Optionality.REQUIRED); public static void main(final String... args) { new UnzipperCommand().runWithoutQuitting(args); } @Override protected int onRun(final CommandMap command) { final Path inputPath = prepareInput((Path) command.get(INPUT_ZIP_FILE_PARAMETER)); final Path outputFile = prepareOutput((Path) command.get(OUTPUT_FILE_PARAMETER)); try { Extractor.extractZipArchive(outputFile).extract(inputPath); } catch (final Exception oops) { throw new CoreException("Error when extracting: {} -> {}", inputPath, outputFile, oops); } return 0; } @Override protected SwitchList switches() { return new SwitchList().with(OUTPUT_FILE_PARAMETER, INPUT_ZIP_FILE_PARAMETER); } private Path prepareInput(final Path inputPath) { if (!Files.isReadable(inputPath)) { throw new CoreException("Can't read {} or it doesn't exist", inputPath); } return inputPath; } private Path prepareOutput(final Path outputFile) { if (Files.exists(outputFile) && !Files.isDirectory(outputFile)) { throw new CoreException("{} already exists and is not a directory", outputFile); } try { Files.createDirectories(outputFile); } catch (final Exception oops) { throw new CoreException("Can't create {}", outputFile, oops); } return outputFile; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/archive/ZipperCommand.java ================================================ package org.openstreetmap.atlas.utilities.archive; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import org.apache.commons.compress.archivers.ArchiveException; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.runtime.Command; import org.openstreetmap.atlas.utilities.runtime.CommandMap; /** * Demonstrates the Archiver class * * @author cstaylor */ public class ZipperCommand extends Command { private static final Switch INPUT_FILE_PARAMETER = new Switch<>("input", "Input files to store in a zip file", Paths::get, Optionality.REQUIRED); private static final Switch OUTPUT_ZIP_FILE_PARAMETER = new Switch<>("zip", "Zip file to store all of the data", Paths::get, Optionality.REQUIRED); private static final Flag COMPRESSION_FLAG = new Flag("compress", "Enable compression of all files"); public static void main(final String... args) { new ZipperCommand().runWithoutQuitting(args); } @Override protected int onRun(final CommandMap command) { final Path inputPath = prepareInput((Path) command.get(INPUT_FILE_PARAMETER)); final Path outputFile = prepareOutput((Path) command.get(OUTPUT_ZIP_FILE_PARAMETER)); try { Archiver.createZipArchiver(outputFile).compress(inputPath); } catch (final IOException | ArchiveException oops) { throw new CoreException("Error when archiving: {} -> {}", inputPath, outputFile, oops); } return 0; } @Override protected SwitchList switches() { return new SwitchList().with(INPUT_FILE_PARAMETER, OUTPUT_ZIP_FILE_PARAMETER, COMPRESSION_FLAG); } private Path prepareInput(final Path inputPath) { if (!Files.isReadable(inputPath)) { throw new CoreException("Can't read {} or it doesn't exist", inputPath); } return inputPath; } private Path prepareOutput(final Path outputFile) { if (Files.exists(outputFile)) { throw new CoreException("{} already exists. Aborting", outputFile); } try { Files.createDirectories(outputFile.getParent()); } catch (final IOException oops) { throw new CoreException("Can't create parent directories for {}", outputFile, oops); } return outputFile; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/Arrays.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.util.Collection; /** * Utility method for arrays * * @author matthieun */ public final class Arrays { public static int[] addNewItem(final int[] existing, final int newValue) { final int[] result = new int[existing.length + 1]; for (int i = 0; i < existing.length; i++) { result[i] = existing[i]; } result[result.length - 1] = newValue; return result; } public static long[] addNewItem(final long[] existing, final long newValue) { final long[] result = new long[existing.length + 1]; for (int i = 0; i < existing.length; i++) { result[i] = existing[i]; } result[result.length - 1] = newValue; return result; } public static long[] addNewItemAndResizeOnlyIfNecessary(final long[] existing, final long newValue, final int index) { long[] result = existing; if (index >= existing.length) { // Needs resizing. Double the size... result = java.util.Arrays.copyOf(existing, existing.length * 2); } result[index] = newValue; return result; } public static long[] toArray(final Collection list) { final long[] result = new long[list.size()]; int index = 0; for (final Long value : list) { result[index++] = value; } return result; } public static long[] trimToSize(final long[] existing, final int size) { if (size <= 0) { return new long[0]; } return java.util.Arrays.copyOfRange(existing, 0, size - 1); } private Arrays() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/BitArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} for type {@link Boolean} * * @author matthieun */ public class BitArray extends LargeArray { /** * {@link PrimitiveArray} for type {@link Boolean} * * @author matthieun */ public static class BitPrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = 5686729947785133814L; private final boolean[] array = new boolean[size()]; public BitPrimitiveArray(final int size) { super(size); } @Override public Boolean get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new BitPrimitiveArray(size); } @Override public void set(final int index, final Boolean item) { this.array[index] = item; } } private static final long serialVersionUID = -342493023854989301L; public BitArray(final long maximumSize) { super(maximumSize); } public BitArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new BitPrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/BooleanArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} for type {@link Boolean} * * @author matthieun */ public class BooleanArray extends LargeArray { /** * {@link PrimitiveArray} for type {@link Boolean} * * @author matthieun */ public static class BooleanPrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = -3932998946137654750L; private final boolean[] array = new boolean[size()]; public BooleanPrimitiveArray(final int size) { super(size); } @Override public Boolean get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new BooleanPrimitiveArray(size); } @Override public void set(final int index, final Boolean item) { this.array[index] = item; } } private static final long serialVersionUID = -6015640040842668449L; public BooleanArray(final long maximumSize) { super(maximumSize); } public BooleanArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new BooleanPrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/ByteArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} for type {@link Byte} * * @author matthieun */ public class ByteArray extends LargeArray { /** * {@link PrimitiveArray} for type {@link Byte} * * @author matthieun */ public static class BytePrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = 6928093441524751750L; private final byte[] array = new byte[size()]; public BytePrimitiveArray(final int size) { super(size); } @Override public Byte get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new BytePrimitiveArray(size); } @Override public void set(final int index, final Byte item) { this.array[index] = item; } } private static final long serialVersionUID = 6401198662134364211L; public ByteArray(final long maximumSize) { super(maximumSize); } public ByteArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new BytePrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/ByteArrayOfArrays.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.util.Arrays; import java.util.Objects; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoByteArrayOfArraysAdapter; /** * {@link LargeArray} of arrays of byte (byte[]) * * @author matthieun * @author lcram */ public class ByteArrayOfArrays extends LargeArray implements ProtoSerializable { /** * {@link PrimitiveArray} for type {@link Byte} * * @author matthieun */ public static class PrimitiveByteArrayArray extends PrimitiveArray { private static final long serialVersionUID = -89500356133572470L; private final byte[][] array = new byte[size()][]; public PrimitiveByteArrayArray(final int size) { super(size); } @Override public byte[] get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitiveByteArrayArray(size); } @Override public void set(final int index, final byte[] item) { this.array[index] = item; } } private static final long serialVersionUID = 9114627155973700091L; public ByteArrayOfArrays(final long maximumSize) { super(maximumSize); } public ByteArrayOfArrays(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a * {@link ByteArrayOfArrays} that it can use to grab the correct {@link ProtoAdapter}. The * object initialized with this constructor will be corrupted for general use and should be * discarded. */ @SuppressWarnings("unused") private ByteArrayOfArrays() { super(); } @Override public boolean equals(final Object other) { if (other instanceof ByteArrayOfArrays) { if (this == other) { return true; } final ByteArrayOfArrays that = (ByteArrayOfArrays) other; if (!Objects.equals(this.getName(), that.getName())) { return false; } if (this.size() != that.size()) { return false; } for (long index = 0; index < this.size(); index++) { final byte[] thisSubArray = this.get(index); final byte[] thatSubArray = that.get(index); if (thisSubArray.length != thatSubArray.length) { return false; } for (int subIndex = 0; subIndex < thisSubArray.length; subIndex++) { if (thisSubArray[subIndex] != thatSubArray[subIndex]) { return false; } } } return true; } return false; } @Override public ProtoAdapter getProtoAdapter() { return new ProtoByteArrayOfArraysAdapter(); } /* * NOTE: This hashCode implementation uses the array object references and not the actual array * values. Keep this in mind before using. */ @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; int hash = initialPrime; for (int index = 0; index < this.size(); index++) { hash = hashSeed * hash + Arrays.hashCode(this.get(index)); } return hash; } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitiveByteArrayArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/IntegerArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} for type {@link Integer} * * @author matthieun */ public class IntegerArray extends LargeArray { /** * {@link PrimitiveArray} for type {@link Integer} * * @author matthieun */ public static class IntegerPrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = -3932998946137654750L; private final int[] array = new int[size()]; public IntegerPrimitiveArray(final int size) { super(size); } @Override public Integer get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new IntegerPrimitiveArray(size); } @Override public void set(final int index, final Integer item) { this.array[index] = item; } } private static final long serialVersionUID = -6015640040842668449L; public IntegerArray(final long maximumSize) { super(maximumSize); } public IntegerArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new IntegerPrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/IntegerArrayOfArrays.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.util.Arrays; import java.util.Objects; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoIntegerArrayOfArraysAdapter; /** * {@link LargeArray} of arrays of int (int[]) * * @author matthieun * @author lcram */ public class IntegerArrayOfArrays extends LargeArray implements ProtoSerializable { /** * {@link PrimitiveArray} for type {@link Long} * * @author matthieun */ public static class PrimitiveIntegerArrayArray extends PrimitiveArray { private static final long serialVersionUID = -4397674333836117261L; private final int[][] array = new int[size()][]; public PrimitiveIntegerArrayArray(final int size) { super(size); } @Override public int[] get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitiveIntegerArrayArray(size); } @Override public void set(final int index, final int[] item) { this.array[index] = item; } } private static final long serialVersionUID = 6793499910834785994L; public IntegerArrayOfArrays(final long maximumSize) { super(maximumSize); } public IntegerArrayOfArrays(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a * {@link IntegerArrayOfArrays} that it can use to grab the correct {@link ProtoAdapter}. The * object initialized with this constructor will be corrupted for general use and should be * discarded. */ @SuppressWarnings("unused") private IntegerArrayOfArrays() { super(); } @Override public boolean equals(final Object other) { if (other instanceof IntegerArrayOfArrays) { if (this == other) { return true; } final IntegerArrayOfArrays that = (IntegerArrayOfArrays) other; if (!Objects.equals(this.getName(), that.getName())) { return false; } if (this.size() != that.size()) { return false; } for (long index = 0; index < this.size(); index++) { final int[] thisSubArray = this.get(index); final int[] thatSubArray = that.get(index); if (thisSubArray.length != thatSubArray.length) { return false; } for (int subIndex = 0; subIndex < thisSubArray.length; subIndex++) { if (thisSubArray[subIndex] != thatSubArray[subIndex]) { return false; } } } return true; } return false; } @Override public ProtoAdapter getProtoAdapter() { return new ProtoIntegerArrayOfArraysAdapter(); } /* * NOTE: This hashCode implementation uses the array object references and not the actual array * values. Keep this in mind before using. */ @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; final int nameHash = this.getName() == null ? 0 : this.getName().hashCode(); int hash = hashSeed * initialPrime + nameHash; hash = hashSeed * hash + Long.valueOf(this.size()).hashCode(); for (long index = 0; index < this.size(); index++) { hash = hashSeed * hash + Arrays.hashCode(this.get(index)); } return hash; } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitiveIntegerArrayArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/LargeArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.io.Serializable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Objects; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.scalars.Ratio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A LargeArray that can have more than {@link Integer}.MAX_VALUE items. It is backed by multiple * T[] arrays. Items can be added and gotten, but not removed. The get operations take long indices. * * @param * The type of items in the array * @author matthieun * @author lcram */ public abstract class LargeArray implements Iterable, Serializable { private static final long serialVersionUID = -8827093729953143426L; private static final Logger logger = LoggerFactory.getLogger(LargeArray.class); private static final int DEFAULT_MEMORY_BLOCK_SIZE = 1024; private final List> arrays; private final long maximumSize; private long nextIndex = 0; private final int memoryBlockSize; private final int subArraySize; private String name = null; /** * Create a {@link LargeArray} * * @param maximumSize * The maximum size of the array */ public LargeArray(final long maximumSize) { this(maximumSize, DEFAULT_MEMORY_BLOCK_SIZE, Integer.MAX_VALUE); } /** * Create a {@link LargeArray} * * @param maximumSize * The maximum size of the array * @param memoryBlockSize * The initial block size for sub-array creation. If small, it might resize often, if * large, there might be unused space. * @param subArraySize * The maximum size of the sub arrays */ public LargeArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { if (memoryBlockSize < 0) { throw new CoreException("memoryBlockSize ({}) cannot be negative", memoryBlockSize); } if (subArraySize < 0) { throw new CoreException("subArraySize ({}) cannot be negative", subArraySize); } this.maximumSize = maximumSize; this.arrays = new ArrayList<>(); this.memoryBlockSize = memoryBlockSize; this.subArraySize = subArraySize; } /** * This nullary constructor exists solely for subclasses of {@link LargeArray} that wish to * implement their own nullary constructor. These nullary constructors should only be used by * serialization code in {@link PackedAtlasSerializer} that needs to obtain * {@link ProtoAdapter}s. The objects they initialize are corrupted for general use and should * be discarded. */ protected LargeArray() { this.arrays = null; this.maximumSize = 0; this.memoryBlockSize = 0; this.subArraySize = 0; } /** * Add an item to the array * * @param item * The item to add * @throws CoreException * if the array is full. */ public void add(final T item) { if (this.nextIndex >= this.maximumSize) { throw new CoreException("The array is full. Cannot add " + item); } final int arrayIndex = arrayIndex(this.nextIndex); final int indexInside = indexInside(this.nextIndex); if (this.arrays.size() <= arrayIndex) { // Set an array size this.arrays.add(getNewArray(this.memoryBlockSize)); } if (indexInside >= this.arrays.get(arrayIndex).size()) { final PrimitiveArray old = this.arrays.get(arrayIndex); final int maximumSizeFromDoubling = Math.min(2 * old.size(), this.subArraySize); final long filledArraysSize = filledArraysSize(); final int maximumSizeFromTotal = (int) Math.min(this.maximumSize - filledArraysSize, this.subArraySize); final int newSize = Math.min(maximumSizeFromDoubling, maximumSizeFromTotal); logger.warn("Resizing array {} of {} ({}), from {} to {}.", arrayIndex, getName() == null ? super.toString() : getName(), this.getClass().getSimpleName(), old.size(), newSize); this.arrays.set(arrayIndex, old.withNewSize(newSize)); } this.arrays.get(arrayIndex(this.nextIndex)).set(indexInside(this.nextIndex), item); this.nextIndex++; } /** * A basic equals() implementation. Note that if this class is parameterized with an array type, * this method may not work as expected (due to array equals() performing a reference * comparison). Child classes of LargeArray may want to override this method to improve its * behavior in special cases. */ @Override public boolean equals(final Object other) { if (other instanceof LargeArray) { if (this == other) { return true; } @SuppressWarnings("unchecked") final LargeArray that = (LargeArray) other; if (!Objects.equals(this.getName(), that.getName())) { return false; } if (this.size() != that.size()) { return false; } for (long index = 0; index < this.size(); index++) { if (!this.get(index).equals(that.get(index))) { return false; } } return true; } return false; } /** * Get an item from the array * * @param index * The index to get * @return The item at the specified index * @throws CoreException * If the index is out of bounds. */ public T get(final long index) { if (index >= this.nextIndex) { throw new CoreException(index + " is out of bounds (size = " + size() + ")"); } return this.arrays.get(arrayIndex(index)).get(indexInside(index)); } /** * @return The name of this array */ public String getName() { return this.name; } @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; final int nameHash = this.getName() == null ? 0 : this.getName().hashCode(); int hash = hashSeed * initialPrime + nameHash; hash = hashSeed * hash + Long.valueOf(this.size()).hashCode(); for (long index = 0; index < this.size(); index++) { hash = hashSeed * hash + this.get(index).hashCode(); } return hash; } public boolean isEmpty() { return size() == 0; } @Override public Iterator iterator() { return Iterables.indexBasedIterable(size(), this::get).iterator(); } /** * Replace an already existing value * * @param index * The index to replace the value at * @param item * The new value to put */ public void set(final long index, final T item) { if (index >= size()) { throw new CoreException("Cannot replace an element that is not there. Index: " + index + ", size: " + size()); } this.arrays.get(arrayIndex(index)).set(indexInside(index), item); } public void setName(final String name) { this.name = name; } /** * @return The used size (which will always be smaller or equal to the size allocated in memory) */ public long size() { return this.nextIndex; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("["); builder.append(this.getClass().getSimpleName()); builder.append(" "); this.forEach(t -> { if (builder.length() > 0) { builder.append(", "); } builder.append(t); }); builder.append("]"); return builder.toString(); } /** * Trim this {@link LargeArray}. WARNING: As much as this might save memory on arrays filled a * small amount compared to the memoryBlockSize, it will resize the last {@link PrimitiveArray} * of this {@link LargeArray} which might take a lot of time, and (temporarily) a lot of memory. */ public void trim() { trimIfLessFilledThan(Ratio.MAXIMUM); } /** * Trim this {@link LargeArray} if and only if the fill {@link Ratio} of the last * {@link PrimitiveArray} of this {@link LargeArray} is less than the provided {@link Ratio} * * @param ratio * The provided reference {@link Ratio} */ public void trimIfLessFilledThan(final Ratio ratio) { logger.trace("Trimming {} with Ratio {}", getName(), ratio); if (this.arrays.isEmpty()) { return; } final int arrayIndex = this.arrays.size() - 1; final PrimitiveArray rightmost = this.arrays.get(arrayIndex); /* * Exit early in the case that the rightmost subarray is full - there is nothing to trim. In * fact, the trim logic below breaks in this case, and will wipe out the rightmost subarray * instead of trimming it. */ if (rightmostSubarrayIsFull()) { return; } // Here nextIndex is actually the size, and not size-1 final int indexInside = indexInside(this.nextIndex); if (Ratio.ratio((double) indexInside / rightmost.size()).isLessThan(ratio)) { this.arrays.set(arrayIndex, rightmost.trimmed(indexInside)); } } public LargeArray withName(final String name) { setName(name); return this; } /** * @return the arrays, for testing */ protected List> getArrays() { return this.arrays; } /** * Create a new primitive array of the Item. * * @param size * The size of the array * @return The new primitive array. */ protected abstract PrimitiveArray getNewArray(int size); private int arrayIndex(final long index) { return (int) (index / this.subArraySize); } private long filledArraysSize() { if (this.arrays.size() <= 1) { return 0L; } long result = 0L; for (int i = 0; i < this.arrays.size() - 2; i++) { result += this.arrays.get(i).size(); } return result; } private int indexInside(final long index) { return (int) (index % this.subArraySize); } private boolean rightmostSubarrayIsFull() { return this.nextIndex > 0 && indexInside(this.nextIndex) == 0; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/LongArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoLongArrayAdapter; /** * {@link LargeArray} for type {@link Long} * * @author matthieun */ public class LongArray extends LargeArray implements ProtoSerializable { /** * {@link PrimitiveArray} for type {@link Long} * * @author matthieun */ public static class PrimitiveLongArray extends PrimitiveArray { private static final long serialVersionUID = 8325024756132919495L; private final long[] array = new long[size()]; public PrimitiveLongArray(final int size) { super(size); } @Override public Long get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitiveLongArray(size); } @Override public void set(final int index, final Long item) { this.array[index] = item; } } private static final long serialVersionUID = -6368556371326217582L; public LongArray(final long maximumSize) { super(maximumSize); } public LongArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a {@link LongArray} * that it can use to grab the correct {@link ProtoAdapter}. The object initialized with this * constructor will be corrupted for general use and should be discarded. */ @SuppressWarnings("unused") private LongArray() { super(); } @Override public ProtoAdapter getProtoAdapter() { return new ProtoLongArrayAdapter(); } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitiveLongArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/LongArrayOfArrays.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.util.Objects; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoLongArrayOfArraysAdapter; /** * {@link LargeArray} of arrays of long (long[]) * * @author matthieun * @author lcram */ public class LongArrayOfArrays extends LargeArray implements ProtoSerializable { /** * {@link PrimitiveArray} for type {@link Long} * * @author matthieun */ public static class PrimitiveLongArrayArray extends PrimitiveArray { private static final long serialVersionUID = 8377199081867533638L; private final long[][] array = new long[size()][]; public PrimitiveLongArrayArray(final int size) { super(size); } @Override public long[] get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitiveLongArrayArray(size); } @Override public void set(final int index, final long[] item) { this.array[index] = item; } } private static final long serialVersionUID = -1847060797258075365L; public LongArrayOfArrays(final long maximumSize) { super(maximumSize); } public LongArrayOfArrays(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a * {@link LongArrayOfArrays} that it can use to grab the correct {@link ProtoAdapter}. The * object initialized with this constructor will be corrupted for general use and should be * discarded. */ @SuppressWarnings("unused") private LongArrayOfArrays() { super(); } @Override public boolean equals(final Object other) { if (other instanceof LongArrayOfArrays) { if (this == other) { return true; } final LongArrayOfArrays that = (LongArrayOfArrays) other; if (!Objects.equals(this.getName(), that.getName())) { return false; } if (this.size() != that.size()) { return false; } for (long index = 0; index < this.size(); index++) { final long[] thisSubArray = this.get(index); final long[] thatSubArray = that.get(index); if (thisSubArray.length != thatSubArray.length) { return false; } for (int subIndex = 0; subIndex < thisSubArray.length; subIndex++) { if (thisSubArray[subIndex] != thatSubArray[subIndex]) { return false; } } } return true; } return false; } @Override public ProtoAdapter getProtoAdapter() { return new ProtoLongArrayOfArraysAdapter(); } /* * NOTE: This hashCode implementation uses the array object references and not the actual array * values. Keep this in mind before using. */ @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; final int nameHash = this.getName() == null ? 0 : this.getName().hashCode(); int hash = hashSeed * initialPrime + nameHash; hash = hashSeed * hash + Long.valueOf(this.size()).hashCode(); for (long index = 0; index < this.size(); index++) { hash = hashSeed * hash + this.get(index).hashCode(); } return hash; } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitiveLongArrayArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/PolyLineArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.StringCompressedPolyLine; import org.openstreetmap.atlas.geography.StringCompressedPolyLine.PolyLineCompressionException; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoPolyLineArrayAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link LargeArray} of {@link StringCompressedPolyLine}s, with an interface using {@link PolyLine} * s. * * @author matthieun */ public class PolyLineArray extends LargeArray implements ProtoSerializable { /** * @author matthieun * @param * An implementation of {@link PolyLine} */ public abstract static class PrimitivePointsArray extends PrimitiveArray { private static final long serialVersionUID = 3532399057462343784L; private final byte[][] encodings; public PrimitivePointsArray(final int size) { super(size); this.encodings = new byte[size][]; } @Override public void set(final int index, final Poly item) { // Here, whether it is a polyline or a polygon does not matter, the encodings will be // the same StringCompressedPolyLine compressed; try { compressed = new StringCompressedPolyLine(item); } catch (final PolyLineCompressionException e) { logger.error("Unable to compress polyLine {} at index {}. Sending to Null Island.", item, index, e); compressed = new StringCompressedPolyLine(new PolyLine(Location.CENTER)); } this.encodings[index] = compressed.getEncoding(); } protected byte[][] getEncodings() { return this.encodings; } } /** * A primitive array specifically for {@link PolyLine} * * @author matthieun */ public static class PrimitivePolyLineArray extends PrimitivePointsArray { private static final long serialVersionUID = -9008848366079793820L; public PrimitivePolyLineArray(final int size) { super(size); } @Override public PolyLine get(final int index) { return new StringCompressedPolyLine(getEncodings()[index]).asPolyLine(); } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitivePolyLineArray(size); } } private static final Logger logger = LoggerFactory.getLogger(PolyLineArray.class); private static final long serialVersionUID = -4475168018638543482L; public PolyLineArray(final long maximumSize) { super(maximumSize); } public PolyLineArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a * {@link PolyLineArray} that it can use to grab the correct {@link ProtoAdapter}. The object * initialized with this constructor will be corrupted for general use and should be discarded. */ @SuppressWarnings("unused") private PolyLineArray() { super(); } @Override public ProtoAdapter getProtoAdapter() { return new ProtoPolyLineArrayAdapter(); } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitivePolyLineArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/PolygonArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.StringCompressedPolygon; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasSerializer; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoPolygonArrayAdapter; import org.openstreetmap.atlas.utilities.arrays.PolyLineArray.PrimitivePointsArray; /** * {@link LargeArray} of {@link StringCompressedPolygon}s, with an interface of {@link Polygon}s. * * @author matthieun */ public class PolygonArray extends LargeArray implements ProtoSerializable { /** * Primitive array for polygons * * @author matthieun */ public static class PrimitivePolygonArray extends PrimitivePointsArray { private static final long serialVersionUID = 1115133908622542632L; public PrimitivePolygonArray(final int size) { super(size); } @Override public Polygon get(final int index) { return new StringCompressedPolygon(getEncodings()[index]).asPolygon(); } @Override public PrimitiveArray getNewArray(final int size) { return new PrimitivePolygonArray(size); } } private static final long serialVersionUID = -2337695414673604456L; public PolygonArray(final long maximumSize) { super(maximumSize); } public PolygonArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } /** * This nullary constructor is solely for use by the {@link PackedAtlasSerializer}, which calls * it using reflection. It allows the serializer code to obtain a handle on a * {@link PolygonArray} that it can use to grab the correct {@link ProtoAdapter}. The object * initialized with this constructor will be corrupted for general use and should be discarded. */ @SuppressWarnings("unused") private PolygonArray() { super(); } @Override public ProtoAdapter getProtoAdapter() { return new ProtoPolygonArrayAdapter(); } @Override protected PrimitiveArray getNewArray(final int size) { return new PrimitivePolygonArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/PrimitiveArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; import java.io.Serializable; import java.util.stream.IntStream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.collections.StringList; /** * Wrapper for a primitive array corresponding to a given type. * * @author matthieun * @param * The non-primitive type of the primitive type this array is wrapping. */ public abstract class PrimitiveArray implements Serializable { private static final long serialVersionUID = 7815464773512459667L; private final int size; /** * @param size * The size of the primitive array */ public PrimitiveArray(final int size) { this.size = size; } /** * @param index * The index wanted * @return The item at the index */ public abstract T get(int index); /** * Build a new empty primitive array of the specified size. * * @param size * the specified size * @return a new empty primitive array of the specified size */ public abstract PrimitiveArray getNewArray(int size); /** * Set an item at the specified index in the array. * * @param index * The specified index * @param item * The item to set at the specified index */ public abstract void set(int index, T item); /** * @return The size of this primitive array */ public int size() { return this.size; } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append("["); final StringList list = new StringList(); for (int i = 0; i < size(); i++) { list.add(this.get(i).toString()); } builder.append(list.join(", ")); builder.append("]"); return builder.toString(); } /** * Trim this array * * @param size * The size to trim to * @return The trimmed array with data originating from this array up to the size */ public PrimitiveArray trimmed(final int size) { if (size >= this.size) { return this; } final PrimitiveArray result = getNewArray(size); // Equivalent of for (int i = 0; i < size; i++) IntStream.range(0, size).forEach(i -> result.set(i, get(i))); return result; } /** * Copy this primitive array into another primitive array of bigger size. * * @param newSize * The bigger size * @return The new bigger array, copied from this one. */ public PrimitiveArray withNewSize(final int newSize) { if (newSize < size()) { throw new CoreException("Cannot copy into a smaller array. This is " + size() + " and the new size asked is " + newSize); } final PrimitiveArray young = getNewArray(newSize); for (int i = 0; i < size(); i++) { young.set(i, get(i)); } return young; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/ShortArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} for type {@link Short} * * @author matthieun */ public class ShortArray extends LargeArray { /** * {@link PrimitiveArray} for type {@link Short} * * @author matthieun */ public static class ShortPrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = 3177690048477030833L; private final short[] array = new short[size()]; public ShortPrimitiveArray(final int size) { super(size); } @Override public Short get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new ShortPrimitiveArray(size); } @Override public void set(final int index, final Short item) { this.array[index] = item; } } private static final long serialVersionUID = 6867216948199207925L; public ShortArray(final long maximumSize) { super(maximumSize); } public ShortArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new ShortPrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/arrays/StringArray.java ================================================ package org.openstreetmap.atlas.utilities.arrays; /** * {@link LargeArray} of Strings * * @author matthieun */ public class StringArray extends LargeArray { /** * {@link PrimitiveArray} of Strings. * * @author matthieun */ public static class StringPrimitiveArray extends PrimitiveArray { private static final long serialVersionUID = 6050182547243598715L; private final String[] array; public StringPrimitiveArray(final int size) { super(size); this.array = new String[size]; } @Override public String get(final int index) { return this.array[index]; } @Override public PrimitiveArray getNewArray(final int size) { return new StringPrimitiveArray(size); } @Override public void set(final int index, final String item) { this.array[index] = item; } } private static final long serialVersionUID = 5462179570391723788L; public StringArray(final long maximumSize) { super(maximumSize); } public StringArray(final long maximumSize, final int memoryBlockSize, final int subArraySize) { super(maximumSize, memoryBlockSize, subArraySize); } @Override protected PrimitiveArray getNewArray(final int size) { return new StringPrimitiveArray(size); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/ConcurrentResourceCache.java ================================================ package org.openstreetmap.atlas.utilities.caching; import java.net.URI; import java.util.Optional; import java.util.UUID; import java.util.function.Function; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.caching.strategies.CachingStrategy; import org.openstreetmap.atlas.utilities.caching.strategies.NamespaceCachingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** *

* The threadsafe {@link ResourceCache} implementation. There is a caveat, related to the fact that * some caching strategies utilize system-wide global state e.g. {@link NamespaceCachingStrategy} * uses the tmp filesystem. In doing so it becomes impossible to guarantee concurrency safety from * within the {@link ConcurrentResourceCache} alone.
* {@link ConcurrentResourceCache} needs a specified {@link CachingStrategy} and default fetching * {@link Function} at creation time. The cache then loads a resource using a given {@link URI}. * Since using {@link URI} objects can often be cumbersome, users of this class are encouraged to * extend it and overload the {@link ConcurrentResourceCache#get} method to take more convenient * parameters. *

* * @author lcram */ public class ConcurrentResourceCache implements ResourceCache { private static final Logger logger = LoggerFactory.getLogger(ConcurrentResourceCache.class); private final CachingStrategy cachingStrategy; private final Function> fetcher; private final UUID cacheID; /** * Create a new {@link ConcurrentResourceCache} with the given fetcher and strategy. * * @param cachingStrategy * the caching strategy * @param fetcher * the default fetcher */ public ConcurrentResourceCache(final CachingStrategy cachingStrategy, final Function> fetcher) { this.cachingStrategy = cachingStrategy; this.fetcher = fetcher; this.cacheID = UUID.randomUUID(); logger.info("Initialized cache {} with ID {}", this.getClass().getName(), this.cacheID); } @Override public Optional get(final URI resourceURI) { Optional cachedResource; // We must synchronize the application of the caching strategy since we cannot guarantee // that the strategy does not utilize internal global state. synchronized (this) { cachedResource = this.cachingStrategy.attemptFetch(resourceURI, this.fetcher); } if (cachedResource.isEmpty()) { logger.warn("CacheID {}: cache fetch of {} failed, falling back to default fetcher...", this.cacheID, resourceURI); // We must also synchronize the application of the fetcher, since it may rely on state // shared by the calling threads. synchronized (this) { cachedResource = this.fetcher.apply(resourceURI); } } return cachedResource; } /** * Get the name of the backing {@link CachingStrategy}. * * @return the name */ public String getStrategyName() { return this.cachingStrategy.getName(); } @Override public void invalidate() { logger.info("CacheID {}: invalidating cache", this.cacheID); // Synchronize invalidation with the same lock used to fetch and cache. This prevents // invalidation corruption. synchronized (this) { this.cachingStrategy.invalidate(); } } @Override public void invalidate(final URI resourceURI) { logger.info("CacheID {}: invalidating resource {}", this.cacheID, resourceURI); // Synchronize invalidation with the same lock used to fetch and cache. This prevents // invalidation corruption. synchronized (this) { this.cachingStrategy.invalidate(resourceURI); } } /** * Get a {@link UUID} for this cache instance. This is useful for logging. * * @return The cache instance {@link UUID} */ protected UUID getCacheID() { return this.cacheID; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/LocalFileInMemoryCache.java ================================================ package org.openstreetmap.atlas.utilities.caching; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Paths; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.caching.strategies.ByteArrayCachingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An example of how to extend the {@link ConcurrentResourceCache} to enhance functionality. This * class caches local files in memory, and abstracts the the messiness of file URIs behind a cleaner * interface. The {@link LocalFileInMemoryCache} forces the caching strategy to be * {@link ByteArrayCachingStrategy}, and forces the default fetcher to simply load a local file. * * @author lcram */ public class LocalFileInMemoryCache extends ConcurrentResourceCache { private static final Logger logger = LoggerFactory.getLogger(LocalFileInMemoryCache.class); /** * Create a new {@link LocalFileInMemoryCache} with the default {@link FileSystem}. See * {@link FileSystems#getDefault()} for more information. */ public LocalFileInMemoryCache() { this(FileSystems.getDefault()); } /** * Create a new {@link LocalFileInMemoryCache} with the given {@link FileSystem}. * * @param fileSystem * the {@link FileSystem} to use for file loading */ public LocalFileInMemoryCache(final FileSystem fileSystem) { super(new ByteArrayCachingStrategy(), uri -> { final File file = new File(uri.getPath(), fileSystem); if (!file.exists()) { logger.warn("File {} does not exist!", file); return Optional.empty(); } return Optional.of(file); }); } /** * Attempt to get the resource specified by the given path. * * @param path * the path to the desired resource * @return an {@link Optional} wrapping the {@link Resource} */ @Override public Optional get(final String path) { return this.get(Paths.get(path).toAbsolutePath().toUri()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/README.md ================================================ # utilities.caching package --- This package adds extensible `Resource` caching capabilities to `Atlas`. The basic idea is that users can create a cache with a default population mechanism (fetcher), choose a caching strategy, and then fetch desired resources (keyed by `URI`s) using the strategy. A very simple use case might look like: ```java // Let's read a resource at an arbitrary URI into a Resource. // Notice the fetcher function provided to constructor conforms to the // Function functional interface. final URI LOCAL_TEST_FILE_URI = URI.create("file:///path/to/some/file.txt"); // cache file contents into memory (a byte array) final ConcurrentResourceCache resourceCache = new ConcurrentResourceCache(new ByteArrayCachingStrategy(), uri -> Optional.of(new File(uri.getPath()))); // this will cache miss the first time and populate the cache using the provided fetcher Resource r1 = resourceCache.get(LOCAL_TEST_FILE_URI).get(); // this time we hit the cache, and read the bytes from memory instead of with the fetcher (from disk) Resource r2 = resourceCache.get(LOCAL_TEST_FILE_URI).get(); // Manually setting fetchers and CachingStrategies can be a pain. You can abstract // all this away by creating case-specific subclasses. This subclass cache uses // ByteArrayCachingStrategy by default and abstracts the get URI parameter by just // taking a path string as a parameter instead. final LocalFileInMemoryCache fileCache = new LocalFileInMemoryCache(); Resource r3 = fileCache.get("/path/to/another/file.txt").get(); ``` See the `CachingTests` class for more usage examples, and the `LocalFileInMemoryCache` class for an example of how to extend `ConcurrentResourceCache`. ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/ResourceCache.java ================================================ package org.openstreetmap.atlas.utilities.caching; import java.net.URI; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.caching.strategies.CachingStrategy; /** * All {@link ResourceCache}s must conform to this interface. Of particular interest is the thread * safe implementation of this interface, the {@link ConcurrentResourceCache}. * * @author lcram */ public interface ResourceCache { /** * Attempt to get the resource specified by the given string URI. * * @param resourceURIString * the resource {@link URI} as a {@link String} * @return an {@link Optional} wrapping the {@link Resource} */ default Optional get(final String resourceURIString) { return this.get(URI.create(resourceURIString)); } /** * Attempt to get the resource specified by the given URI. * * @param resourceURI * the resource {@link URI} * @return an {@link Optional} wrapping the {@link Resource} */ Optional get(URI resourceURI); /** * Invalidate the contents of this cache. Generally, this method should rely on the * {@link CachingStrategy#invalidate} implementation of the underlying strategy. However this is * not enforced by the interface. since some implementations may need to do extra housekeeping * to perform an invalidation. See {@link ConcurrentResourceCache} for an example. */ void invalidate(); /** * Invalidate the cached {@link Resource} for a given {@link URI}, if it exists. * * @param resourceURI * The {@link URI} of the {@link Resource} to invalidate */ void invalidate(URI resourceURI); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/AbstractCachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An incomplete implementation of the {@link CachingStrategy} interface. Provides some additional * functionality for subclasses to leverage. * * @author lcram */ public abstract class AbstractCachingStrategy implements CachingStrategy { private static final Logger logger = LoggerFactory.getLogger(AbstractCachingStrategy.class); /* * Cache the UUIDs for each URI so we only have to compute them once. Caching them in a * comparatively small map is significantly faster than recomputing them every time. Subclasses * may want to use this cache to associate a UUID with a given URI. */ private final Map uriStringToUUIDCache; private final UUID strategyID; public AbstractCachingStrategy() { this.uriStringToUUIDCache = new ConcurrentHashMap<>(); this.strategyID = UUID.randomUUID(); logger.info("Initialized strategy {} with ID {}", this.getClass().getName(), this.strategyID); } /** * Get a {@link UUID} for this strategy instance. This is useful for logging. * * @return The strategy instance {@link UUID} */ protected UUID getStrategyID() { return this.strategyID; } /** * Given a URI, compute a universally unique identifier ({@link UUID}) for that URI. This method * uses the {@link String} representation of a URI to compute the UUID. It will also cache * computed UUIDs, so subsequent fetches will not incur a re-computation performance penalty. * * @param resourceURI * the {@link URI} * @return the {@link UUID} of the given {@link URI} */ protected UUID getUUIDForResourceURI(final URI resourceURI) { final String uriString = resourceURI.toString(); if (!this.uriStringToUUIDCache.containsKey(uriString)) { // As of Java 8, this method computes the MD5 sum of the URI string. // This can be relatively slow (10 digests per 1 ms), so we will cache the result in // memory for subsequent requests. final UUID newUUID = UUID.nameUUIDFromBytes(uriString.getBytes()); this.uriStringToUUIDCache.put(uriString, newUUID); return newUUID; } else { return this.uriStringToUUIDCache.get(uriString); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/ByteArrayCachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.function.Function; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Caching strategy that attempts to cache a {@link Resource} in a byte array, in memory. * * @author lcram */ public class ByteArrayCachingStrategy extends AbstractCachingStrategy { /* * Default size is arbitrarily set to 2 MiB */ private static final long DEFAULT_BYTE_ARRAY_SIZE = 1024L * 1024 * 2; private static final Logger logger = LoggerFactory.getLogger(ByteArrayCachingStrategy.class); private final Map resourceCache; private long initialArraySize; private boolean useExactResourceSize; public ByteArrayCachingStrategy() { this.resourceCache = new HashMap<>(); this.initialArraySize = DEFAULT_BYTE_ARRAY_SIZE; this.useExactResourceSize = false; } @Override public Optional attemptFetch(final URI resourceURI, final Function> defaultFetcher) { final UUID resourceUUID = this.getUUIDForResourceURI(resourceURI); if (!this.resourceCache.containsKey(resourceUUID)) { logger.trace( "StrategyID {}: attempting to cache resource {} in byte array keyed on UUID {}", this.getStrategyID(), resourceURI, resourceUUID); final Optional resource = defaultFetcher.apply(resourceURI); if (resource.isEmpty()) { logger.warn( "StrategyID {}: application of default fetcher for {} returned empty Optional!", this.getStrategyID(), resourceURI); return Optional.empty(); } final ByteArrayResource resourceBytes; if (this.useExactResourceSize) { final long resourceLength = resource.get().length(); logger.trace("StrategyID {}: using exact resource length {}", this.getStrategyID(), resourceLength); resourceBytes = new ByteArrayResource(resourceLength); } else { logger.trace("StrategyID {}: using initial array size {}", this.getStrategyID(), this.initialArraySize); resourceBytes = new ByteArrayResource(this.initialArraySize); } resourceBytes.writeAndClose(resource.get().readBytesAndClose()); this.resourceCache.put(resourceUUID, resourceBytes); } logger.trace("StrategyID {}: returning cached resource {} from byte array keyed on UUID {}", this.getStrategyID(), resourceURI, resourceUUID); return Optional.of(this.resourceCache.get(resourceUUID)); } @Override public String getName() { return "ByteArrayCachingStrategy"; } @Override public void invalidate() { this.resourceCache.clear(); } @Override public void invalidate(final URI resourceURI) { final UUID resourceUUID = this.getUUIDForResourceURI(resourceURI); this.resourceCache.remove(resourceUUID); } /** * Use the exact resource size of the byte arrays of the cache. This may cause performance * degradation on cache misses, since some resources do not store their length as metadata. * * @return the configured {@link ByteArrayCachingStrategy} */ public ByteArrayCachingStrategy useExactResourceSize() { this.useExactResourceSize = true; return this; } /** * Set an initial array size for the byte arrays of the cache. * * @param initialSize * the initial size * @return the configured {@link ByteArrayCachingStrategy} */ public ByteArrayCachingStrategy withInitialArraySize(final long initialSize) { this.initialArraySize = initialSize; return this; } Map getResourceCache() { return this.resourceCache; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/CachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.util.Optional; import java.util.function.Function; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Interface definition for a caching strategy. A caching strategy must provide a method for * obtaining a resource based on a {@link URI}. * * @author lcram */ public interface CachingStrategy { /** * Attempt to fetch the resource located at the given URI. * * @param resourceURI * the {@link URI} if the desired {@link Resource} * @param defaultFetcher * the initial {@link Function} used to populate the cache * @return the {@link Resource} wrapped in an {@link Optional} */ Optional attemptFetch(URI resourceURI, Function> defaultFetcher); /** * Get a strategy name for logging purposes. * * @return the strategy name */ String getName(); /** * Invalidate the {@link Resource} given by the {@link URI}. The contract of this method is the * same as {@link CachingStrategy#invalidate()}, but only for the given {@link URI}. * * @param resourceURI * The {@link URI} of the {@link Resource} to invalidate */ default void invalidate(final URI resourceURI) { } /** * Invalidate the contents of this strategy. The contract of this method is the following: a * {@link URI} that produces a cache hit on an {@link CachingStrategy#attemptFetch} before an * {@link CachingStrategy#invalidate} call must produce a cache miss on the first * {@link CachingStrategy#attemptFetch} after an {@link CachingStrategy#invalidate} call. */ default void invalidate() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/GlobalNamespaceCachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.nio.file.FileSystem; import java.util.Optional; import java.util.function.Function; import org.openstreetmap.atlas.streaming.resource.Resource; /** * A special case of {@link NamespaceCachingStrategy} that uses a predefined, global namespace. This * means that all instances of {@link GlobalNamespaceCachingStrategy} will share the same underlying * contents. From this it follows that fetches and invalidates will manifest across instances. To * prevent concurrency issues, the global namespace is locked using a class lock. It is worth noting * that this still does not protect the namespace's integrity from multiple JVMs running concurrent * {@link GlobalNamespaceCachingStrategy} objects. * * @author lcram */ public class GlobalNamespaceCachingStrategy extends NamespaceCachingStrategy { /* * This is a random SHA256 hash. Collisions with this namespace are astronomically unlikely (due * to the 256 bits of entropy). */ private static final String GLOBAL_NAMESPACE = "3707740A818531237051A0F1E086CF701E2C38483675FCD1AAD8F5C5C33F19BC"; public GlobalNamespaceCachingStrategy(final FileSystem fileSystem) { super(GLOBAL_NAMESPACE, fileSystem); } public GlobalNamespaceCachingStrategy() { super(GLOBAL_NAMESPACE); } @Override public Optional attemptFetch(final URI resourceURI, final Function> defaultFetcher) { synchronized (GlobalNamespaceCachingStrategy.class) { return super.attemptFetch(resourceURI, defaultFetcher); } } @Override public String getName() { return "GlobalNamespaceCachingStrategy"; } @Override public void invalidate() { synchronized (GlobalNamespaceCachingStrategy.class) { super.invalidate(); } } @Override public void invalidate(final URI resourceURI) { synchronized (GlobalNamespaceCachingStrategy.class) { super.invalidate(resourceURI); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/NamespaceCachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Optional; import java.util.UUID; import java.util.function.Function; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.AbstractResource; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.caching.ConcurrentResourceCache; import org.openstreetmap.atlas.utilities.runtime.Retry; import org.openstreetmap.atlas.utilities.scalars.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Caching strategy that attempts to cache a {@link Resource} within a user-defined namespace at the * standard system temporary location. It should be noted that this strategy has no inherent * concurrency safety. Since the namespaces are implemented as directories in the underlying * filesystem, two {@link NamespaceCachingStrategy} objects with the same namespace can possibly * step on each other's toes if used improperly. It is up to the users of the strategy to prevent * concurrent access to {@link NamespaceCachingStrategy} objects that share a namespace. One way to * ensure concurrency safety is to carefully associate a given namespace (and its * {@link NamespaceCachingStrategy}) with exactly one {@link ConcurrentResourceCache} object * throughout your code, and stick to this restriction consistently. * * @author lcram */ public class NamespaceCachingStrategy extends AbstractCachingStrategy { private static final Logger logger = LoggerFactory.getLogger(NamespaceCachingStrategy.class); private static final String FILE_EXTENSION_DOT = "."; private static final String PROPERTY_LOCAL_TEMPORARY_DIRECTORY = "java.io.tmpdir"; private static final String TEMPORARY_DIRECTORY_STRING = System .getProperty(PROPERTY_LOCAL_TEMPORARY_DIRECTORY); private static final int RETRY_NUMBER = 5; private static final Retry RETRY = new Retry(RETRY_NUMBER, Duration.ONE_SECOND) .withQuadratic(true); private final String namespace; private boolean preserveFileExtension; private final FileSystem fileSystem; public NamespaceCachingStrategy(final String namespace) { this(namespace, FileSystems.getDefault()); } public NamespaceCachingStrategy(final String namespace, final FileSystem fileSystem) { super(); if (namespace.contains("/") || namespace.contains("\\")) { throw new IllegalArgumentException( "The namespace cannot contain characters '\\' or '/'"); } this.namespace = this.getName() + "_" + namespace + "_" + UUID.nameUUIDFromBytes(namespace.getBytes()).toString(); this.preserveFileExtension = true; this.fileSystem = fileSystem; } @Override public Optional attemptFetch(final URI resourceURI, final Function> defaultFetcher) { if (TEMPORARY_DIRECTORY_STRING == null) { logger.error("StrategyID {}: failed to read property {}, skipping cache fetch...", this.getStrategyID(), PROPERTY_LOCAL_TEMPORARY_DIRECTORY); return Optional.empty(); } if (resourceURI == null) { logger.warn("StrategyID {}: resourceURI was null, skipping cache fetch...", this.getStrategyID()); return Optional.empty(); } final File cachedFile = getCachedFile(resourceURI); attemptToCacheFileLocally(cachedFile, defaultFetcher, resourceURI); if (cachedFile.exists()) { logger.trace("StrategyID {}: returning local copy of resource {}", this.getStrategyID(), resourceURI); return Optional.of(cachedFile); } // If we got here, something went wrong in attemptToCacheFileLocally(). logger.warn("StrategyID {}: could not find local copy of resource {}", this.getStrategyID(), resourceURI); return Optional.empty(); } @Override public String getName() { return "NamespaceCachingStrategy"; } @Override public void invalidate() { final Path storageDirectory = this.getStorageDirectory(); try { new File(storageDirectory.toString(), this.fileSystem).deleteRecursively(); } catch (final Exception exception) { logger.warn("StrategyID {}: invalidate failed due to {}", this.getStrategyID(), exception.getClass().getName(), exception); } } @Override public void invalidate(final URI resourceURI) { try { getCachedFile(resourceURI).delete(); } catch (final Exception exception) { logger.warn("StrategyID {}: invalidate of resource {} failed due to {}", this.getStrategyID(), resourceURI, exception.getClass().getName(), exception); } } /** * Preserve the file extension of the cached URI when saving it as a file to the temporary * location. For example, if the URI of the resource was "hdfs://foo/bar/baz.txt", then after * computing the hash of the URI, {@link NamespaceCachingStrategy} will append a '.txt' * extension to the filename. This is useful for e.g. in cases where resource loading code may * be looking for specific file extensions in order to decide between various load strategies. * * @param preserveFileExtension * if true, preserve the original extension * @return this instance for chaining */ public NamespaceCachingStrategy withFileExtensionPreservation( final boolean preserveFileExtension) { this.preserveFileExtension = preserveFileExtension; return this; } protected void validateLocalFile(final File localFile) { // Do nothing here, leave to extensions to decide. } /* * Package-private for unit testing */ Path getStorageDirectory() { return this.fileSystem.getPath(TEMPORARY_DIRECTORY_STRING, this.namespace); } private void attemptToCacheFileLocally(final File cachedFile, final Function> defaultFetcher, final URI resourceURI) { if (!cachedFile.exists()) { logger.trace("StrategyID {}: attempting to cache resource {} in temporary file {}", this.getStrategyID(), resourceURI, cachedFile); final Optional resourceFromDefaultFetcher = defaultFetcher.apply(resourceURI); if (resourceFromDefaultFetcher.isEmpty()) { logger.warn( "StrategyID {}: application of default fetcher for {} returned empty Optional!", this.getStrategyID(), resourceURI); return; } final File temporaryLocalFile = File.temporary(this.fileSystem); RETRY.run(() -> { try { /* * We have to explicitly set the decompressor here. Why? Because if the resource * ends with a '.gz' extension, the 'copyTo' method will apply GZIP * decompression to it. The problem? When the user goes to fetch the contents of * the cached copy, it will still have the '.gz' extension but it will now be * decompressed. So our automatic decompression code will run on an uncompressed * file! This will cause the contents fetch to fail since Java's GZIPInputStream * won't be able to find the GZIP magic number! */ final AbstractResource abstractResource = (AbstractResource) resourceFromDefaultFetcher .get(); abstractResource.setDecompressor(Decompressor.NONE); abstractResource.copyTo(temporaryLocalFile); validateLocalFile(temporaryLocalFile); } catch (final Exception exception) { throw new CoreException( "StrategyID {}: something went wrong copying {} to temporary local file {}", this.getStrategyID(), resourceFromDefaultFetcher, temporaryLocalFile, exception); } }); // now that we have pulled down the file to a unique temporary location, attempt to // atomically move it to the cache after re-checking for existence if (!cachedFile.exists()) { try { final Path temporaryLocalFilePath = this.fileSystem .getPath(temporaryLocalFile.getPathString()); final Path cachedFilePath = this.fileSystem.getPath(cachedFile.getPathString()); Files.move(temporaryLocalFilePath, cachedFilePath, StandardCopyOption.ATOMIC_MOVE); validateLocalFile(cachedFile); } catch (final FileAlreadyExistsException exception) { logger.trace("StrategyID {}: file {} is already cached", this.getStrategyID(), cachedFile); } catch (final Exception exception) { throw new CoreException("StrategyID {}: something went wrong moving {} to {}", this.getStrategyID(), temporaryLocalFile, cachedFile, exception); } } } } private File getCachedFile(final URI resourceURI) { final Path storageDirectory = getStorageDirectory(); final Optional resourceExtensionOptional = getFileExtensionFromURI(resourceURI); final String cachedFileName; cachedFileName = resourceExtensionOptional .map(extension -> this.getUUIDForResourceURI(resourceURI).toString() + FILE_EXTENSION_DOT + extension) .orElseGet(() -> this.getUUIDForResourceURI(resourceURI).toString()); final Path cachedFilePath = this.fileSystem.getPath(storageDirectory.toString(), cachedFileName); return new File(cachedFilePath.toString(), this.fileSystem); } private Optional getFileExtensionFromURI(final URI resourceURI) { if (!this.preserveFileExtension) { return Optional.empty(); } final String asciiString = resourceURI.toASCIIString(); final int lastIndexOfDot = asciiString.lastIndexOf(FILE_EXTENSION_DOT); if (lastIndexOfDot < 0) { return Optional.empty(); } final String extension = asciiString.substring(lastIndexOfDot + 1); if (extension.isEmpty()) { return Optional.empty(); } else { return Optional.of(extension); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/caching/strategies/NoCachingStrategy.java ================================================ package org.openstreetmap.atlas.utilities.caching.strategies; import java.net.URI; import java.util.Optional; import java.util.function.Function; import org.openstreetmap.atlas.streaming.resource.Resource; /** * Caching strategy that always produces a cache miss. * * @author lcram */ public class NoCachingStrategy extends AbstractCachingStrategy { @Override public Optional attemptFetch(final URI resourceURI, final Function> defaultFetcher) { return Optional.empty(); } @Override public String getName() { return "NoCachingStrategy"; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/checkstyle/ArrangementCheck.java ================================================ package org.openstreetmap.atlas.utilities.checkstyle; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.StringList; import com.puppycrawl.tools.checkstyle.api.AbstractCheck; import com.puppycrawl.tools.checkstyle.api.DetailAST; import com.puppycrawl.tools.checkstyle.api.TokenTypes; /** * Check for specified ordering of elements in a source file. * * @author matthieun */ public class ArrangementCheck extends AbstractCheck { /** * @author matthieun */ private enum Type { INTERFACE(TokenTypes.INTERFACE_DEF), ENUM(TokenTypes.ENUM_DEF), CLASS(TokenTypes.CLASS_DEF), FIELD(TokenTypes.VARIABLE_DEF), STATIC_INITIALIZER_BLOCK(TokenTypes.STATIC_INIT), INITIALIZER_BLOCK(TokenTypes.INSTANCE_INIT), METHOD(TokenTypes.METHOD_DEF), CONSTRUCTOR(TokenTypes.CTOR_DEF); private final int tokenType; public static Type forName(final String name) { for (final Type type : Type.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } } throw new CoreException("Invalid name {}", name); } public static Type forType(final int tokenType) { for (final Type type : Type.values()) { if (type.tokenType == tokenType) { return type; } } throw new CoreException("Invalid token type {}", tokenType); } Type(final int tokenType) { this.tokenType = tokenType; } } /** * @author matthieun */ private enum Visibility { PUBLIC, PROTECTED, PACKAGE_PRIVATE, PRIVATE; public static Visibility forName(final String name) { for (final Visibility visibility : Visibility.values()) { if (visibility.name().equalsIgnoreCase(name)) { return visibility; } } if (name.isEmpty()) { return Visibility.PACKAGE_PRIVATE; } throw new CoreException("Invalid name \"{}\"", name); } } /** * @author matthieun */ private class ObjectRawType { private final Visibility visibility; private final Type type; private final boolean isStatic; ObjectRawType(final Visibility visibility, final Type type, final boolean isStatic) { this.visibility = visibility; this.type = type; this.isStatic = isStatic; } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || getClass() != other.getClass()) { return false; } final ObjectRawType that = (ObjectRawType) other; return isStatic() == that.isStatic() && getVisibility() == that.getVisibility() && getType() == that.getType(); } public Type getType() { return this.type; } public Visibility getVisibility() { return this.visibility; } @Override public int hashCode() { return Objects.hash(getVisibility(), getType(), isStatic()); } public boolean isStatic() { return this.isStatic; } } /** * @author matthieun */ private class ObjectType implements Comparable { private final ObjectRawType objectRawType; private final String name; ObjectType(final DetailAST object) { final Type type = Type.forType(object.getType()); final Visibility visibility; final boolean isStatic; final Optional ident = findFirstToken(object, TokenTypes.IDENT); if (ident.isPresent()) { this.name = ident.get().getText(); } else if (Type.INITIALIZER_BLOCK == type || Type.STATIC_INITIALIZER_BLOCK == type) { this.name = type.name().toLowerCase(); } else { this.name = ""; } final Optional modifiers = findFirstToken(object, TokenTypes.MODIFIERS); if (modifiers.isPresent()) { if (findFirstToken(modifiers.get(), TokenTypes.LITERAL_PRIVATE).isPresent()) { visibility = Visibility.PRIVATE; } else if (findFirstToken(modifiers.get(), TokenTypes.LITERAL_PROTECTED).isPresent()) { visibility = Visibility.PROTECTED; } else if (findFirstToken(modifiers.get(), TokenTypes.LITERAL_PUBLIC).isPresent()) { visibility = Visibility.PUBLIC; } else { visibility = Visibility.PACKAGE_PRIVATE; } isStatic = findFirstToken(modifiers.get(), TokenTypes.LITERAL_STATIC).isPresent(); } else { visibility = Visibility.PACKAGE_PRIVATE; isStatic = false; } this.objectRawType = new ObjectRawType(visibility, type, isStatic); } @Override public int compareTo(final ObjectType that) { return ArrangementCheck.this.getObjectTypeComparator().compare(this, that); } @Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null || getClass() != other.getClass()) { return false; } final ObjectType that = (ObjectType) other; return getObjectRawType().equals(that.getObjectRawType()) && getName().equals(that.getName()); } public String getName() { return this.name; } public ObjectRawType getObjectRawType() { return this.objectRawType; } public Type getType() { return this.objectRawType.getType(); } public Visibility getVisibility() { return this.objectRawType.getVisibility(); } @Override public int hashCode() { return Objects.hash(getVisibility(), getType(), isStatic(), getName()); } public boolean isStatic() { return this.objectRawType.isStatic(); } @Override public String toString() { return "ObjectType{" + "visibility=" + this.getVisibility() + ", type=" + this.getType() + ", isStatic=" + this.isStatic() + ", name='" + this.name + '\'' + '}'; } } /** * @author matthieun */ private class ObjectTypeComparator implements Comparator { private static final int ARRANGEMENT_LINE_SIZE = 3; private static final String ARRANGEMENT_FILE = "arrangement.txt"; private final Map rawTypeToOrderIndex; ObjectTypeComparator() { this(new InputStreamResource( () -> ArrangementCheck.class.getResourceAsStream(ARRANGEMENT_FILE))); } ObjectTypeComparator(final Resource ordering) { int index = 0; try { this.rawTypeToOrderIndex = new HashMap<>(); for (final String line : ordering.lines()) { final StringList split = StringList.split(line, ","); if (line.isEmpty() || line.startsWith("#")) { index++; continue; } if (split.size() != ARRANGEMENT_LINE_SIZE) { throw new CoreException("Malformed line: \"{}\"", line); } final Type type = Type.forName(split.get(0)); final Visibility visibility = Visibility.forName(split.get(1)); final boolean isStatic = "static".equalsIgnoreCase(split.get(2)); this.rawTypeToOrderIndex.put(new ObjectRawType(visibility, type, isStatic), index); index++; } } catch (final Exception e) { throw new CoreException( "Unable to parse file defining arrangement (was at line {}): {}", index + 1, ArrangementCheck.class.getResource(ARRANGEMENT_FILE).getPath(), e); } } @Override public int compare(final ObjectType left, final ObjectType right) { final int difference = this.rawTypeToOrderIndex.get(left.getObjectRawType()) - this.rawTypeToOrderIndex.get(right.getObjectRawType()); if (difference == 0 && Type.FIELD != left.getObjectRawType().getType()) { final String leftName = left.getName(); final String rightName = right.getName(); return leftName.compareTo(rightName); } else { return difference; } } public boolean isComparable(final ObjectType object) { return this.rawTypeToOrderIndex.containsKey(object.getObjectRawType()) && !"serialVersionUID".equals(object.getName()); } } private ObjectTypeComparator objectTypeComparator; private String arrangementDefinition = ""; public static Optional findFirstToken(final DetailAST source, final int tokenType) { return Optional.ofNullable(source.findFirstToken(tokenType)); } @Override public int[] getAcceptableTokens() { return getDefaultTokens(); } @Override public int[] getDefaultTokens() { return new int[] { TokenTypes.CLASS_DEF, TokenTypes.ENUM_DEF, TokenTypes.METHOD_DEF, TokenTypes.VARIABLE_DEF, TokenTypes.CTOR_DEF, TokenTypes.INTERFACE_DEF, TokenTypes.STATIC_INIT, TokenTypes.INSTANCE_INIT }; } @Override public int[] getRequiredTokens() { return getDefaultTokens(); } public void setArrangementDefinition(final String arrangementDefinition) { this.arrangementDefinition = arrangementDefinition; } @Override public void visitToken(final DetailAST object) { final ObjectType left = new ObjectType(object); if (!acceptedTokens().contains(object.getType()) || !this.getObjectTypeComparator().isComparable(left)) { return; } Optional nextSibling = Optional.ofNullable(object.getNextSibling()); while (nextSibling.isPresent() && (!acceptedTokens().contains(nextSibling.get().getType()) || !this.getObjectTypeComparator().isComparable(new ObjectType(nextSibling.get())))) { nextSibling = Optional.ofNullable(nextSibling.get().getNextSibling()); } if (nextSibling.isPresent()) { final DetailAST nextSiblingGet = nextSibling.get(); final ObjectType right = new ObjectType(nextSiblingGet); if (left.compareTo(right) > 0) { String moreInfo = ""; if (!left.getName().isEmpty()) { moreInfo = moreInfo + ", " + left.getName(); } if (!right.getName().isEmpty()) { moreInfo = moreInfo + ", " + right.getName(); } log(nextSiblingGet.getLineNo(), "Invalid order" + moreInfo); } } } private Set acceptedTokens() { final HashSet result = new HashSet<>(); for (final int value : getAcceptableTokens()) { result.add(value); } return result; } private ObjectTypeComparator getObjectTypeComparator() { if (this.objectTypeComparator == null) { if (this.arrangementDefinition.isEmpty()) { this.objectTypeComparator = new ObjectTypeComparator(); } else if (this.arrangementDefinition.startsWith("/")) { this.objectTypeComparator = new ObjectTypeComparator( new File(this.arrangementDefinition)); } else { throw new CoreException("Invalid configuration for ArrangementCheck: {}", this.arrangementDefinition); } } return this.objectTypeComparator; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/checkstyle/README.md ================================================ # ArrangementCheck This is a Checkstyle plugin that checks for member ordering in Java source files. ## Setup The default setup takes the [default ordering definition](/src/main/resources/org/openstreetmap/atlas/utilities/checkstyle/arrangement.txt). ```xml ``` or with a specified ordering definition: ```xml ``` To use this with gradle, the project needs this dependency: ```groovy dependencies { checkstyle "com.puppycrawl.tools:checkstyle:" checkstyle "org.openstreetmap.atlas:atlas:" } ``` **With Atlas 5.6.9+, the versions of checkstyle that are supported are up to `8.20`.** Checkstyle `8.21` and `8.22`+ contain [a breaking change](https://github.com/checkstyle/checkstyle/compare/checkstyle-8.20...checkstyle-8.21#diff-2ecb5f79d7dcce6cfd4fa27ef2ec99d1R29) that is not yet supported by `ArrangementCheck`. ## Ordering definition The ordering can be defined in a text file which has the following format: ``` type,visibility,static/non_static ``` with each line in order of importance. All the combinations that are not matched in that file will be ignored during processing. For example, a file like this: ``` method,public,non_static method,protected,non_static ``` will only check that non-static public methods come before non-static protected methods, and it will ignore everything else. ### Alphabetical order By default all types that are comparable according to the ordering file will also have to be alphabetically ordered. This is mandatory, and not configurable yet. ## Examples The following examples are based on the [default ordering definition](/src/main/resources/org/openstreetmap/atlas/utilities/checkstyle/arrangement.txt). ### Error: Field vs. Method ```java public class MyClass { public void method() { } private boolean field; } ``` ### Error: Visibility ```java public class MyClass { void methodA() { } public void methodB() { } } ``` ### Error: Name ```java public class MyClass { public void methodB() { } public void methodA() { } } ``` ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/AbstractHDFSOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.util.Optional; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperation; /** * Brings all of the common argument values for remote HDFS operations into a single superclass * along with permitting custom hadoop configuration settings * * @author cstaylor */ public abstract class AbstractHDFSOperation extends AbstractOperation { private Optional customConfiguration; private Optional customHostname; protected AbstractHDFSOperation() { this.customConfiguration = Optional.empty(); this.customHostname = Optional.empty(); } public AbstractHDFSOperation withConfiguration(final String configuration) { this.customConfiguration = Optional.ofNullable(configuration); return this; } public AbstractHDFSOperation withCustomHostname(final String customHostname) { this.customHostname = Optional.ofNullable(customHostname); return this; } protected String preparePath(final String input) { if (input == null || input.startsWith("hdfs://")) { return input; } if (this.customHostname.isPresent()) { return String.format("hdfs://%s%s", this.customHostname.get(), input); } return input; } protected SSHOperation prepareSSH() { final SSHOperation returnValue = this.ssh(); returnValue.addArgs("hdfs"); this.customConfiguration.ifPresent(configuration -> { returnValue.addArgs("--config", configuration); }); returnValue.addArgs("dfs"); return returnValue; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/AbstractOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperation; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Helpful base class containing some shared methods used by the various remote ssh commands * * @author cstaylor */ public abstract class AbstractOperation implements Operation { private final SSHOperation ssh; protected AbstractOperation() { this.ssh = new SSHOperation(); } @Override public AbstractOperation asUser(final String username) { this.ssh.asUser(username); return this; } @Override public AbstractOperation onHost(final String hostname) { this.ssh.onHost(hostname); return this; } @Override public AbstractOperation onPort(final int portNumber) { this.ssh.onPort(portNumber); return this; } protected String getHost() { return this.ssh.getHost(); } protected String getUser() { return this.ssh.getUser(); } protected SSHOperationResults handleResults(final SSHOperationResults results) { // We can do logging here in a central place for all of our commands return results; } protected SSHOperation ssh() { return this.ssh; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/CheckIfFileExistsOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * This command checks if a file exists on a remote server via SSH * * @author cstaylor */ public class CheckIfFileExistsOperation extends AbstractOperation { @Override public CheckIfFileExistsOperation asUser(final String username) { super.asUser(username); return this; } public boolean exists(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } this.ssh().addArgs("stat", remotePath.toString()); final SSHOperationResults results = handleResults(this.ssh().execute()); return !results.getOutput().contains("No such file or directory"); } @Override public CheckIfFileExistsOperation onHost(final String host) { super.onHost(host); return this; } @Override public CheckIfFileExistsOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/DeepLSOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Uses find on a remote system to find all files from a search path and returns them as a list of * Path objects * * @author cstaylor */ public class DeepLSOperation extends AbstractOperation { @Override public DeepLSOperation asUser(final String username) { super.asUser(username); return this; } public List list(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } this.ssh().addArgs("find", remotePath.toString(), "-type", "f"); final SSHOperationResults results = handleResults(this.ssh().execute()); if (results.getReturnValue() == 1) { throw new CoreException("Error: {}@{}:{} doesn't exist", getUser(), getHost(), remotePath); } return Arrays.asList(results.getOutput().split("\n")).stream() .map(child -> remotePath.resolve(child)).collect(Collectors.toList()); } @Override public DeepLSOperation onHost(final String host) { super.onHost(host); return this; } @Override public DeepLSOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSCatOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.util.Arrays; import java.util.stream.Collectors; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS cat command (dump contents of files in HDFS to stdout) over SSH * * @author cstaylor */ public class HDFSCatOperation extends AbstractHDFSOperation { @Override public HDFSCatOperation asUser(final String username) { super.asUser(username); return this; } public SSHOperationResults cat(final String... paths) throws InterruptedException, IOException { if (paths.length == 0) { throw new IllegalArgumentException("source can't be null"); } final String pathsAsString = Arrays.asList(paths).stream().map(this::preparePath) .collect(Collectors.joining(" ")); prepareSSH().addArgs("-cat", pathsAsString); return handleResults(this.ssh().execute()); } @Override public HDFSCatOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSCatOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } @Override public HDFSCatOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSCheckIfFileExistsOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS stat command so we can check if a file exists in a remote HDFS cluster * * @author cstaylor */ public class HDFSCheckIfFileExistsOperation extends AbstractHDFSOperation { @Override public HDFSCheckIfFileExistsOperation asUser(final String username) { super.asUser(username); return this; } public boolean exists(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } prepareSSH().addArgs("-stat", preparePath(remotePath.toString())); final SSHOperationResults results = handleResults(this.ssh().execute()); return results.getReturnValue() == STANDARD_SUCCESS_CODE; } @Override public HDFSCheckIfFileExistsOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSCheckIfFileExistsOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } @Override public HDFSCheckIfFileExistsOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSCopyOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS copy command (copy files from one area of HDFS to another) over SSH * * @author cstaylor */ public class HDFSCopyOperation extends AbstractHDFSOperation { @Override public HDFSCopyOperation asUser(final String username) { super.asUser(username); return this; } public SSHOperationResults copy(final Path source, final Path destination) throws InterruptedException, IOException { if (source == null) { throw new IllegalArgumentException("source can't be null"); } if (destination == null) { throw new IllegalArgumentException("destination can't be null"); } prepareSSH().addArgs("-cp", preparePath(source.toString()), preparePath(destination.toString())); return handleResults(this.ssh().execute()); } @Override public HDFSCopyOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSCopyOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } @Override public HDFSCopyOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSLSOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import java.util.stream.Stream; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS ls command over SSH and returns the file names, not the absolute paths * * @author cstaylor */ public class HDFSLSOperation extends AbstractHDFSOperation { @Override public HDFSLSOperation asUser(final String username) { super.asUser(username); return this; } public Stream list(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } prepareSSH().addArgs("-ls", preparePath(remotePath.toString())); final SSHOperationResults results = handleResults(this.ssh().execute()); return Stream.of(results.getOutput().split("\n")).filter(i -> i.indexOf('/') != -1) .map(line -> { final String[] pieces = line.split(" "); return pieces[pieces.length - 1]; }); } @Override public HDFSLSOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSLSOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } @Override public HDFSLSOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSMkdirOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import java.util.stream.Stream; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperation; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS mkdir command over SSH * * @author cstaylor */ public class HDFSMkdirOperation extends AbstractHDFSOperation { @Override public HDFSMkdirOperation asUser(final String username) { super.asUser(username); return this; } public boolean mkdir(final Path... remotePaths) throws InterruptedException, IOException { if (remotePaths.length == 0) { throw new IllegalArgumentException("Need at least one remote path"); } final SSHOperation operation = prepareSSH().addArgs("-mkdir", "-p"); Stream.of(remotePaths).map(Path::toString).map(this::preparePath) .forEach(operation::addArgs); final SSHOperationResults results = handleResults(this.ssh().execute()); return results.getReturnValue() == STANDARD_SUCCESS_CODE; } @Override public HDFSMkdirOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSMkdirOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } @Override public HDFSMkdirOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/HDFSPutOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Implements the HDFS put command (copy a file from the local filesystem into HDFS) over SSH * * @author cstaylor */ public class HDFSPutOperation extends AbstractHDFSOperation { @Override public HDFSPutOperation asUser(final String username) { super.asUser(username); return this; } @Override public HDFSPutOperation onHost(final String host) { super.onHost(host); return this; } @Override public HDFSPutOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } public SSHOperationResults put(final Path source, final Path destination) throws InterruptedException, IOException { if (source == null) { throw new IllegalArgumentException("source can't be null"); } if (destination == null) { throw new IllegalArgumentException("destination can't be null"); } prepareSSH().addArgs("-put", preparePath(source.toString()), preparePath(destination.toString())); return handleResults(this.ssh().execute()); } @Override public HDFSPutOperation withConfiguration(final String configuration) { super.withConfiguration(configuration); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/LSOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Returns the list of filenames from a directory on a remote server over SSH * * @author cstaylor */ public class LSOperation extends AbstractOperation { @Override public LSOperation asUser(final String username) { super.asUser(username); return this; } public List list(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } this.ssh().addArgs("ls", "-1", remotePath.toString()); final SSHOperationResults results = handleResults(this.ssh().execute()); return Arrays.asList(results.getOutput().split("\n")); } @Override public LSOperation onHost(final String host) { super.onHost(host); return this; } @Override public LSOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/MkdirOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Stream; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * Creates one or more directories on a remote system over SSH * * @author cstaylor */ public class MkdirOperation extends AbstractOperation { private Optional>> errorHandler = Optional.empty(); @Override public MkdirOperation asUser(final String username) { super.asUser(username); return this; } public boolean mkdir(final Path... remotePaths) throws InterruptedException, IOException { if (remotePaths.length == 0) { throw new IllegalArgumentException("Need at least one remote path"); } this.ssh().addArgs("mkdir", "-p"); Stream.of(remotePaths).map(Path::toString).forEach(item -> this.ssh().addArgs(item)); final SSHOperationResults results = handleResults(this.ssh().execute()); if (results.getReturnValue() != STANDARD_SUCCESS_CODE) { this.errorHandler.ifPresent(handler -> { Stream.of(results.getOutput().split("\n")).map(line -> StringList.split(line, ":")) .forEach(stringList -> { handler.accept( new Tuple<>(Paths.get(stringList.get(1)), stringList.get(2))); }); }); } return results.getReturnValue() == STANDARD_SUCCESS_CODE; } @Override public MkdirOperation onHost(final String host) { super.onHost(host); return this; } @Override public MkdirOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } public MkdirOperation withErrorHandler(final Consumer> errorHandler) { this.errorHandler = Optional.ofNullable(errorHandler); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/Operation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; /** * All commands must have a remote user on a particular host * * @author cstaylor */ public interface Operation { int STANDARD_SUCCESS_CODE = 0; /** * @param username * the SSH username for connecting with the remote server * @return fluent interface returns this */ Operation asUser(String username); /** * @param hostname * the hostname or IP of the remote server * @return fluent interface returns this */ Operation onHost(String hostname); /** * @param portNumber * the non-standard SSH port we should use * @return fluent interface returns this */ Operation onPort(int portNumber); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/RMDirOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations; import java.io.IOException; import java.nio.file.Path; import org.openstreetmap.atlas.utilities.cli.operations.base.SSHOperationResults; /** * Removes a directory from a remote server over SSH. *

* See that rm -rf? Be very careful... * * @author cstaylor */ public class RMDirOperation extends AbstractOperation { @Override public RMDirOperation asUser(final String username) { super.asUser(username); return this; } @Override public RMDirOperation onHost(final String host) { super.onHost(host); return this; } @Override public RMDirOperation onPort(final int portNumber) { super.onPort(portNumber); return this; } public boolean rmdir(final Path remotePath) throws InterruptedException, IOException { if (remotePath == null) { throw new IllegalArgumentException("remotePath can't be null"); } /** * Not the best sanity check in the world. */ if (remotePath.toString().contains("*")) { throw new IllegalArgumentException("Please, be careful with the asterisks"); } this.ssh().addArgs("rm", "-rf", remotePath.toString()); final SSHOperationResults results = handleResults(this.ssh().execute()); return results.getReturnValue() == STANDARD_SUCCESS_CODE; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/AvailableSocketFinder.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.io.IOException; import java.net.ServerSocket; import org.openstreetmap.atlas.exception.CoreException; /** * Simple utility class that will try and find a socket to bind to and return that port number after * the socket has been closed * * @author cstaylor */ final class AvailableSocketFinder { static int takePort() { Integer returnValue = null; try { final ServerSocket socket = new ServerSocket(0); returnValue = socket.getLocalPort(); socket.close(); } catch (final IOException oops) { throw new CoreException("Error when trying to reserve a port", oops); } return returnValue; } private AvailableSocketFinder() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/OperationResults.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.time.Duration; /** * Shared methods of both SSH and SCP operation results. * * @author cstaylor */ public interface OperationResults { /** * @return how long did it take to run? */ Duration getElapsedTime(); /** * @return Was there any output on stderr or stdout from the remote command? */ String getOutput(); /** * @return The exit code returned by the remote command */ int getReturnValue(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/RemoteObject.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.nio.file.Path; /** * When making a transfer call via SCP, a RemoteObject is something to read or write from a remote * machine. * * @author cstaylor */ public class RemoteObject { /** * SSH connect string */ private final String connectString; public RemoteObject(final String username, final String hostname, final Path path) { this.connectString = String.format("%s@%s:%s", username, hostname, path.toString()); } /** * @return a valid parameter for a remote file resource (username@hostname:path) to be used with * SCP */ @Override public String toString() { return this.connectString; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/SCPOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.apache.commons.io.IOUtils; /** * Wrapper for making a remote call via the command-line scp utility * * @author cstaylor */ public class SCPOperation { private static final String QUIET_ARG = "-q"; private static final String SCP_COMMAND = "scp"; private static final String PORT_OVERRIDE = "-P"; private Optional possiblePortNumber = Optional.empty(); /** * Send locally to remote with scp * * @param fromLocal * the local file to send * @param toRemote * the remote file saved * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem or the local file can't be read * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ public SCPOperationResults copy(final Path fromLocal, final RemoteObject toRemote) throws IOException, InterruptedException { return copy(fromLocal.toString(), toRemote.toString()); } /** * Read remote file and save it locally * * @param fromRemote * the remote file to read * @param toLocal * the local file saved * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ public SCPOperationResults copy(final RemoteObject fromRemote, final Path toLocal) throws IOException, InterruptedException { return copy(fromRemote.toString(), toLocal.toString()); } /** * Send remote file to another remote file * * @param fromRemote * the remote file to read * @param toRemote * the remote file saved * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ public SCPOperationResults copy(final RemoteObject fromRemote, final RemoteObject toRemote) throws IOException, InterruptedException { return copy(fromRemote.toString(), toRemote.toString()); } public SCPOperation onPort(final int portNumber) { this.possiblePortNumber = Optional.of(portNumber); return this; } /** * The helper method that does the actual call to SCP * * @param fromResource * the local or remote file to be read * @param toResource * the local or remote file that will be saved * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem or the local file can't be read or saved * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ private SCPOperationResults copy(final String fromResource, final String toResource) throws IOException, InterruptedException { final List args = new ArrayList<>(); args.add(SCP_COMMAND); args.add(QUIET_ARG); this.possiblePortNumber.ifPresent(portNumber -> { args.add(PORT_OVERRIDE); args.add(String.valueOf(portNumber)); }); args.add(fromResource); args.add(toResource); final ProcessBuilder builder = new ProcessBuilder(args); builder.redirectErrorStream(true); final SCPOperationResults results = new SCPOperationResults(fromResource, toResource); final Process process = builder.start(); final String remoteOutput = new String(IOUtils.toByteArray(process.getInputStream())); final int returnCode = process.waitFor(); return results.finish(remoteOutput, returnCode); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/SCPOperationResults.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.time.Duration; import java.time.temporal.ChronoUnit; /** * When an scp command is run, the output from the command is captured by this class, including: *

    *
  • stdout/stderr output
    • *
  • duration of call
    • *
  • source file (local or remote) sent
    • *
  • destination file (local or remote) written
    • *
* * @see SCPOperation * @see OperationResults * @author cstaylor */ public class SCPOperationResults implements OperationResults { private final String destination; private long end = -1; private String output; private int returnValue; private final String source; private long start = -1; /** * @param source * the local or remote file being read * @param destination * the local or remote file being written */ SCPOperationResults(final String source, final String destination) { this.start = System.currentTimeMillis(); this.source = source; this.destination = destination; } /** * @return the local or remote file being written */ public String getDestination() { return this.destination; } @Override public Duration getElapsedTime() { if (this.end == -1) { this.end = System.currentTimeMillis(); } return Duration.of(this.end - this.start, ChronoUnit.MILLIS); } @Override public String getOutput() { return this.output; } @Override public int getReturnValue() { return this.returnValue; } /** * @return the local or remote file being read */ public String getSource() { return this.source; } @Override public String toString() { return String.format("%d\n%s\n", getReturnValue(), getOutput()); } /** * When the scp operation is completed, this method should be called so the duration is * recorded, the stdout/stderr output captured, and the remote return code saved * * @param output * possible stdout/stderr from the scp process * @param returnCode * standard unix exit code. See man scp for details * @return fluent interface returns this */ SCPOperationResults finish(final String output, final int returnCode) { this.output = output; this.returnValue = returnCode; getElapsedTime(); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/SSHForwarder.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.openstreetmap.atlas.exception.CoreException; /** * This class spawns a child process that will connect via SSH to a remote host that will act as a * packet forwarder for us. *

* We need this to bypass some of the firewall restrictions. * * @author cstaylor */ public class SSHForwarder { private static final int DEFAULT_SSH_PORT = 22; private static final String LOGIN_FORMAT = "%s@%s"; private static final String PROXY_FORMAT = "%d:%s:%d"; private static final String FORWARD_CREDENTIALS = "-A"; private static final String PORT_MAPPING = "-L"; private static final String DISABLE_STRICT_HOST_CHECKING = "-oStrictHostKeyChecking=no"; private static final String SSH_COMMAND = "ssh"; private static final String CAT_COMMAND = "cat"; private static final String READ_FROM_STDIN = "-"; private static final int SSH_OPERATION_FAILURE_CODE = 255; private String hostname; private String username; private int forwardingLocalPort = -1; private int forwardingRemotePort = DEFAULT_SSH_PORT; private String forwardingToHostname; private Process remoteConnection; public SSHForwarder() { } /** * @param username * the SSH username for connecting with the remote server * @return fluent interface returns this */ public SSHForwarder asUser(final String username) { this.username = username; return this; } public String getHost() { return this.hostname; } public String getUser() { return this.username; } public SSHForwarder onHost(final String hostname) { this.hostname = hostname; return this; } /** * Connects via ssh to the remote server and holds the connection open * * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ public int startProxy() throws IOException, InterruptedException { if (this.hostname == null) { throw new IllegalStateException("Hostname must be defined"); } if (this.username == null) { this.username = System.getProperty("user.name"); } if (this.forwardingToHostname == null) { throw new IllegalStateException("forwardingToHostname must be defined"); } if (this.forwardingLocalPort < 0) { this.forwardingLocalPort = AvailableSocketFinder.takePort(); } final List arguments = new ArrayList<>(); arguments.add(SSH_COMMAND); arguments.add(FORWARD_CREDENTIALS); arguments.add(DISABLE_STRICT_HOST_CHECKING); arguments.add(PORT_MAPPING); arguments.add(String.format(PROXY_FORMAT, this.forwardingLocalPort, this.forwardingToHostname, this.forwardingRemotePort)); arguments.add(String.format(LOGIN_FORMAT, this.username, this.hostname)); arguments.add(CAT_COMMAND); arguments.add(READ_FROM_STDIN); final ProcessBuilder builder = new ProcessBuilder(arguments); builder.redirectErrorStream(true); // We need to check for errors this.remoteConnection = builder.start(); if (this.remoteConnection.waitFor(1L, TimeUnit.SECONDS) && this.remoteConnection.exitValue() == SSH_OPERATION_FAILURE_CODE) { final String remoteOutput = new String( IOUtils.toByteArray(this.remoteConnection.getInputStream())); throw new CoreException("Error when connecting to proxy: {}", remoteOutput); } return this.forwardingLocalPort; } public void stopProxy() throws IOException, InterruptedException { this.remoteConnection.destroyForcibly(); this.remoteConnection.waitFor(); } public SSHForwarder withForwardingHostname(final String hostname) { this.forwardingToHostname = hostname; return this; } public SSHForwarder withForwardingLocalPort(final int port) { this.forwardingLocalPort = port; return this; } public SSHForwarder withForwardingRemotePort(final int port) { this.forwardingRemotePort = port; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/SSHOperation.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Joiner; /** * Wrapper for running a remote program with the local ssh utility * * @author cstaylor */ public class SSHOperation { private static final Logger logger = LoggerFactory.getLogger(SSHOperation.class); private static final String LOGIN_FORMAT = "%s@%s"; private static final String SSH_COMMAND = "ssh"; private static final String DISABLE_STRICT_HOST_CHECKING = "-oStrictHostKeyChecking=no"; private static final String QUIET_MODE = "-q"; private static final String PORT_OVERRIDE = "-p"; private final List args; private String hostname; private String username; private Optional possiblePort; private boolean debug; public SSHOperation() { this.args = new ArrayList<>(); this.possiblePort = Optional.empty(); } /** * Add arguments that will be sent to the remote server via SSH * * @param args * remote linux commands and their arguments * @return fluent interface returns this */ public SSHOperation addArgs(final String... args) { if (args == null || args.length == 0) { return this; } this.args.addAll(Arrays.asList(args)); return this; } /** * @param username * the SSH username for connecting with the remote server * @return fluent interface returns this */ public SSHOperation asUser(final String username) { this.username = username; return this; } public SSHOperation enableDebug() { this.debug = true; return this; } /** * Connects via ssh to the remote server and executes the linux command * * @return the results (error code, stdout/stderr output, timing) * @throws IOException * if there's a network problem * @throws InterruptedException * if our thread is interrupted while waiting for the remote command to finish */ public SSHOperationResults execute() throws IOException, InterruptedException { if (this.hostname == null) { throw new IllegalStateException("Hostname must be defined"); } if (this.username == null) { this.username = System.getProperty("user.name"); } final List arguments = buildArguments(); final ProcessBuilder builder = new ProcessBuilder(arguments); if (this.debug) { logger.debug(Joiner.on(" ").join(arguments)); } builder.redirectErrorStream(true); final SSHOperationResults results = new SSHOperationResults(); final Process process = builder.start(); final String remoteOutput = new String(IOUtils.toByteArray(process.getInputStream())); final int returnCode = process.waitFor(); if (this.debug) { logger.debug("[{}] with output:\n{}", returnCode, remoteOutput); } return results.finish(remoteOutput, returnCode); } public String getHost() { return this.hostname; } public String getUser() { return this.username; } public SSHOperation onHost(final String hostname) { this.hostname = hostname; return this; } public SSHOperation onPort(final int port) { this.possiblePort = Optional.of(port); return this; } private List buildArguments() { if (this.args.size() == 0) { throw new IllegalStateException("You must have at least one argument"); } final List arguments = new ArrayList<>(); arguments.add(SSH_COMMAND); arguments.add(DISABLE_STRICT_HOST_CHECKING); arguments.add(QUIET_MODE); this.possiblePort.ifPresent(port -> { arguments.add(PORT_OVERRIDE); arguments.add(String.valueOf(port)); }); arguments.add(String.format(LOGIN_FORMAT, this.username, this.hostname)); arguments.addAll(this.args); return arguments; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/cli/operations/base/SSHOperationResults.java ================================================ package org.openstreetmap.atlas.utilities.cli.operations.base; import java.time.Duration; import java.time.temporal.ChronoUnit; /** * When an ssh command is run, the output from the command is captured by this class, including: *

    *
  • stdout/stderr output
    • *
  • duration of call
    • *
* * @see SSHOperation * @see OperationResults * @author cstaylor */ public class SSHOperationResults implements OperationResults { private long end = -1; private String output; private int returnValue; private long start = -1; public SSHOperationResults() { this.start = System.currentTimeMillis(); } @Override public Duration getElapsedTime() { if (this.end == -1) { this.end = System.currentTimeMillis(); } return Duration.of(this.end - this.start, ChronoUnit.MILLIS); } @Override public String getOutput() { return this.output; } @Override public int getReturnValue() { return this.returnValue; } /** * When the ssh operation is completed, this method should be called so the duration is * recorded, the stdout/stderr output captured, and the remote return code saved * * @param output * possible stdout/stderr from the scp process * @param returnValue * standard unix exit code. See man scp for details * @return fluent interface returns this */ protected SSHOperationResults finish(final String output, final int returnValue) { this.output = output; this.returnValue = returnValue; getElapsedTime(); return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/EnhancedCollectors.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.collect.ImmutableList; /** * Shamelessly stolen from: * http://stackoverflow.com/questions/29090277/java-8-streams-collections-tomap-from-list-how-to- * keep-the-order and from: * https://stackoverflow.com/questions/39130122/java-8-nested-multi-level-group-by/39131049#39131049 * * @author cstaylor * @author mgostintsev */ public final class EnhancedCollectors { public static Collector flatMapping( final Function> mapper, final Collector downstream) { final BiConsumer accumulator = downstream.accumulator(); return Collector.of(downstream.supplier(), (itemA, itemT) -> { try (Stream s = mapper.apply(itemT)) { if (s != null) { s.forEachOrdered(u -> accumulator.accept(itemA, u)); } } }, downstream.combiner(), downstream.finisher(), downstream.characteristics().stream().toArray(Collector.Characteristics[]::new)); } public static > Collector> toImmutableList() { return new ImmutableListCollector<>(); } /** * I wanted a way of quickly converting lists to linked hashmaps, so I found this little block * of code in Stack Overflow * * @param * the type in the incoming list * @param * the key type of the outgoing map * @param * the value type of the outgoing map * @param keyMapper * how we get the key into the map * @param valueMapper * how we get the value into the map * @return the linked hashmap */ public static Collector> toLinkedMap( final Function keyMapper, final Function valueMapper) { return Collectors.toMap(keyMapper, valueMapper, (key, value) -> { throw new IllegalStateException(String.format("Duplicate key %s", key)); }, LinkedHashMap::new); } public static , U> Collector> toUnmodifiableSortedMap( final Function keyMapper, final Function valueMapper) { return new UnmodifiableSortedMapCollector<>(keyMapper, valueMapper); } public static > Collector> toUnmodifiableSortedSet() { return new UnmodifiableSortedSetCollector<>(); } private EnhancedCollectors() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/EnumSetCollector.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.TypeVariable; import java.util.EnumSet; import java.util.HashSet; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; import org.openstreetmap.atlas.exception.CoreException; import com.google.common.reflect.TypeToken; /** * A way to build an EnumSet from a stream of strings, where the case of the strings don't matter * and ALL will automatically add all declared enum constants to the EnumSet. Since we're using * generic types, we have some issues where the parameterized type information is only contained in * concrete subclasses of this collector. That means for each Enum we want to collect, we'll need an * empty subclass so we can recover the type information. For a testcase showing how to make it * work, check out EnumSetCollectionTestCase * * @author cstaylor * @param * The Java enum to store in an EnumSet that can be queried for values */ public abstract class EnumSetCollector> implements Collector, EnumSet> { @SuppressWarnings("rawtypes") private Class enumClass; private Method valueOfMethod; @SuppressWarnings({ "rawtypes", "unchecked" }) /** * We need to know the enum's class, but we can't get at it directly because we're a generic * interface. Following directions listed out in this article: * http://stackoverflow.com/questions/3609799/how-to-get-type-parameter-values-using-java- * reflection */ protected EnumSetCollector() { final TypeToken resolver = TypeToken.of(getClass()); for (final TypeVariable> typeVariable : EnumSetCollector.class .getTypeParameters()) { final TypeToken currentToken = resolver.resolveType(typeVariable); this.enumClass = currentToken.getRawType(); try { this.valueOfMethod = this.enumClass.getMethod("valueOf", String.class); } catch (final NoSuchMethodException oops) { throw new CoreException( String.format("%s isn't a Java enum", this.enumClass.getName())); } } } @Override public BiConsumer, String> accumulator() { return (enumset, value) -> enumset.add(value.toUpperCase()); } @Override public Set characteristics() { return EnumSet.of(Characteristics.UNORDERED); } @Override public BinaryOperator> combiner() { return (left, right) -> { left.addAll(right); return left; }; } @SuppressWarnings({ "unchecked" }) @Override public Function, EnumSet> finisher() { return working -> { if (working.contains("ALL")) { return EnumSet.allOf(this.enumClass); } try { final EnumSet returnValue = EnumSet.noneOf(this.enumClass); for (final String constant : working) { returnValue.add((T) this.valueOfMethod.invoke(null, constant)); } return returnValue; } catch (final InvocationTargetException | IllegalAccessException oops) { throw new CoreException("Can't find enum value for: {}", working); } }; } @Override public Supplier> supplier() { return this.enumClass == null ? null : () -> new HashSet<>(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/FilteredIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Iterator; import java.util.Set; import java.util.function.Function; /** * Takes an iterable and adds in a set of elements to skip over. Useful for when an iterable will be * iterated over multiple times, but some elements can be skipped for efficiency. * * @author samuelgass * @param * the type of the {@link Iterable} * @param * the type for the identifier used by the elements of Type for the {@link Iterable} */ public class FilteredIterable implements Iterable { private final Iterable source; private final Set filterSet; private final Function identifier; /** * Constructor for FilteredIterable. * * @param source * A source iterable to translate to FilteredIterable * @param filterSet * A set of identifiers for elements to skip (can be empty or have members) * @param identifier * A function that takes an element of Type for the {@link Iterable} and returns the * identifier for that element */ public FilteredIterable(final Iterable source, final Set filterSet, final Function identifier) { this.filterSet = filterSet; this.source = source; this.identifier = identifier; } /** * Takes an element and uses the identifier function to add its identifier to the filter set. * * @param type * The element to add to the filter set * @return True if an element was added to the filter set, false if it wasn't (likely in the * case it was already present in the set) */ public boolean addToFilteredSet(final Type type) { return this.filterSet.add(this.identifier.apply(type)); } @Override public Iterator iterator() { return new Iterator() { private final Iterator sourceIterator = FilteredIterable.this.source.iterator(); private Type next = null; private Type current = this.next(); @Override public boolean hasNext() { return this.next != null && !FilteredIterable.this.filterSet .contains(FilteredIterable.this.identifier.apply(this.next)); } @Override public Type next() { this.current = this.next; this.next = null; while (this.sourceIterator.hasNext()) { final Type nextCandidate = this.sourceIterator.next(); if (FilteredIterable.this.filterSet .contains(FilteredIterable.this.identifier.apply(nextCandidate))) { continue; } else { this.next = nextCandidate; break; } } return this.current; } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/FixedSizePriorityQueue.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.AbstractQueue; import java.util.Comparator; import java.util.Iterator; import java.util.PriorityQueue; /** * A fixed size priority queue (binary heap) for use case like getting top n, implemented based on * {@link PriorityQueue}. *

* After reaching the given maximum size, any additional element offered to the queue will be added * into the queue first and then remove the new head, so the size of the queue will remain the same * * @param * The type of element * @author tony */ public class FixedSizePriorityQueue extends AbstractQueue { private final int maximumSize; private final PriorityQueue priorityQueue; public FixedSizePriorityQueue(final int maximumSize) { this.maximumSize = maximumSize; this.priorityQueue = new PriorityQueue<>(maximumSize + 1); } public FixedSizePriorityQueue(final int maximumSize, final Comparator comparator) { this.maximumSize = maximumSize; this.priorityQueue = new PriorityQueue<>(maximumSize + 1, comparator); } @Override public Iterator iterator() { return this.priorityQueue.iterator(); } @Override public boolean offer(final E e) { final boolean flag = this.priorityQueue.offer(e); if (this.priorityQueue.size() > this.maximumSize) { poll(); } return flag; } @Override public E peek() { return this.priorityQueue.peek(); } @Override public E poll() { return this.priorityQueue.poll(); } @Override public int size() { return this.priorityQueue.size(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/ImmutableListCollector.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; /** * Converts a stream of objects into an {@link ImmutableList} * * @author mgostintsev * @param * the type of incoming objects we want in the {@link ImmutableList} */ public class ImmutableListCollector> implements Collector, ImmutableList> { @Override public BiConsumer, T> accumulator() { return (builder, item) -> builder.add(item); } @Override public Set characteristics() { return ImmutableSet.of(Characteristics.UNORDERED); } @Override public BinaryOperator> combiner() { return (builder1, builder2) -> { builder1.addAll(builder2.build()); return builder1; }; } @Override public Function, ImmutableList> finisher() { return builder -> builder.build(); } @Override public Supplier> supplier() { return ImmutableList::builder; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/Iterables.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Function; import java.util.function.LongFunction; import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.StreamSupport; /** * Iterable utility methods * * @author matthieun */ public final class Iterables { /** * Adds the contents of the from Iterable to the addHere collection, and returns true if items * were added. It's possible that nothing has changed if addHere is a set * * @param addHere * where to add the items * @param from * where to get the items * @param * what kind of objects we're copying * @return true if addHere has changed, false otherwise */ public static boolean addAll(final Collection addHere, final Iterable from) { final int oldSize = addHere.size(); StreamSupport.stream(from.spliterator(), false).forEach(addHere::add); return oldSize < addHere.size(); } /** * Strip down any {@link Iterable} into .. just an {@link Iterable}. * * @param types * The {@link Iterable} to strip down * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static Iterable asIterable(final Iterable types) { return types::iterator; } /** * Translate an {@link Iterable} into a {@link List} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static List asList(final Iterable types) { if (types instanceof List) { return (List) types; } final int initialSize; if (types instanceof Collection) { initialSize = ((Collection) types).size(); } else { initialSize = 0; } final List result = new ArrayList<>(initialSize); types.forEach(result::add); return result; } /** * Translate an array to an {@link List}. Unlike {@link java.util.Arrays#asList}, this method * does not use the given array as the backing array. * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static List asList(final T[] types) { // This avoids several grow calls, and ensures that we aren't using the backing array. return new ArrayList<>(Arrays.asList(types)); } /** * Translate an iterable list of Map entries to a map * * @param types * The {@link Iterable} to translate * @param * The type of key of the entry * @param * The type of value of the entry * @return The translated {@link Iterable} */ public static Map asMap(final Iterable> types) { final Map result = new HashMap<>(); types.forEach(entry -> result.put(entry.getKey(), entry.getValue())); return result; } /** * Translate an {@link Iterable} into a {@link Queue} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static Queue asQueue(final Iterable types) { final Queue result = new LinkedList<>(); types.forEach(result::add); return result; } /** * Translate an {@link Iterable} into a {@link Set} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static Set asSet(final Iterable types) { final Set result = new HashSet<>(); types.forEach(result::add); return result; } /** * Translate an array to an {@link Set} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static Set asSet(final T[] types) { final Set result = new HashSet<>(); for (final T type : types) { result.add(type); } return result; } /** * Translate an {@link Iterable} of items into a {@link SortedSet} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static SortedSet asSortedSet(final Iterable types) { final SortedSet result = new TreeSet<>(); types.forEach(result::add); return result; } /** * Test if an {@link Iterable} iterates at some point on an item. * * @param types * The {@link Iterable} to test * @param type * The item to test * @param * The type of the {@link Iterable} * @return True if the {@link Iterable} iterates at some point on the item. */ public static boolean contains(final Iterable types, final T type) { if (types instanceof Collection) { return ((Collection) types).contains(type); } for (final T candidate : types) { if (candidate.equals(type)) { return true; } } return false; } /** * Count a set of values * * @param types * The {@link Iterable} of input type * @param typeCounter * The function from type to count * @param * The type of the {@link Iterable} * @return The total count */ public static long count(final Iterable types, final ToLongFunction typeCounter) { long result = 0; for (final T type : types) { result += typeCounter.applyAsLong(type); } return result; } /** * @param example * A random object to specify the type * @param * The type of the {@link Iterable} * @return An empty {@link Iterable} of the right type */ public static Iterable emptyIterable(final T example) // NOSONAR { return () -> new Iterator() { @Override public boolean hasNext() { return false; } @Override public T next() { throw new NoSuchElementException(); } }; } /** * Test if two {@link Iterable}s iterate on the same items. * * @param that * The first {@link Iterable} * @param other * The second iterable * @param * The type of the {@link Iterable} * @return True if the two {@link Iterable}s iterate on the same items. */ public static boolean equals(final Iterable that, final Iterable other) { // Handle null iterables // If they are both null, then equal // If only one of them is null, then NOT equal final boolean thatIsNull = that == null; final boolean otherIsNull = other == null; if (thatIsNull || otherIsNull) { return thatIsNull && otherIsNull; } // Iterables are not null, let's check for size first // Then the values final long thatSize = Iterables.size(that); if (thatSize != Iterables.size(other)) { return false; } final Iterator thatIterator = that.iterator(); final Iterator otherIterator = other.iterator(); while (thatIterator.hasNext()) { if (!thatIterator.next().equals(otherIterator.next())) { return false; } } return true; } /** * Filter an {@link Iterable} * * @param input * The {@link Iterable} to filter * @param matcher * The {@link Predicate} used to filter * @param * The type of the {@link Iterable} * @return The filtered {@link Iterable} */ public static Iterable filter(final Iterable input, final Predicate matcher) { return filterTranslate(input, item -> item, matcher); } /** * Translate an {@link Iterable} of items into a {@link FilteredIterable} * * @param types * The {@link Iterable} to translate * @param filterSet * A set of identifiers for elements to skip (can be empty or have members) * @param identifier * A function that takes an element of T for the {@link Iterable} and returns the * identifier for that element * @param * The type of the {@link Iterable} * @param * The type of the Identifier object for the elements in the {@link Iterable} * @return The translated {@link Iterable} */ public static FilteredIterable filter(final Iterable types, final Set filterSet, final Function identifier) { return new FilteredIterable<>(types, filterSet, identifier); } /** * Translate an {@link Iterable} of type I to an {@link Iterable} of type O. * * @param input * The input {@link Iterable} * @param converter * The converter from I to O * @param matcher * A {@link Predicate} on I that filters only the items to match * @param * The type of the input {@link Iterable} * @param * The type of the output {@link Iterable} * @return The {@link Iterable} of O */ public static Iterable filterTranslate(final Iterable input, final Function converter, final Predicate matcher) { return new Iterable() { @Override public Iterator iterator() { return new Iterator() { private boolean consumed = true; private final Iterator iterator = input.iterator(); private I next = null; private boolean valid = false; @Override public boolean hasNext() { if (this.consumed) { this.next = null; this.valid = false; while (this.iterator.hasNext() && !this.valid) { this.next = this.iterator.next(); this.valid = matcher.test(this.next); } this.consumed = false; } return this.valid; } @Override public O next() { if (hasNext()) { this.consumed = true; return converter.apply(this.next); } throw new NoSuchElementException(); } }; } @SuppressWarnings("unused") public void useless() { // Unused } }; } /** * Get the first element of an {@link Iterable} * * @param types * The items * @param * The type of the {@link Iterable} * @return The first element in the {@link Iterable}, or empty if none. */ public static Optional first(final Iterable types) { return nth(types, 0); } public static Optional firstMatching(final Iterable types, final Predicate matcher) { return first(filter(types, matcher)); } /** * Create an {@link Iterable} from an {@link Enumeration} * * @param types * The {@link Enumeration} * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ public static Iterable from(final Enumeration types) { return () -> new Iterator() { @Override public boolean hasNext() { return types.hasMoreElements(); } @Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } return types.nextElement(); } }; } /** * Create an {@link Iterable} from 0 to many items of the provided type * * @param types * The 0 to many array of items to include * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ @SafeVarargs public static Iterable from(final T... types) { return asList(types); } /** * Get the head (first) element of an {@link Iterable} * * @param types * The items * @param * The type of the {@link Iterable} * @return The head element in the {@link Iterable}, or null if none. */ public static T head(final Iterable types) { final Iterator iterator = types.iterator(); return iterator.hasNext() ? iterator.next() : null; } /** * Get an {@link Iterable} based on something that can return the value at a specific index. * * @param size * The total size of the collection * @param supplier * The provider of the value based on the index * @param * The type to return within the {@link Iterable} * @return The index based {@link Iterable} */ public static Iterable indexBasedIterable(final long size, final LongFunction supplier) { return () -> new Iterator() { private long index = 0L; @Override public boolean hasNext() { return this.index < size; } @Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } return supplier.apply(this.index++); } }; } /** * Determines if the given iterable is empty * * @param types * The iterable to check * @return {@code true} if the iterable contains no elements */ public static boolean isEmpty(final Iterable types) { if (types instanceof Collection) { return ((Collection) types).isEmpty(); } return !types.iterator().hasNext(); } /** * Translate a passed array of Items to an {@link Iterable} of Items * * @param types * The items * @param * The type of the {@link Iterable} * @return An {@link Iterable} of items. */ public static Iterable iterable(@SuppressWarnings("unchecked") final T... types) { return indexBasedIterable(types.length, index -> types[(int) index]); } /** * Build an new Iterable by prepending the head element to the tail iterable. * * @param head * The item to place in the head position * @param tail * The items positioned after the head * @param * The type of the head and tail {@link Iterable} * @return An {@link Iterable} */ public static Iterable join(final T head, final Iterable tail) { return () -> new Iterator() { private final Iterator tailIterator = tail.iterator(); private boolean headConsumed = false; @Override public boolean hasNext() { return !this.headConsumed || this.tailIterator.hasNext(); } @Override public T next() { if (this.headConsumed) { return this.tailIterator.next(); } this.headConsumed = true; return head; } }; } /** * Get the last element of an {@link Iterable} * * @param types * The items * @param * The type of the {@link Iterable} * @return The last element in the {@link Iterable} */ public static Optional last(final Iterable types) { T result = null; if (types instanceof List) { final List list = (List) types; if (!list.isEmpty()) { result = list.get(list.size() - 1); } } else { for (final T type : types) { result = type; } } return Optional.ofNullable(result); } public static Optional lastMatching(final Iterable types, final Predicate matcher) { return last(filter(types, matcher)); } /** * Get the nth element of an {@link Iterable} * * @param types * The items * @param index * The index at which to pick * @param * The type of the {@link Iterable} * @return The first element in the {@link Iterable}, or empty if the iterable has no element at * this index. */ public static Optional nth(final Iterable types, final long index) { long counter = 0L; final Iterator iterator = types.iterator(); T result = iterator.hasNext() ? iterator.next() : null; while (counter++ < index) { if (iterator.hasNext()) { result = iterator.next(); } else { result = null; break; } } return Optional.ofNullable(result); } /** * Create a {@link StreamIterable} that uses parallelization * * @param source * The {@link Iterable} to use as source * @param * The type of the source {@link Iterable} * @return The corresponding {@link StreamIterable} */ public static StreamIterable parallelStream(final Iterable source) { return new StreamIterable<>(source, true); } public static void print(final Iterable input, final String name) { System.out.println(toString(input, name)); // NOSONAR } /** * Iterate over an {@link Iterable} to get its size. If the {@link Iterable} is a sub instance * of {@link Collection}, then it reads the size from it directly; it will not iterate * unnecessarily. * * @param * The type of the {@link Iterable} * @param types * The input {@link Iterable} * @return The size of the {@link Iterable} */ public static long size(final Iterable types) { if (types instanceof Collection) { return ((Collection) types).size(); } return count(types, type -> 1L); } /** * Create a {@link StreamIterable} * * @param source * The {@link Iterable} to use as source * @param * The type of the source {@link Iterable} * @return The corresponding {@link StreamIterable} */ public static StreamIterable stream(final Iterable source) { return new StreamIterable<>(source); } /** * Get an {@link Iterable} of all elements beyond the head. * * @param types * The items * @param * The type of the {@link Iterable} * @return An {@link Iterable} */ public static Iterable tail(final Iterable types) { final Iterator iterator = types.iterator(); if (iterator.hasNext()) { iterator.next(); } return () -> iterator; } /** * Translate an array to an {@link List} * * @param types * The {@link Iterable} to translate * @param * The type of the {@link Iterable} * @return The translated {@link Iterable} */ @SafeVarargs public static List toList(final T... types) { return asList(types); } /** * Translate an {@link Iterable} to a {@link String} * * @param input * The input {@link Iterable} * @param * The type of the {@link Iterable} * @param name * The name of the input {@link Iterable} * @return A {@link String} representation of the {@link Iterable} */ public static String toString(final Iterable input, final String name) { return toString(input, name, ", "); } /** * Translate an {@link Iterable} to a {@link String} * * @param input * The input {@link Iterable} * @param * The type of the {@link Iterable} * @param name * The name of the input {@link Iterable} * @param separator * The separator to use between each item in the input {@link Iterable} * @return A {@link String} representation of the {@link Iterable} */ public static String toString(final Iterable input, final String name, final String separator) { final StringBuilder builder = new StringBuilder(); builder.append("["); builder.append(name); builder.append(": "); long index = 0; for (final T type : input) { if (index > 0) { builder.append(separator); } builder.append(type.toString()); index++; } builder.append("]"); return builder.toString(); } /** * Translate an {@link Iterable} of type I to an {@link Iterable} of type O. * * @param input * The input {@link Iterable} * @param converter * The converter from I to O * @param * The type of the input {@link Iterable} * @param * The type of the output {@link Iterable} * @return The {@link Iterable} of O */ public static Iterable translate(final Iterable input, final Function converter) { return filterTranslate(input, converter, item -> true); } /** * Translate an {@link Iterable} of type I to an {@link Iterable} of type O. * * @param input * The input {@link Iterable} * @param converter * The converter from I to O * @param * The type of the input {@link Iterable} * @param * The type of the output {@link Iterable} * @param matcher * A {@link Predicate} on O that filters only the items to match * @return The {@link Iterable} of O */ public static Iterable translateFilter(final Iterable input, final Function converter, final Predicate matcher) { return Iterables.filter(Iterables.translate(input, converter), matcher); } /** * Translate an {@link Iterable} of type I to an {@link Iterable} of O where each converter * yields multiple O for each I. * * @param iterableIn * The input {@link Iterable} * @param converter * The converter from I to multiple O * @param * The type of the input {@link Iterable} * @param * The type of the output {@link Iterable} * @return The {@link Iterable} of O */ public static Iterable translateMulti(final Iterable iterableIn, final Function> converter) { return new MultiIterable<>(Iterables.translate(iterableIn, converter)); } /** * Truncate an {@link Iterable}. * * @param types * The {@link Iterable} to translate * @param startIndex * The index before which to truncate from the start * @param indexFromEnd * The index after which to truncate from the end * @param * The type of the {@link Iterable} * @return The truncated {@link Iterable} */ public static Iterable truncate(final Iterable types, final int startIndex, final int indexFromEnd) { return new SubIterable<>(types, startIndex, indexFromEnd); } private Iterables() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/JoinedCollection.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Optional; import org.openstreetmap.atlas.exception.CoreException; /** * A Joined Collection is simply an array of elements that have been joined together from a * ParallelIterator. This object is some what specific to the ParallelIterable object which uses it * to join single elements from multiple iterable lists. * * @author cuthbertm */ public class JoinedCollection { private final Object[] elements; public JoinedCollection(final int originalSize) { this.elements = new Object[originalSize]; for (int index = 0; index < this.elements.length; index++) { this.elements[index] = null; } } @SuppressWarnings("unchecked") public Type get(final int index) throws ClassCastException, CoreException { if (index >= 0 && index < this.elements.length) { return (Type) this.elements[index]; } throw new CoreException("Invalid index {}, needs to be value between -1 and {}", index, this.elements.length); } public Optional getOption(final int index) throws ClassCastException { final Type returnType = get(index); if (returnType == null) { return Optional.empty(); } return Optional.of(returnType); } public void set(final int index, final Type value) throws CoreException { if (index >= 0 && index < this.elements.length) { this.elements[index] = value; } else { throw new CoreException("Invalid index {}, needs to be value between -1 and {}", index, this.elements.length); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/Maps.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.HashMap; import java.util.Map; import org.openstreetmap.atlas.exception.CoreException; /** * @author matthieun */ public final class Maps { /** * Return a {@link HashMap} from a even number of keys and values of the same type * * @param items * a even number of keys and values of the same type * @param * The type for the map * @return A {@link HashMap} translated from the keys and values in items */ @SafeVarargs public static Map hashMap(final T... items) { if (items.length % 2 != 0) { throw new CoreException("Needs to have an even number of arguments"); } final Map result = new HashMap<>(); for (int i = 0; i < items.length; i += 2) { result.put(items[i], items[i + 1]); } return result; } public static Map stringMap(final String... items) { return hashMap(items); } @SafeVarargs public static Map withMaps(final boolean rejectCollisions, final Map... items) { if (items.length == 0) { return new HashMap<>(); } if (items.length == 1) { return items[0]; } final Map result = new HashMap<>(); for (final Map item : items) { for (final Map.Entry entry : item.entrySet()) { if (rejectCollisions && result.containsKey(entry.getKey())) { throw new CoreException("Cannot merge maps! Collision on key."); } result.put(entry.getKey(), entry.getValue()); } } return result; } @SafeVarargs public static Map withMaps(final Map... items) { return withMaps(true, items); } private Maps() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/MultiIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Iterator; import org.openstreetmap.atlas.exception.CoreException; /** * Iterator made of multiple sub-iterators * * @author matthieun * @param * The type of all the iterators */ public class MultiIterable implements Iterable { private final Iterable> iterables; public MultiIterable(final Iterable> iterables) { this.iterables = iterables; } @SafeVarargs public MultiIterable(final Iterable... iterables) { if (iterables.length == 0) { throw new CoreException("Cannot have an empty set of Iterables."); } this.iterables = Iterables.asList(iterables); } @Override public Iterator iterator() { return new Iterator() { private final Iterator> iterablesIterator = MultiIterable.this.iterables .iterator(); private Iterator currentIterator = this.iterablesIterator.hasNext() ? this.iterablesIterator.next().iterator() : null; @Override public boolean hasNext() { if (this.currentIterator == null) { return false; } if (this.currentIterator != null && this.currentIterator.hasNext()) { return true; } while (this.iterablesIterator.hasNext()) { this.currentIterator = this.iterablesIterator.next().iterator(); if (this.currentIterator != null && this.currentIterator.hasNext()) { return true; } } return false; } @Override public T next() { if (hasNext()) { return this.currentIterator.next(); } return null; } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/OptionalIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Iterator; import java.util.Optional; /** * Iterator that will skip over the empty {@link Optional}s * * @param * The type of the {@link Optional} iterator * @author cuthbertm * @author jklamer */ public class OptionalIterable implements Iterable { private final Iterable> iterable; public OptionalIterable(final Iterable> iterable) { this.iterable = iterable; } @Override public Iterator iterator() { return new Iterator() { private final Iterator> iterator = OptionalIterable.this.iterable .iterator(); private Optional previousElement = Optional.empty(); @Override public boolean hasNext() { if (this.previousElement.isPresent()) { return true; } else { while (this.iterator.hasNext()) { final Optional current = this.iterator.next(); if (current.isPresent()) { this.previousElement = current; return true; } } return false; } } @Override public T next() { if (this.previousElement.isPresent()) { final T returnElement = this.previousElement.get(); this.previousElement = Optional.empty(); return returnElement; } else { while (this.iterator.hasNext()) { final Optional item = this.iterator.next(); if (item.isPresent()) { return item.get(); } } return null; } } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/ParallelIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; /** * This allows you iterator in parallel over multiple iterators. So simplest example would be 2 int * iterators {1, 2} and {3, 4}. First iteration would return {1, 3}, second iteration would return * {2, 4}. The Iterable can handle different list types (eg. {1, 2} and {"one", "two"}) and can * handle varying lengths (eg. {1, 2} and {1, 2, 3}). In the example of varying lengths when * retrieving the third iterations elements it will return null or Optional.empty(), depending on * how you request it, for the 1st list. * * @author cuthbertm */ @SuppressWarnings("rawtypes") public class ParallelIterable implements Iterable { private final List iterables; public ParallelIterable(final Iterable... iterables) { this.iterables = Arrays.asList(iterables); } public List getIteratorList() { final List iteratorList = new ArrayList<>(); this.iterables.forEach(iterator -> iteratorList.add(iterator.iterator())); return iteratorList; } @Override public Iterator iterator() { return new Iterator() { private final List iterators = getIteratorList(); @Override public boolean hasNext() { // this strange line will filter out any of the iterables that do not have any more // elements, and then we will check to see if the result has any more elements. return Iterables.filter(this.iterators, Iterator::hasNext).iterator().hasNext(); } @Override public JoinedCollection next() { final JoinedCollection joined = new JoinedCollection(this.iterators.size()); for (int index = 0; index < this.iterators.size(); index++) { final Iterator iterator = this.iterators.get(index); if (iterator.hasNext()) { joined.set(index, iterator.next()); } } return joined; } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/Sets.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.HashSet; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import org.openstreetmap.atlas.exception.CoreException; /** * @author matthieun */ public final class Sets { @SafeVarargs public static Set hashSet(final T... elements) { final Set result = new HashSet<>(); for (final T element : elements) { result.add(element); } return result; } @SafeVarargs public static > SortedSet treeSet(final T... elements) { final SortedSet result = new TreeSet<>(); for (final T element : elements) { result.add(element); } return result; } @SafeVarargs public static Set withSets(final boolean rejectCollisions, final Set... items) { if (items.length == 0) { return new HashSet<>(); } if (items.length == 1) { return items[0]; } final Set result = new HashSet<>(); for (final Set item : items) { for (final V entry : item) { if (rejectCollisions && result.contains(entry)) { throw new CoreException("Cannot merge sets! Collision on element."); } result.add(entry); } } return result; } @SafeVarargs public static Set withSets(final Set... items) { return withSets(true, items); } @SafeVarargs public static SortedSet withSortedSets(final boolean rejectCollisions, final SortedSet... items) { if (items.length == 0) { return new TreeSet<>(); } if (items.length == 1) { return items[0]; } final SortedSet result = new TreeSet<>(); for (final SortedSet item : items) { for (final V entry : item) { if (rejectCollisions && result.contains(entry)) { throw new CoreException("Cannot merge sets! Collision on element."); } result.add(entry); } } return result; } @SafeVarargs public static SortedSet withSortedSets(final SortedSet... items) { return withSortedSets(true, items); } private Sets() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/ShardBucketCollection.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.io.Serializable; import java.lang.reflect.Array; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.openstreetmap.atlas.geography.Located; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.index.RTree; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.geography.sharding.SlippyTile; import org.openstreetmap.atlas.utilities.scalars.Counter; /** * A collection wrapper for a set of collections associated with shards containing located items, * such as CheckFlags or AtlasEntities. Supports concurrent add, contains, remove. The term bucket * is used because of the usability of the collections mostly as storage for various location based * sorting tasks. Does not support safe iteration while modifying. * * @param * Located item type * @param * Type of the Collection of LocatedType associated with shards * @author jklamer */ public abstract class ShardBucketCollection & Serializable> implements Collection, Serializable { /** * A helper class that associates a shard with the index that its collection is at in the * collectionBuckets array. */ private static class ShardToCollectionIndex implements Located, Serializable { private static final long serialVersionUID = 4050100671815503794L; private final int index; private final Shard shard; ShardToCollectionIndex(final int index, final Shard shard) { this.index = index; this.shard = shard; } @Override public Rectangle bounds() { return this.getShard().bounds(); } public int getIndex() { return this.index; } public Shard getShard() { return this.shard; } } private static final long serialVersionUID = -7892704554302160820L; private final CollectionType[] collectionBuckets; private final RTree collectionIndex; private final HashMap initializedShards = new HashMap<>(); private final Rectangle maximumBounds; public ShardBucketCollection(final Rectangle maximumBounds, final Integer zoomLevel) { this(maximumBounds, SlippyTile.allTiles(zoomLevel, maximumBounds)); } public ShardBucketCollection(final Rectangle maximumBounds, final Sharding sharding) { this(maximumBounds, sharding.shards(maximumBounds)); } /** * Construct the collection by allocating space for a collection for each of the shards and * assigning the index back. * * @param maximumBounds * maximum bound of the collection * @param shards * shards to use */ @SuppressWarnings("unchecked") private ShardBucketCollection(final Rectangle maximumBounds, final Iterable shards) { this.maximumBounds = maximumBounds; this.collectionIndex = new RTree<>(); final Counter counter = new Counter(); shards.forEach(shardBucket -> { final ShardToCollectionIndex shardToCollectionIndex = new ShardToCollectionIndex( (int) counter.getValueAndIncrement(), shardBucket); this.collectionIndex.add(shardToCollectionIndex.bounds(), shardToCollectionIndex); }); this.collectionBuckets = (CollectionType[]) Array.newInstance( this.initializeBucketCollection().getClass(), (int) counter.getValue()); } @Override public final boolean add(final LocatedType item) { if (Objects.nonNull(item) && item.bounds().overlaps(this.maximumBounds)) { final List indexes = this.collectionIndex.get(item.bounds()); if (indexes.size() == 1) { return this.addFunction(item, this.getOrCreateBucketCollectionAt(indexes.get(0)), indexes.get(0).getShard()); } else if (this.allowMultipleBucketInsertion()) { final long addedAmount = indexes.stream() .filter(index -> this.addFunction(item, this.getOrCreateBucketCollectionAt(index), index.getShard())) .count(); return addedAmount > 0; } else { final Shard toInsertAt = this.resolveShard(item, indexes.stream() .map(ShardToCollectionIndex::getShard).collect(Collectors.toList())); final Optional toAddTo = indexes.stream() .filter(index -> toInsertAt.equals(index.getShard())).findFirst(); return toAddTo .map(index -> this.addFunction(item, this.getOrCreateBucketCollectionAt(index), index.getShard())) .orElse(false); } } return false; } @Override public boolean addAll(final Collection collection) { if (Objects.nonNull(collection)) { final long addCount = collection.stream().filter(this::add).count(); return addCount > 0; } return false; } @Override public void clear() { synchronized (this.collectionBuckets) { for (int i = 0; i < this.collectionBuckets.length; i++) { this.collectionBuckets[i] = null; } this.initializedShards.clear(); } } @Override @SuppressWarnings("unchecked") public boolean contains(final Object object) { final Optional typedItem = this.castToLocatedType(object); if (typedItem.isPresent()) { final LocatedType item = typedItem.get(); return this.getBucketCollectionsForBounds(item.bounds()) .anyMatch(collection -> collection.contains(item)); } return false; } @Override public boolean containsAll(final Collection collection) { if (Objects.nonNull(collection)) { return collection.stream().allMatch(this::contains); } return false; } /** * A stream of only distinct elements in the collection. Useful if allowing multiple bucket * insertion * * @return A disticnt stream of the collection */ public Stream distinctStream() { return this.stream().distinct(); } /** * @return a stream of all initialized bucket collections */ public Stream getAllBucketCollections() { return Arrays.stream(this.collectionBuckets).filter(Objects::nonNull); } /** * Get a map of all Shards to their corresponding collection. This will only return shard * collection entries with initialized collections. * * @return Map of shard to collection */ public Map getAllShardBucketCollectionPairs() { return this.initializedShards.values().stream() .collect(Collectors.toMap(ShardToCollectionIndex::getShard, this::getCollectionAt)); } /** * Get the collection for a given shard. It will return optional empty if the shard's collection * isn't initialized * * @param shard * shard whose collection you want. * @return Optional of the initialized Collection */ public Optional getBucketCollectionForShard(final Shard shard) { return Optional.ofNullable(this.initializedShards.get(shard)).map(this::getCollectionAt) .filter(Objects::nonNull); } /** * A stream of all the bucket collects whose associated shard overlaps with the bounds. Note it * will return more than one collection if given a shard bound because the edges of shards * overlap * * @param bounds * to use * @return stream of a subset of bucket collections */ public Stream getBucketCollectionsForBounds(final Rectangle bounds) { return this.collectionIndex.get(bounds).stream().map(this::getCollectionAt) .filter(Objects::nonNull); } /** * Get the max bounds of what located can be in the collection. * * @return rectangle bounds */ public Rectangle getMaximumBounds() { return this.maximumBounds; } @Override public boolean isEmpty() { return this.size() == 0; } @Override public Iterator iterator() { return this.stream().iterator(); } @Override @SuppressWarnings("unchecked") public boolean remove(final Object object) { final Optional typedItem = this.castToLocatedType(object); if (typedItem.isPresent()) { final LocatedType item = typedItem.get(); if (item.bounds().overlaps(this.maximumBounds)) { final long removeCount = this.getBucketCollectionsForBounds(item.bounds()) .filter(collection -> collection.remove(item)).count(); return removeCount > 0; } } return false; } @Override public boolean removeAll(final Collection collection) { if (Objects.nonNull(collection)) { final long removeCount = collection.stream().filter(this::remove).count(); return removeCount > 0; } return false; } @Override public boolean retainAll(final Collection collection) { final List toRemove = this.stream().filter(item -> !collection.contains(item)) .collect(Collectors.toList()); return this.removeAll(toRemove); } @Override public int size() { synchronized (this.collectionBuckets) { return this.getAllBucketCollections().mapToInt(CollectionType::size).sum(); } } @Override public Stream stream() { return this.getAllBucketCollections().flatMap(CollectionType::stream); } @Override public Object[] toArray() { Object[] toReturn = new Object[0]; synchronized (this.collectionBuckets) { final Iterator bucketIterator = this.getAllBucketCollections() .iterator(); while (bucketIterator.hasNext()) { toReturn = ArrayUtils.addAll(toReturn, bucketIterator.next().toArray()); } } return toReturn; } @Override @SuppressWarnings("unchecked") public T1[] toArray(final T1[] otherArray) { return ArrayUtils.addAll(otherArray, (T1[]) this.toArray()); } /** * To add the item into the collection in a special way or dependent on the shard, this function * can be overridden. The return contract is the same of {@link Collection}'s add. This function * must be deterministic. * * @param item * to add * @param collection * to add to * @param shard * shard associated with the collection * @return true if the collection has been added to successfully and changed as a result, false * otherwise */ protected boolean addFunction(final LocatedType item, final CollectionType collection, final Shard shard) { return collection.add(item); } /** * @return true if items are allowed to be in multiple buckets. False otherwise */ protected abstract boolean allowMultipleBucketInsertion(); /** * The collection should be agnostic to the shard. Deciding the collection to insert into and * how to insert into based on the shard should be handled by resolveShard and addFunction * respectively. * * @return an intialized empty bucket collection. */ protected abstract CollectionType initializeBucketCollection(); /** * Resolve which of multiple overlapping shards. Note these shards overlap due to a bounds call, * agnostic of the items geometry. * * @param item * located item to insert * @param possibleBuckets * possible buckets for the item to work into * @return Shard whose collection you should add to. */ protected Shard resolveShard(final LocatedType item, final List possibleBuckets) { throw new UnsupportedOperationException( "Implement this method when not allowing multiple bucket insertion"); } @SuppressWarnings("unchecked") private Optional castToLocatedType(final Object object) { try { return Optional.ofNullable(object).map(cast -> (LocatedType) cast); } catch (final ClassCastException e) { return Optional.empty(); } } private void createBucketCollectionAt(final ShardToCollectionIndex index) { synchronized (this.collectionBuckets) { if (Objects.isNull(this.collectionBuckets[index.getIndex()])) { this.collectionBuckets[index.getIndex()] = this.initializeBucketCollection(); this.initializedShards.put(index.getShard(), index); } } } private CollectionType getCollectionAt(final ShardToCollectionIndex index) { synchronized (this.collectionBuckets) { return this.collectionBuckets[index.getIndex()]; } } private CollectionType getOrCreateBucketCollectionAt(final ShardToCollectionIndex index) { final CollectionType collection = this.getCollectionAt(index); if (Objects.isNull(collection)) { this.createBucketCollectionAt(index); return this.getCollectionAt(index); } else { return collection; } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/StreamIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; import java.util.stream.StreamSupport; /** * An {@link Iterable} that offers similar methods as the {@link Stream} API. To construct one, use * {@link Iterables}: *

* * Iterables.stream(someIterable).map(...).filter(...).collect(); * Note: StreamIterable is not thread safe with parallelization usage. * * @author matthieun * @param * The type of the {@link Iterable} */ public class StreamIterable implements Iterable { private final Iterable source; private boolean parallel = false; protected StreamIterable(final Iterable source) { this.source = source; } /** * Construct a new StreamIterable. * * @param source * The source iterable to construct the StreamIterable from * @param parallel * Controls whether to use parallelization or not when streaming */ protected StreamIterable(final Iterable source, final boolean parallel) { this.source = source; this.parallel = parallel; } /** * Test whether all elements from iterable match the given predicate. * * @param predicate * Predicate to test * @return {@code true} when given predicate is true for all entities in iterable, else false */ public boolean allMatch(final Predicate predicate) { return StreamSupport.stream(this.source.spliterator(), this.parallel).allMatch(predicate); } /** * Test whether any of the elements from iterable matches the given predicate * * @param predicate * Predicate to test * @return {@code true} when given predicate is true for any one entity in iterable, else false */ public boolean anyMatch(final Predicate predicate) { return StreamSupport.stream(this.source.spliterator(), this.parallel).anyMatch(predicate); } /** * @return The original {@link Iterable} from this {@link StreamIterable} */ public Iterable collect() { return this.source; } /** * @return The original {@link Iterable} from this {@link StreamIterable}, collected into a * {@link List} */ public List collectToList() { return Iterables.asList(this.source); } /** * @return The original {@link Iterable} from this {@link StreamIterable}, collected into a * {@link Set} */ public Set collectToSet() { return Iterables.asSet(this.source); } /** * @return The original {@link Iterable} from this {@link StreamIterable}, collected into a * {@link SortedSet} */ public SortedSet collectToSortedSet() { return Iterables.asSortedSet(this.source); } /** * Disable parallelization in streams from this StreamIterator * * @return The StreamIterator with parallelization disabled */ public StreamIterable disableParallelization() { this.parallel = false; return this; } /** * Enable parallelization in streams from this StreamIterator * * @return The StreamIterator with parallelization enabled */ public StreamIterable enableParallelization() { this.parallel = true; return this; } /** * Filter an {@link Iterable} * * @param filter * The filter function * @return The filtered {@link Iterable} as a {@link StreamIterable} */ public StreamIterable filter(final Predicate filter) { return new StreamIterable<>(Iterables.filter(this.source, filter), this.parallel); } /** * Filter an {@link Iterable} using a set of known elements to filter and an idenfitier function * * @param filterSet * The set of IdentifierTypes to filter * @param identifier * The function mapping an element of type T to its identifier of type IdentifierType * @param * The type for the object identifier for elements of the {@link Iterable} * @return The filtered {@link Iterable} as a {@link StreamIterable} */ public StreamIterable filter(final Set filterSet, final Function identifier) { return new StreamIterable<>(new FilteredIterable<>(this.source, filterSet, identifier), this.parallel); } public Optional firstMatching(final Predicate filter) { return Iterables.firstMatching(this.source, filter); } /** * Flat Map an {@link Iterable}. This means each input item can return one to many results. * * @param * The type of the output {@link Iterable} * @param flatMap * The function to flat map * @return The flat mapped {@link Iterable} */ public StreamIterable flatMap(final Function> flatMap) { return new StreamIterable<>(Iterables.translateMulti(this.source, flatMap), this.parallel); } @Override public Iterator iterator() { return this.source.iterator(); } public Optional lastMatching(final Predicate filter) { return Iterables.lastMatching(this.source, filter); } /** * Map an {@link Iterable} to a value * * @param * The new type of the output {@link Iterable} * @param map * The map function * @return The mapped {@link Iterable} */ public StreamIterable map(final Function map) { return new StreamIterable<>(Iterables.translate(this.source, map), this.parallel); } /** * Truncate an {@link Iterable} from start and end * * @param startIndex * The index before which to truncate from the start * @param indexFromEnd * The index after which to truncate from the end * @return The truncated {@link Iterable} */ public StreamIterable truncate(final int startIndex, final int indexFromEnd) { return new StreamIterable<>(Iterables.truncate(this.source, startIndex, indexFromEnd), this.parallel); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/StringList.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.io.Serializable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.regex.RegexUtils; /** * {@link List} of {@link String}s with convenience methods * * @author matthieun */ public class StringList implements Iterable, Serializable { private static final long serialVersionUID = -7923796535827613632L; private final List list; public static StringList split(final String item, final String separator) { return split(item, separator, 0); } /** * Split the string item up to limit pieces. Because the separator is wrapped in regex quotes * (\Q and \E), this method does not support regex. * * @param item * The string to split * @param separator * A string used to separate the input item * @param limit * The limit parameter controls the number of times the pattern is applied and * therefore affects the length of the resulting array. If the limit n is greater * than zero then the pattern will be applied at most n - 1 times, the array's length * will be no greater than n, and the array's last entry will contain all input * beyond the last matched delimiter. * @return A StringList object */ public static StringList split(final String item, final String separator, final int limit) { final Pattern compiledPattern = RegexUtils.getCompiledPattern(Pattern.quote(separator)); return new StringList(Iterables.asList(compiledPattern.split(item, limit))); } public static StringList splitByRegex(final String item, final String separator) { return splitByRegex(item, separator, 0); } /** * Split the string item up to limit pieces, supports regex. * * @param item * The string to split * @param regex * A string used to separate the input item * @param limit * The limit parameter controls the number of times the pattern is applied and * therefore affects the length of the resulting array. If the limit n is greater * than zero then the pattern will be applied at most n - 1 times, the array's length * will be no greater than n, and the array's last entry will contain all input * beyond the last matched delimiter. * @return A StringList object */ public static StringList splitByRegex(final String item, final String regex, final int limit) { final Pattern compiledPattern = RegexUtils.getCompiledPattern(regex); return new StringList(Iterables.asList(compiledPattern.split(item, limit))); } public StringList() { this.list = new ArrayList<>(); } public StringList(final Iterable list) { this.list = Iterables.asList(list); } public StringList(final List list) { this.list = list; } public StringList(final String... array) { this.list = Iterables.asList(array); } public void add(final Object string) { this.list.add(String.valueOf(string)); } public void add(final String string) { this.list.add(string); } public void addAll(final Iterable split) { split.forEach(this::add); } public boolean contains(final String item) { for (final String candidate : this) { if (candidate.equals(item)) { return true; } } return false; } public Optional first() { if (this.size() <= 0) { return Optional.empty(); } return Optional.of(this.get(0)); } public synchronized String get(final int index) { if (index >= size()) { throw new CoreException("Cannot get item out of bounds: {} in size = {}", index, size()); } return this.list.get(index); } public List getUnderlyingList() { return this.list; } public boolean isEmpty() { return this.size() == 0; } @Override public Iterator iterator() { return this.list.iterator(); } public String join(final String separator) { final StringBuilder result = new StringBuilder(); int index = 0; for (final String string : this) { result.append(string); index++; if (index < size()) { result.append(separator); } } return result.toString(); } public Optional last() { if (this.size() <= 0) { return Optional.empty(); } return Optional.of(this.get(size() - 1)); } public void remove(final int index) { this.list.remove(index); } public int size() { return this.list.size(); } /** * @param item * The item to test for * @return True if the item starts with some element in this list. */ public boolean startsWithContains(final String item) { for (final String candidate : this) { if (item.startsWith(candidate)) { return true; } } return false; } public Stream stream() { return this.list.stream(); } public String[] toArray() { final String[] result = new String[this.size()]; for (int index = 0; index < this.size(); index++) { result[index] = this.get(index); } return result; } @Override public String toString() { return this.list.toString(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/SubIterable.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Iterator; import java.util.LinkedList; import java.util.Queue; /** * Takes an {@link Iterable} and truncates some items at the beginning and some items at the end. * * @author matthieun * @param * The type of the {@link Iterable} */ public class SubIterable implements Iterable { private final Iterable source; private final int startIndex; private final int indexFromEnd; public SubIterable(final Iterable source, final int startIndex, final int indexFromEnd) { this.source = source; this.startIndex = startIndex > 0 ? startIndex : 0; this.indexFromEnd = indexFromEnd > 0 ? indexFromEnd : 0; } @Override public Iterator iterator() { return new Iterator() { private int index = 0; private final Iterator sourceIterator = SubIterable.this.source.iterator(); private final Queue lookAheadStore = new LinkedList<>(); @Override public boolean hasNext() { while (this.index < SubIterable.this.startIndex) { if (this.sourceIterator.hasNext()) { this.sourceIterator.next(); } this.index++; } while (this.sourceIterator.hasNext() && this.lookAheadStore.size() < SubIterable.this.indexFromEnd) { this.lookAheadStore.add(this.sourceIterator.next()); } if (this.lookAheadStore.size() == SubIterable.this.indexFromEnd) { return this.sourceIterator.hasNext(); } else { return false; } } @Override public Type next() { if (hasNext()) { final Type result = this.lookAheadStore.isEmpty() ? this.sourceIterator.next() : this.lookAheadStore.poll(); this.index++; return result; } else { return null; } } }; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/UnmodifiableSortedMapCollector.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Collections; import java.util.EnumSet; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; import com.google.common.collect.Ordering; /** * Converts a stream of objects into an immutable map * * @author cstaylor * @param * the type of incoming objects we want to map * @param * the type of keys stored in the map * @param * the values stored in the map */ public class UnmodifiableSortedMapCollector, U> implements Collector, SortedMap> { private final Function keyMapper; private final Function valueMapper; public UnmodifiableSortedMapCollector(final Function keyMapper, final Function valueMapper) { this.keyMapper = keyMapper; this.valueMapper = valueMapper; } @Override public BiConsumer, T> accumulator() { return (builder, item) -> builder.put(this.keyMapper.apply(item), this.valueMapper.apply(item)); } @Override public Set characteristics() { return EnumSet.of(Characteristics.UNORDERED); } @Override public BinaryOperator> combiner() { return (builder1, builder2) -> { builder1.putAll(builder2); return builder1; }; } @Override public Function, SortedMap> finisher() { return original -> Collections.unmodifiableSortedMap(original); } @Override public Supplier> supplier() { return () -> new TreeMap<>(Ordering.natural()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/collections/UnmodifiableSortedSetCollector.java ================================================ package org.openstreetmap.atlas.utilities.collections; import java.util.Collections; import java.util.EnumSet; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; import com.google.common.collect.Ordering; /** * Converts a stream of objects into an immutable set * * @author cstaylor * @param * the values stored in the map */ public class UnmodifiableSortedSetCollector> implements Collector, SortedSet> { @Override public BiConsumer, U> accumulator() { return (builder, item) -> builder.add(item); } @Override public Set characteristics() { return EnumSet.of(Characteristics.UNORDERED); } @Override public BinaryOperator> combiner() { return (builder1, builder2) -> { builder1.addAll(builder2); return builder1; }; } @Override public Function, SortedSet> finisher() { return set -> Collections.unmodifiableSortedSet(set); } @Override public Supplier> supplier() { return () -> new TreeSet<>(Ordering.natural()); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/ActiveModuleIndexWriter.java ================================================ package org.openstreetmap.atlas.utilities.command; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.util.HashSet; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import com.google.common.base.Objects; /** * @author lcram */ public class ActiveModuleIndexWriter { private static final String VERBOSE = "--verbose"; // Use ASCII record separator as delimiter private static final String DELIMITER = Character.toString((char) 0x1E); private final boolean useVerbose; private final String outputPath; public static void main(final String[] args) { String outputPath = null; String verboseFlag = null; if (args.length < 1) { throw new CoreException("Missing required output path argument"); } else if (args.length == 1) { outputPath = args[0]; } else { outputPath = args[0]; verboseFlag = args[1]; } if (Objects.equal(VERBOSE, verboseFlag)) { new ActiveModuleIndexWriter(outputPath, true).printLookupTable(); } else { new ActiveModuleIndexWriter(outputPath, false).printLookupTable(); } } public ActiveModuleIndexWriter(final String outputPath, final boolean useVerbose) { this.outputPath = outputPath; this.useVerbose = useVerbose; } private void diagnosticIfVerbose(final String message) { if (this.useVerbose) { System.out.println(message); // NOSONAR } } private void printLookupTable() { final Set commands = ReflectionUtilities .getSubcommandInstances(); final Set namesWeHaveAlreadySeen = new HashSet<>(); try (PrintWriter printWriter = new PrintWriter(new FileWriter(this.outputPath))) { // print a line break diagnosticIfVerbose(""); for (final AbstractAtlasShellToolsCommand command : commands) { diagnosticIfVerbose("Found command definition in " + command.getClass().getName()); diagnosticIfVerbose("Validating command definition..."); // validate the command name and description command.throwIfInvalidNameOrDescription(); // Validate the command options/args/manpage - will throw if something is awry command.registerOptionsAndArguments(); command.registerManualPageSections(); diagnosticIfVerbose("Generating index entry..."); final StringBuilder builder = new StringBuilder(); String name = command.getCommandName(); String nameWithSuffix = name; int uniqueSuffix = 2; while (namesWeHaveAlreadySeen.contains(nameWithSuffix)) { nameWithSuffix = name + uniqueSuffix; uniqueSuffix++; } name = nameWithSuffix; builder.append(name); namesWeHaveAlreadySeen.add(name); builder.append(DELIMITER); builder.append(command.getClass().getName()); builder.append(DELIMITER); builder.append(command.getSimpleDescription()); printWriter.println(builder.toString()); diagnosticIfVerbose("Command " + command.getCommandName() + " registered OK."); // print a line break diagnosticIfVerbose(""); } } catch (final IOException exception) { throw new CoreException("Could not write index", exception); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/AtlasShellToolsException.java ================================================ package org.openstreetmap.atlas.utilities.command; import org.openstreetmap.atlas.exception.CoreException; /** * A special core exception for cases that should never happen. If users see this, it's a bug! * * @author lcram */ public class AtlasShellToolsException extends CoreException { private static final long serialVersionUID = -2538051525989047548L; public AtlasShellToolsException() { super("This should never happen - you found a bug! Please report this stack trace and the command line that caused it."); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/ReflectionUtilities.java ================================================ package org.openstreetmap.atlas.utilities.command; import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsMarkerInterface; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfoList; import io.github.classgraph.ScanResult; /** * @author lcram */ public final class ReflectionUtilities { @SuppressWarnings("unchecked") public static Set getSubcommandInstances() { final List> subcommandClasses = new ArrayList<>(); final Set instantiatedCommands = new HashSet<>(); try (ScanResult scanResult = new ClassGraph().enableAllInfo().scan()) { final ClassInfoList classInfoList = scanResult .getClassesImplementing(AtlasShellToolsMarkerInterface.class.getName()); classInfoList.loadClasses().forEach(klass -> subcommandClasses .add((Class) klass)); } subcommandClasses.stream().forEach(klass -> instantiateSubcommand(klass.getName()) .ifPresent(instantiatedCommands::add)); return instantiatedCommands; } private static Optional instantiateSubcommand( final String classname) { final Class subcommandClass; try { subcommandClass = Class.forName(classname); } catch (final ClassNotFoundException exception) { throw new CoreException("Class {} was not found", classname, exception); } if (Modifier.isAbstract(subcommandClass.getModifiers())) { return Optional.empty(); } final Constructor constructor; try { constructor = subcommandClass.getConstructor(); } catch (final NoSuchMethodException exception) { throw new CoreException("Class {} does not have a matching constructor", classname, exception); } catch (final SecurityException exception) { throw new CoreException("Error instantiating class {}", classname, exception); } final AbstractAtlasShellToolsCommand subcommand; try { subcommand = (AbstractAtlasShellToolsCommand) constructor.newInstance(new Object[] {}); // NOSONAR } catch (final ClassCastException exception) { throw new CoreException("Class {} not a subtype of {}", classname, AbstractAtlasShellToolsCommand.class.getName(), exception); } catch (final Exception exception) { throw new CoreException("Error instantiating class {}", classname, exception); } return Optional.of(subcommand); } private ReflectionUtilities() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/AbstractAtlasShellToolsCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.abstractcommand; import java.io.InputStream; import java.io.PrintStream; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.command.documentation.DocumentationFormatter; import org.openstreetmap.atlas.utilities.command.documentation.DocumentationRegistrar; import org.openstreetmap.atlas.utilities.command.documentation.PagerHelper; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.parsing.SimpleOptionAndArgumentParser; import org.openstreetmap.atlas.utilities.command.parsing.SimpleOptionAndArgumentParser.SimpleOption; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.AmbiguousAbbreviationException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.UnknownOptionException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.UnparsableContextException; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.openstreetmap.atlas.utilities.command.terminal.TTYStringBuilder; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A partial implementation of an Atlas Shell Tools command. Contains significant functionality to * aid in command development, including some builtin options. * * @author lcram * @author bbreithaupt */ public abstract class AbstractAtlasShellToolsCommand implements AtlasShellToolsMarkerInterface { /** * DEFAULT_CONTEXT is the integer value of a command's default context. Subclasses may register * additional option parser contexts. They should use this value when performing contextual * operations on the default context. See {@link SimpleOptionAndArgumentParser} for more * information about contexts. */ public static final int DEFAULT_CONTEXT = 3; private static final String LINE_SEPARATOR = "line.separator"; private static final Logger logger = LoggerFactory .getLogger(AbstractAtlasShellToolsCommand.class); /** * Until Java supports the ability to do granular TTY configuration checking thru an interface * like isatty(3), we must rely on special tail arguments. An external wrapper (bash, perl, * etc.) can do the necessary TTY config check, and then pass these sentinels to tell the * subcommand if it should use special formatting. A ticket to support better TTY in Java * checking has been open for years, with no avail: * https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4099017. In addition to the tail * arguments declared here, this command also expects a TTY maximum column value (a single * integer). See {@link AbstractAtlasShellToolsCommand#runSubcommandAndExit(String...)} for the * code that unpacks the tail arguments. */ private static final String JAVA_COLOR_STDOUT = "___atlas-shell-tools_color_stdout_SPECIALARGUMENT___"; private static final String JAVA_NO_COLOR_STDOUT = "___atlas-shell-tools_nocolor_stdout_SPECIALARGUMENT___"; private static final String JAVA_COLOR_STDERR = "___atlas-shell-tools_color_stderr_SPECIALARGUMENT___"; private static final String JAVA_NO_COLOR_STDERR = "___atlas-shell-tools_nocolor_stderr_SPECIALARGUMENT___"; private static final String JAVA_USE_PAGER = "___atlas-shell-tools_use_pager_SPECIALARGUMENT___"; private static final String JAVA_NO_USE_PAGER = "___atlas-shell-tools_no_use_pager_SPECIALARGUMENT___"; private static final String JAVA_MARKER_SENTINEL = "___atlas-shell-tools_LAST_ARG_MARKER_SENTINEL___"; private static final int NUMBER_SENTINELS = 5; private static final int STDOUT_COLOR_OFFSET = 5; private static final int STDERR_COLOR_OFFSET = 4; private static final int PAGER_OFFSET = 3; private static final int TERMINAL_COLUMN_OFFSET = 2; /* * The following options are default-registered. */ private static final String VERBOSE_OPTION_LONG = "verbose"; private static final Character VERBOSE_OPTION_SHORT = 'v'; private static final String VERBOSE_OPTION_DESCRIPTION = "Show verbose output messages."; private static final String HELP_OPTION_LONG = "help"; private static final Character HELP_OPTION_SHORT = 'h'; private static final String HELP_OPTION_DESCRIPTION = "Show this help menu."; private static final String VERSION_OPTION_LONG = "version"; private static final Character VERSION_OPTION_SHORT = 'V'; private static final String VERSION_OPTION_DESCRIPTION = "Print the command version and exit."; private static final int HELP_OPTION_CONTEXT = 1; private static final int VERSION_OPTION_CONTEXT = 2; /* * This is a hack option that can be supplied by callers to override the option parser's default * behaviour. If present, it is stripped from ARGS before reaching the option parser. */ private static final String IGNORE_UNKNOWN_OPTIONS = "--ignore-unknown-options"; /* * Maximum allowed column width. If the user's terminal is very wide, we don't want to display * documentation all the way to the max column, since it may become hard to read. */ private static final int MAXIMUM_ALLOWED_COLUMN = 225; private final SimpleOptionAndArgumentParser parser = new SimpleOptionAndArgumentParser(); private final DocumentationRegistrar registrar = new DocumentationRegistrar(); /* * These variables control terminal-related parameters. Their values can be set by the caller * from the command line using sentinal arguments. Try running the atlas-shell-tools "atlas" * wrapper perl program with the '--debug' option to see how this works. Also, see the method * "unpackSentinelArguments". */ private boolean useColorStdout = false; private boolean useColorStderr = false; private boolean usePager = false; private int maximumColumn = DocumentationFormatter.DEFAULT_MAXIMUM_COLUMN; private String version = "default_version_value"; private boolean ignoreUnknownOptions = false; /* * Why are we using PrintStreams instead of a logger? Well, we aren't totally. We still * encourage the use of a logger with various levels for diagnostic messages (see the many * examples of this in subclass implementations). However, for simplicity's sake we use * System.out/err for direct messages intended for the user to read on the command line. This * makes it easier for users to adjust their log configuration to show the desired level of * diagnostics without affecting the actual command outputs. We are declaring explicit stream * variables here so that client classes (i.e. unit tests) can override the streams for testing * purposes. */ /** * See {@link AbstractAtlasShellToolsCommand#setNewOutStream(PrintStream)}. */ private PrintStream outStream = System.out; // NOSONAR /** * See {@link AbstractAtlasShellToolsCommand#setNewErrStream(PrintStream)}. */ private PrintStream errStream = System.err; // NOSONAR /** * See {@link AbstractAtlasShellToolsCommand#setNewInStream(InputStream)}. */ private InputStream inStream = System.in; /** * The default value here is {@link FileSystems#getDefault()}. See * {@link AbstractAtlasShellToolsCommand#setNewFileSystem(FileSystem)} for more information. */ private FileSystem fileSystem = FileSystems.getDefault(); /** * By default the command environment will be given by {@link System#getenv()}. See * {@link AbstractAtlasShellToolsCommand#setNewEnvironment(Map)} for more information. */ private Map environment = null; /** * Add a section to this command's manual page. The section name will be made all capitalized. * Also, use the supplied input stream to read contents into the section. * * @param section * the name of the section * @param sectionResourceFileStream * an input stream to the section resource file (easily specified like * CommandName.class.getResourceAsStream("resourcefile.txt")) */ public void addManualPageSection(final String section, final InputStream sectionResourceFileStream) { this.registrar.addManualPageSection(section, sectionResourceFileStream); } /** * Execute the command logic. Subclasses of {@link AbstractAtlasShellToolsCommand} must * implement this method, but in general it should not be called directly. See * {@link AbstractAtlasShellToolsCommand#runSubcommandAndExit(String...)}. * * @return the return code of the command */ public abstract int execute(); /** * Force the current command to exit with a given exit code. * * @param exitCode * the exit code */ public void forceExit(final int exitCode) { System.exit(exitCode); } /** * The simple name of the command. This should be easy to type for ease of command line use. * * @return the simple name of the command */ public abstract String getCommandName(); /** * Get a {@link CommandOutputDelegate} bound to this {@link AbstractAtlasShellToolsCommand}. * * @return a delegate bound to this command */ public CommandOutputDelegate getCommandOutputDelegate() { return new CommandOutputDelegate(this); } /** * Get the value of an environment variable. Command implementations should always defer to this * method rather than {@link System#getenv()}, since unit tests may be utilizing * {@link AbstractAtlasShellToolsCommand#setNewEnvironment(Map)} to inject a custom environment * for testing purposes. * * @param name * the name of the environment variable * @return the string value of the variable, or {@code null} if the variable is not defined in * the environment */ public String getEnvironmentValue(final String name) { if (this.environment == null) { return System.getenv(name); } return this.environment.getOrDefault(name, null); } /** * Get the {@link PrintStream} for this command's err stream. * * @return the {@link PrintStream} for this command's err stream. */ public PrintStream getErrStream() { return this.errStream; } /** * Get the {@link FileSystem} for this command. * * @return the {@link FileSystem} for this command. */ public FileSystem getFileSystem() { return this.fileSystem; } /** * Get the {@link InputStream} for this command. * * @return the {@link InputStream} for this command. */ public InputStream getInStream() { return this.inStream; } /** * Get the maximum column of the current terminal. * * @return the maximum column */ public int getMaximumColumn() { return this.maximumColumn; } /** * Get an {@link OptionAndArgumentDelegate} bound to this * {@link AbstractAtlasShellToolsCommand}. * * @return a fetcher bound to this command */ public OptionAndArgumentDelegate getOptionAndArgumentDelegate() { return new OptionAndArgumentDelegate(this); } /** * Get the {@link PrintStream} for this command's out stream. * * @return the {@link PrintStream} for this command's out stream. */ public PrintStream getOutStream() { return this.outStream; } /** * A simple description of the command. It should be brief - see the NAME section of any man * page for an example. * * @return the description */ public abstract String getSimpleDescription(); /** * Get a {@link TTYStringBuilder} that is configured to respect the color settings of stderr. * * @return the configured {@link TTYStringBuilder} */ public TTYStringBuilder getTTYStringBuilderForStderr() { return new TTYStringBuilder(this.useColorStderr); } /** * Get a {@link TTYStringBuilder} that is configured to respect the color settings of stdout. * * @return the configured {@link TTYStringBuilder} */ public TTYStringBuilder getTTYStringBuilderForStdout() { return new TTYStringBuilder(this.useColorStdout); } /** * Register an argument with a given arity. The argument hint is used as a key to retrieve the * argument value(s) later. Additionally, documentation can use the hint to specify what the * argument should be for. * * @param argumentHint * the hint for the argument * @param arity * the argument arity * @param optionality * whether the argument is optional or required * @param contexts * the contexts * @throws CoreException * if the argument could not be registered */ public void registerArgument(final String argumentHint, final ArgumentArity arity, final ArgumentOptionality optionality, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerArgument(argumentHint, arity, optionality, DEFAULT_CONTEXT); } else { this.parser.registerArgument(argumentHint, arity, optionality, contexts); } } /** * Register an empty context. This is useful if you want to have a defined usage case where no * options or arguments are passed. * * @param context * the context id */ public void registerEmptyContext(final int context) { this.parser.registerEmptyContext(context); } /** * Register any desired manual page sections. An OPTIONS section will be automatically * generated, so it is recommended that you register at least a DESCRIPTION and EXAMPLES section * with some appropriate documentation. See other {@link AbstractAtlasShellToolsCommand} * implementations for how this is done. For clarification on best practices and/or other * sections to include, see any system man-page (git(1), curl(1), and less(1) are good places to * start). */ public abstract void registerManualPageSections(); /** * Register a {@link AtlasShellToolsCommandTemplate}'s manual pages for this command. * * @param template * the {@link AtlasShellToolsCommandTemplate} whose man pages sections you want to * register */ public void registerManualPageSectionsFromTemplate( final AtlasShellToolsCommandTemplate template) { template.registerManualPageSections(this); } /** * Register an option with a given long and short form. The option will be a flag option, ie. it * can take no arguments. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOption(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOption(longForm, shortForm, description, optionality, DEFAULT_CONTEXT); } else { this.parser.registerOption(longForm, shortForm, description, optionality, contexts); } } /** * Register an option with a given long form. The option will be a flag option, ie. it can take * no arguments. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOption(final String longForm, final String description, final OptionOptionality optionality, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOption(longForm, description, optionality, DEFAULT_CONTEXT); } else { this.parser.registerOption(longForm, description, optionality, contexts); } } /** * Register an option with a given long and short form that takes an optional argument. The * provided argument hint can be used for generated documentation, and should be a single word * describing the argument. The parser will throw an exception at parse-time if the argument is * not supplied. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOptionWithOptionalArgument(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOptionWithOptionalArgument(longForm, shortForm, description, optionality, argumentHint, DEFAULT_CONTEXT); } else { this.parser.registerOptionWithOptionalArgument(longForm, shortForm, description, optionality, argumentHint, contexts); } } /** * Register an option with a given long form that takes an optional argument. The provided * argument hint can be used for generated documentation, and should be a single word describing * the argument. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOptionWithOptionalArgument(final String longForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOptionWithOptionalArgument(longForm, description, optionality, argumentHint, DEFAULT_CONTEXT); } else { this.parser.registerOptionWithOptionalArgument(longForm, description, optionality, argumentHint, contexts); } } /** * Register an option with a given long form that takes a required argument. The provided * argument hint can be used for generated documentation, and should be a single word describing * the argument. The parser will throw an exception at parse-time if the argument is not * supplied. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOptionWithRequiredArgument(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOptionWithRequiredArgument(longForm, shortForm, description, optionality, argumentHint, DEFAULT_CONTEXT); } else { this.parser.registerOptionWithRequiredArgument(longForm, shortForm, description, optionality, argumentHint, contexts); } } /** * Register an option with a given long form that takes a required argument. The provided * argument hint can be used for generated documentation, and should be a single word describing * the argument. The parser will throw an exception if a required argument option is not * supplied an argument at parse-time. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts * @throws CoreException * if the option could not be registered */ public void registerOptionWithRequiredArgument(final String longForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (contexts.length == 0) { this.parser.registerOptionWithRequiredArgument(longForm, description, optionality, argumentHint, DEFAULT_CONTEXT); } else { this.parser.registerOptionWithRequiredArgument(longForm, description, optionality, argumentHint, contexts); } } /** * Register any necessary options and arguments for the command. Subclasses should override this * method, but call super.registerOptionsAndArguments last in order to pick up super class * options/args. */ public void registerOptionsAndArguments() { // register --help and --version to contexts 1 and 2, respectively registerOption(HELP_OPTION_LONG, HELP_OPTION_SHORT, HELP_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, HELP_OPTION_CONTEXT); registerOption(VERSION_OPTION_LONG, VERSION_OPTION_SHORT, VERSION_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, VERSION_OPTION_CONTEXT); registerEmptyContext(DEFAULT_CONTEXT); // register a default '--verbose' option in all contexts (except the --help and --version) final Integer[] contexts = this.getFilteredRegisteredContexts().toArray(new Integer[0]); registerOption(VERBOSE_OPTION_LONG, VERBOSE_OPTION_SHORT, VERBOSE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, contexts); } /** * Register a {@link AtlasShellToolsCommandTemplate}'s options and arguments for this command. * * @param template * the {@link AtlasShellToolsCommandTemplate} whose options and arguments you want to * register */ public void registerOptionsAndArgumentsFromTemplate( final AtlasShellToolsCommandTemplate template) { template.registerOptionsAndArguments(this); } /** * Run this subcommand using all the special setup and teardown semantics provided by * {@link AbstractAtlasShellToolsCommand}. It automatically registers some default standard * arguments: (help,h) and (verbose,v). * * @param args * the command arguments * @return an {@code int} that can be used as a system return code */ public int runSubcommand(final String... args) { throwIfInvalidNameOrDescription(); final String[] argsCopy = unpackSentinelArguments(args); this.parser.ignoreUnknownOptions(this.ignoreUnknownOptions); // fill out appropriate data structures so the execute() implementation can query registerOptionsAndArguments(); registerManualPageSections(); // parse the options and arguments, throwing exceptions on bad input try { this.parser.parse(Arrays.asList(argsCopy)); } catch (final AmbiguousAbbreviationException | UnknownOptionException | UnparsableContextException exception) { printlnErrorMessage(exception.getMessage()); printSimpleUsageMenu(); printStderr("Try the \'"); printStderr("--help", TTYAttribute.BOLD); printStderr("\' option (e.g. "); printStderr("atlas " + this.getCommandName() + " --help", TTYAttribute.BOLD); printlnStderr(") for more info"); return 1; } catch (final Exception exception) { throw new CoreException("unhandled exception", exception); } logger.debug("Command using context {}", this.parser.getContext()); // handle the hardcoded --help and --version options if (this.parser.hasOption(HELP_OPTION_LONG)) { if (this.usePager) { final PagerHelper helper = new PagerHelper(); helper.pageString(this.getHelpMenu()); } else { printlnStdout(this.getHelpMenu()); } return 0; } if (this.parser.hasOption(VERSION_OPTION_LONG)) { printlnStdout(String.format("%s version %s", getCommandName(), this.version)); return 0; } // run the command return execute(); } /** * Run this subcommand and exit the JVM. An example of how this method should be called to make * the command functional with an external wrapper: * * 

     * public static void main(final String[] args)
     * {
     *     new MySubclassSubcommand().runSubcommandAndExit(args);
     * }
     *
* * @param args * the command arguments */ public void runSubcommandAndExit(final String... args) { System.exit(this.runSubcommand(args)); } /** * Set a new {@link Map} to act as the system environment for this command. This may be useful * for various unit tests which want to inject alternate values into a command's environment * variables (for example, changing the location of user.home to be independent of the machine * context). * * @param newEnvironment * the new environment {@link Map} */ public void setNewEnvironment(final Map newEnvironment) { this.environment = newEnvironment; } /** * Set a new {@link PrintStream} for the stderr writer methods. This may be useful for various * unit tests which want to intercept command error output to check it against some expected * result. * * @param newErrStream * the new err {@link PrintStream} */ public void setNewErrStream(final PrintStream newErrStream) { this.errStream = newErrStream; } /** * Set a new {@link FileSystem} for this command. Implementations should respect the set * {@link FileSystem} when performing file operations. This is particularly useful for * unit-testing, where we may want to use an alternate file system (e.g. in-memory). * * @param newFileSystem * the new {@link FileSystem} to use. */ public void setNewFileSystem(final FileSystem newFileSystem) { this.fileSystem = newFileSystem; } /** * Set a new {@link InputStream} for this command. Implementations should respect the set * {@link InputStream} when reading input from the user. This is particularly useful for * unit-testing, where we may want to inject arbitrary input for testing purposes. * * @param inStream * the new {@link InputStream} to use */ public void setNewInStream(final InputStream inStream) { this.inStream = inStream; } /** * Set a new {@link PrintStream} for the stdout writer methods. This may be useful for various * unit tests which want to intercept command output to check it against some expected result. * * @param newOutStream * the new out {@link PrintStream} */ public void setNewOutStream(final PrintStream newOutStream) { this.outStream = newOutStream; } /** * Check that the command name and description are valid. This should be called before relying * on the return values of getCommandName and getDescription. */ public void throwIfInvalidNameOrDescription() { final String name = this.getCommandName(); if (name == null || name.isEmpty()) { throw new CoreException("{} command name must not be null or empty", this.getClass().getName()); } final String[] split = name.split("\\s+"); if (split.length > 1) { throw new CoreException("{} command name must not contain whitespace", this.getClass().getName()); } for (int index = 0; index < name.length(); index++) { final char currentCharacter = name.charAt(index); if (!Character.isLetterOrDigit(currentCharacter) && currentCharacter != '-' && currentCharacter != '_') { throw new CoreException( "{} command name must only contain letters, digits, hyphens, or underscores", this.getClass().getName()); } } final String simpleDescription = this.getSimpleDescription(); if (simpleDescription == null || simpleDescription.isEmpty()) { throw new CoreException("{} simple description must not be null or empty", this.getClass().getName()); } } /** * Add a given code line to a given manual page section. Code lines are given additional * indentation and are excluded from line-wrap formatting. If a code line contains a newline, * the formatting will not automatically indent after the line break. * * @param section * the section to add to * @param codeLine * the code line * @throws CoreException * if the section does not exist */ protected void addCodeLineToSection(final String section, final String codeLine) { this.registrar.addCodeLineToSection(section, codeLine); } /** * Add a section to this command's manual page. The section name will be made all capitalized. * * @param section * the name of the section */ protected void addManualPageSection(final String section) { this.registrar.addManualPageSection(section); } /** * Add a given paragraph to a given manual page section. * * @param section * the section to add to * @param paragraph * the paragraph * @throws CoreException * if the section does not exist */ protected void addParagraphToSection(final String section, final String paragraph) { this.registrar.addParagraphToSection(section, paragraph); } /** * Set the version of this command. * * @param version * the version string to use (eg. 1.0.0) */ protected void setVersion(final String version) { this.version = version; } SortedSet getFilteredRegisteredContexts() { // filter out the default, hardcoded '--help' and '--version' contexts final Set set = this.parser.getRegisteredContexts().stream().filter( context -> context != HELP_OPTION_CONTEXT && context != VERSION_OPTION_CONTEXT) .collect(Collectors.toSet()); return new TreeSet<>(set); } Optional getOptionArgument(final String longForm) { return this.parser.getOptionArgument(longForm); } Optional getOptionArgument(final String longForm, final StringConverter converter) { return this.parser.getOptionArgument(longForm, converter); } int getParserContext() { return this.parser.getContext(); } Optional getUnaryArgument(final String hint) { return this.parser.getUnaryArgument(hint); } List getVariadicArgument(final String hint) { return this.parser.getVariadicArgument(hint); } boolean hasOption(final String longForm) { return this.parser.hasOption(longForm); } boolean hasVerboseOption() { return this.parser.hasOption(VERBOSE_OPTION_LONG); } void printStderr(final String string, final TTYAttribute... attributes) { final TTYStringBuilder builder = this.getTTYStringBuilderForStderr(); builder.append(string, attributes); this.errStream.print(builder.toString()); // NOSONAR } void printStdout(final String string, final TTYAttribute... attributes) { final TTYStringBuilder builder = this.getTTYStringBuilderForStdout(); builder.append(string, attributes); this.outStream.print(builder.toString()); // NOSONAR } void printlnCommandMessage(final String message) { printStderr(this.getCommandName() + ": "); printStderr(message + System.getProperty(LINE_SEPARATOR)); } void printlnErrorMessage(final String message) { printStderr(this.getCommandName() + ": "); printStderr("error: ", TTYAttribute.BOLD, TTYAttribute.RED); printStderr(message + System.getProperty(LINE_SEPARATOR)); } void printlnStderr(final String string, final TTYAttribute... attributes) { final TTYStringBuilder builder = this.getTTYStringBuilderForStderr(); builder.append(string, attributes); this.errStream.println(builder.toString()); // NOSONAR } void printlnStdout(final String string, final TTYAttribute... attributes) { final TTYStringBuilder builder = this.getTTYStringBuilderForStdout(); builder.append(string, attributes); this.outStream.println(builder.toString()); // NOSONAR } void printlnWarnMessage(final String message) { printStderr(this.getCommandName() + ": "); printStderr("warn: ", TTYAttribute.BOLD, TTYAttribute.MAGENTA); printStderr(message + System.getProperty(LINE_SEPARATOR)); } private String getHelpMenu() { final String name = this.getCommandName(); final String simpleDescription = this.getSimpleDescription(); final Map> optionsWithContext = this.parser .getContextToRegisteredOptions(); final Set allOptions = this.parser.getRegisteredOptions(); final TTYStringBuilder builder = getTTYStringBuilderForStdout(); builder.newline(); DocumentationFormatter.generateTextForNameSection(name, simpleDescription, builder); builder.newline(); DocumentationFormatter.generateTextForSynopsisSection(name, this.maximumColumn, optionsWithContext, this.parser.getRegisteredContexts(), this.parser.getArgumentHintToArity(), this.parser.getArgumentHintToOptionality(), builder); builder.newline(); // Let's manually insert the DESCRIPTION section first, if it exists. // This is typical for manpages, DESCRIPTION always comes before OPTIONS. if (this.registrar.hasDescriptionSection()) { DocumentationFormatter.generateTextForGenericSection( this.registrar.getDescriptionHeader(), this.maximumColumn, builder, this.registrar); builder.newline(); } DocumentationFormatter.generateTextForOptionsSection(this.maximumColumn, allOptions, builder); builder.newline(); // Insert the rest of the user designed sections for (final String section : this.registrar.getSections()) { // Skip DESCRIPTION header, since we already inserted it before OPTIONS if (this.registrar.getDescriptionHeader().equals(section)) { continue; } DocumentationFormatter.generateTextForGenericSection(section, this.maximumColumn, builder, this.registrar); builder.newline(); } return builder.toString(); } private void printSimpleUsageMenu() { final String name = this.getCommandName(); final Map> optionsWithContext = this.parser .getContextToRegisteredOptions(); final TTYStringBuilder builder = getTTYStringBuilderForStderr(); DocumentationFormatter.generateTextForSynopsisSection(name, this.maximumColumn, optionsWithContext, this.parser.getRegisteredContexts(), this.parser.getArgumentHintToArity(), this.parser.getArgumentHintToOptionality(), builder); printlnStderr(builder.toString()); } private String[] unpackSentinelArguments(final String[] args) { List argsAsList = Arrays.asList(args); argsAsList = new ArrayList<>(argsAsList); if (!argsAsList.isEmpty() && argsAsList.contains(IGNORE_UNKNOWN_OPTIONS)) { this.ignoreUnknownOptions = true; argsAsList.remove(IGNORE_UNKNOWN_OPTIONS); } if (!argsAsList.isEmpty() && JAVA_MARKER_SENTINEL.equals(argsAsList.get(argsAsList.size() - 1))) { final String stdoutColorArg = argsAsList.get(argsAsList.size() - STDOUT_COLOR_OFFSET); final String stderrColorArg = argsAsList.get(argsAsList.size() - STDERR_COLOR_OFFSET); final String usePagerArg = argsAsList.get(argsAsList.size() - PAGER_OFFSET); final String terminalColumnArg = argsAsList .get(argsAsList.size() - TERMINAL_COLUMN_OFFSET); if (JAVA_COLOR_STDOUT.equals(stdoutColorArg)) { this.useColorStdout = true; } else if (JAVA_NO_COLOR_STDOUT.equals(stdoutColorArg)) { this.useColorStdout = false; } if (JAVA_COLOR_STDERR.equals(stderrColorArg)) { this.useColorStderr = true; } else if (JAVA_NO_COLOR_STDERR.equals(stderrColorArg)) { this.useColorStderr = false; } if (JAVA_USE_PAGER.equals(usePagerArg)) { this.usePager = true; } else if (JAVA_NO_USE_PAGER.equals(usePagerArg)) { this.usePager = false; } this.maximumColumn = Integer.parseInt(terminalColumnArg); if (this.maximumColumn > MAXIMUM_ALLOWED_COLUMN) { this.maximumColumn = MAXIMUM_ALLOWED_COLUMN; } argsAsList = argsAsList.subList(0, argsAsList.size() - NUMBER_SENTINELS); } return argsAsList.toArray(new String[0]); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/AtlasShellToolsCommandTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.abstractcommand; import org.openstreetmap.atlas.utilities.command.subcommands.templates.ListOfNumbersTemplate; /** * An {@link AtlasShellToolsCommandTemplate} provides an easy way for implementations of * {@link AbstractAtlasShellToolsCommand} to share options, arguments, man page sections, and common * functionality. For example, by using a template implementation, command authors will not need to * re-declare a common option (with the accompanying duplicated option parsing code) across many * different commands. See {@link ListOfNumbersTemplate} for an example implementation. * * @author lcram */ public interface AtlasShellToolsCommandTemplate { /** * Register some manual page sections associated with this template. * * @param parentCommand * the parent {@link AbstractAtlasShellToolsCommand} for this template */ void registerManualPageSections(AbstractAtlasShellToolsCommand parentCommand); /** * Register some options and arguments associated with this template. * * @param parentCommand * the parent {@link AbstractAtlasShellToolsCommand} for this template */ void registerOptionsAndArguments(AbstractAtlasShellToolsCommand parentCommand); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/AtlasShellToolsMarkerInterface.java ================================================ package org.openstreetmap.atlas.utilities.command.abstractcommand; /** * A marker interface for the classpath scanner. No subcommands should directly implement this * interface. They instead should extend {@link AbstractAtlasShellToolsCommand}. * * @author lcram */ public interface AtlasShellToolsMarkerInterface { } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/CommandOutputDelegate.java ================================================ package org.openstreetmap.atlas.utilities.command.abstractcommand; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.openstreetmap.atlas.utilities.command.terminal.TTYStringBuilder; /** * @author lcram */ public class CommandOutputDelegate { private final AbstractAtlasShellToolsCommand parentCommand; public CommandOutputDelegate(final AbstractAtlasShellToolsCommand parentCommand) { this.parentCommand = parentCommand; } /** * Get a {@link TTYStringBuilder} with the correct formatting settings for stderr. * Implementations of {@link AbstractAtlasShellToolsCommand} should use this method instead of * instantiating their own string builders. * * @return the string builder */ public TTYStringBuilder getTTYStringBuilderForStderr() { return this.parentCommand.getTTYStringBuilderForStderr(); } /** * Get a {@link TTYStringBuilder} with the correct formatting settings for stdout. * Implementations of {@link AbstractAtlasShellToolsCommand} should use this method instead of * instantiating their own string builders. * * @return the string builder */ public TTYStringBuilder getTTYStringBuilderForStdout() { return this.parentCommand.getTTYStringBuilderForStdout(); } /** * Print a message (with no ending newline) to STDERR with the supplied attributes. * * @param string * the string to print * @param attributes * the attributes */ public void printStderr(final String string, final TTYAttribute... attributes) { this.parentCommand.printStderr(string, attributes); } /** * Print a message (with no ending newline) to STDOUT with the supplied attributes. * * @param string * the string to print * @param attributes * the attributes */ public void printStdout(final String string, final TTYAttribute... attributes) { this.parentCommand.printStdout(string, attributes); } /** * Prints the supplied message like "commandName: message" to stderr. Automatically appends a * newline to the output. * * @param message * the message */ public void printlnCommandMessage(final String message) { this.parentCommand.printlnCommandMessage(message); } /** * Prints the supplied message like "commandName: error: message" with automatic coloring to * stderr. Automatically appends a newline to the output. * * @param message * the error message */ public void printlnErrorMessage(final String message) { this.parentCommand.printlnErrorMessage(message); } /** * Print a message to STDERR with the supplied attributes. Terminates the message with a * newline. * * @param string * the string to print * @param attributes * the attributes */ public void printlnStderr(final String string, final TTYAttribute... attributes) { this.parentCommand.printlnStderr(string, attributes); } /** * Print a message to STDOUT with the supplied attributes. Terminates the message with a * newline. * * @param string * the string to print * @param attributes * the attributes */ public void printlnStdout(final String string, final TTYAttribute... attributes) { this.parentCommand.printlnStdout(string, attributes); } /** * Prints the supplied message like "commandName: warn: message" with automatic coloring to * stderr. Automatically appends a newline to the output. * * @param message * the warn message */ public void printlnWarnMessage(final String message) { this.parentCommand.printlnWarnMessage(message); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/abstractcommand/OptionAndArgumentDelegate.java ================================================ package org.openstreetmap.atlas.utilities.command.abstractcommand; import java.util.List; import java.util.Optional; import java.util.SortedSet; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.conversion.StringConverter; /** * @author lcram */ public class OptionAndArgumentDelegate { private final AbstractAtlasShellToolsCommand parentCommand; public OptionAndArgumentDelegate(final AbstractAtlasShellToolsCommand parentCommand) { this.parentCommand = parentCommand; } /** * Get all registered contexts for this command. * * @return the set of registered contexts */ public SortedSet getFilteredRegisteredContexts() { return this.parentCommand.getFilteredRegisteredContexts(); } /** * Get the argument of a given option, if present. * * @param longForm * the long form of the option * @return an {@link Optional} wrapping the argument * @throws CoreException * if longForm does not refer to a registered option */ public Optional getOptionArgument(final String longForm) { return this.parentCommand.getOptionArgument(longForm); } /** * Get the argument of a given option, if present. Also, convert it using the supplied * converter. If the converter function returns null, then this method will return * {@link Optional#empty()}. * * @param * the type to convert to * @param longForm * the long form of the option * @param converter * the conversion function * @return an {@link Optional} wrapping the argument * @throws CoreException * if longForm does not refer to a registered option */ public Optional getOptionArgument(final String longForm, final StringConverter converter) { return this.parentCommand.getOptionArgument(longForm, converter); } /** * Get the current context ID of the command's option parser. * * @return the context ID */ public int getParserContext() { return this.parentCommand.getParserContext(); } /** * Given a hint registered as a unary argument, return an optional wrapping the argument value * associated with that hint. * * @param hint * the hint to check * @return an {@link Optional} wrapping the value * @throws CoreException * if the argument hint was not registered or is not unary */ public Optional getUnaryArgument(final String hint) { return this.parentCommand.getUnaryArgument(hint); } /** * Given a hint registered as a variadic argument, return the argument values associated with * that hint. * * @param hint * the hint to check * @return a list of the values * @throws CoreException * if the argument hint was not registered or is not variadic */ public List getVariadicArgument(final String hint) { return this.parentCommand.getVariadicArgument(hint); } /** * Check if a given option was supplied. This will return true even if only the short form was * actually present on the command line. * * @param longForm * the option * @return if the option was supplied * @throws CoreException * if longForm does not refer to a registered option */ public boolean hasOption(final String longForm) { return this.parentCommand.hasOption(longForm); } /** * Check if the user supplied the '--verbose' or '-v' option. This is a default option inherited * by all commands. * * @return if --verbose was set */ public boolean hasVerboseOption() { return this.parentCommand.hasVerboseOption(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/documentation/DocumentationFormatType.java ================================================ package org.openstreetmap.atlas.utilities.command.documentation; /** * @author lcram */ public enum DocumentationFormatType { PARAGRAPH, CODE } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/documentation/DocumentationFormatter.java ================================================ package org.openstreetmap.atlas.utilities.command.documentation; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionArgumentType; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.parsing.SimpleOptionAndArgumentParser; import org.openstreetmap.atlas.utilities.command.parsing.SimpleOptionAndArgumentParser.SimpleOption; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.openstreetmap.atlas.utilities.command.terminal.TTYStringBuilder; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * @author lcram */ public final class DocumentationFormatter { public static final int DEFAULT_MAXIMUM_COLUMN = 80; public static final int DEFAULT_CODE_INDENT_LEVEL = 2; public static final int DEFAULT_CODE_INDENT_WIDTH = 4; public static final int DEFAULT_PARAGRAPH_INDENT_LEVEL = 1; public static final int DEFAULT_INNER_PARAGRAPH_INDENT_LEVEL = 2; public static final int DEFAULT_PARAGRAPH_INDENT_WIDTH = 4; /** * Call * {@link DocumentationFormatter#addCodeLineAtExactIndentation(int, String, TTYStringBuilder)}, * but compute the exact indentation width by multiplying the supplied indentationLevel with the * default INDENTATION_WIDTH. * * @param indentationLevel * the indentation level * @param string * the code block string * @param builder * the builder to be modified */ public static void addCodeLine(final int indentationLevel, final String string, final TTYStringBuilder builder) { DocumentationFormatter.addCodeLineAtExactIndentation( indentationLevel * DEFAULT_CODE_INDENT_WIDTH, string, builder); } /** * Add a string to the builder with a given number of indentation spaces and a given maximum * column width. The string will be treated as a code block, ie. it will not have any special * formatting applied to it. * * @param exactIndentation * the exact number of indentation spaces * @param string * the string to display * @param builder * the builder to be modified */ public static void addCodeLineAtExactIndentation(final int exactIndentation, final String string, final TTYStringBuilder builder) { builder.pushExactIndentWidth(exactIndentation).append(string).popIndentation(); } /** * Call * {@link DocumentationFormatter#addParagraphWithLineWrappingAtExactIndentation(int, int, String, TTYStringBuilder, boolean)}, * but compute the exact indentation width by multiplying the supplied indentationLevel with the * default INDENTATION_WIDTH. * * @param indentationLevel * the indentation level * @param maximumColumn * the max column to wrap at * @param string * the code block string * @param builder * the builder to be modified * @param indentFirstLine * decide to indent the first line */ public static void addParagraphWithLineWrapping(final int indentationLevel, final int maximumColumn, final String string, final TTYStringBuilder builder, final boolean indentFirstLine) { DocumentationFormatter.addParagraphWithLineWrappingAtExactIndentation( indentationLevel * DEFAULT_PARAGRAPH_INDENT_WIDTH, maximumColumn, string, builder, indentFirstLine); } public static void addParagraphWithLineWrappingAtExactIndentation(final int exactIndentation, final int maximumColumn, final String string, final TTYStringBuilder builder, final boolean indentFirstLine) { final int lineWidth = maximumColumn - exactIndentation; int spaceLeft = lineWidth; final String[] words = string.split("\\s+"); boolean firstIteration = true; if (indentFirstLine) { builder.pushExactIndentWidth(exactIndentation); } else { builder.pushExactIndentWidth(0); } for (final String word : words) { // Word fits exactly in the remaining space if (word.length() == spaceLeft) { builder.append(word).pushExactIndentWidth(0).newline() .pushExactIndentWidth(exactIndentation); spaceLeft = lineWidth; } // Word plus a whitespace is longer than the remaining space else if (word.length() + " ".length() > spaceLeft) { /* * This is a special edge case that can occur if the first word of the documentation * is longer than the line length: if we are on the first iteration, we already * new-lined and indented so just skip these steps. */ if (!firstIteration) { builder.newline(); builder.pushExactIndentWidth(exactIndentation); } builder.append(word + " ").pushExactIndentWidth(0); spaceLeft = lineWidth - (word.length() + " ".length()); } // Word plus a whitespace fits in the remaining space else { builder.append(word + " ").pushExactIndentWidth(0); spaceLeft = spaceLeft - (word.length() + " ".length()); } firstIteration = false; } } public static void generateTextForGenericSection(final String sectionName, final int maximumColumn, final TTYStringBuilder builder, final DocumentationRegistrar registrar) { final List> sectionContents = registrar .getSectionContents(sectionName); final List> sectionContentsFiltered = new ArrayList<>(); // Filter out any empty sections for (final Tuple contents : sectionContents) { if (!contents.getSecond().isEmpty()) { sectionContentsFiltered.add(contents); } } builder.clearIndentationStack(); builder.append(sectionName, TTYAttribute.BOLD).newline(); for (int index = 0; index < sectionContentsFiltered.size(); index++) { final Tuple contents = sectionContentsFiltered .get(index); final DocumentationFormatType type = contents.getFirst(); final String text = contents.getSecond(); if (type == DocumentationFormatType.CODE) { DocumentationFormatter.addCodeLine(DocumentationFormatter.DEFAULT_CODE_INDENT_LEVEL, text, builder); builder.newline(); } else if (type == DocumentationFormatType.PARAGRAPH) { DocumentationFormatter.addParagraphWithLineWrapping( DocumentationFormatter.DEFAULT_PARAGRAPH_INDENT_LEVEL, maximumColumn, text, builder, true); builder.newline(); } // Add an extra newline unless we are on the last element if (index < sectionContentsFiltered.size() - 1) { builder.newline(); } } } public static void generateTextForNameSection(final String name, final String simpleDescription, final TTYStringBuilder builder) { builder.append("NAME", TTYAttribute.BOLD).newline(); builder.clearIndentationStack(); builder.withLevelWidth(DEFAULT_PARAGRAPH_INDENT_WIDTH); builder.pushIndentLevel(DEFAULT_PARAGRAPH_INDENT_LEVEL) .append(name + " -- " + simpleDescription).popIndentation(); builder.newline(); } public static String generateTextForOptionsSection(final int maximumColumn, final Set options, final TTYStringBuilder builder) { final List sortedOptions = new ArrayList<>(options); Collections.sort(sortedOptions); builder.clearIndentationStack(); builder.withLevelWidth(DEFAULT_PARAGRAPH_INDENT_WIDTH); builder.append("OPTIONS", TTYAttribute.BOLD).newline(); for (int index = 0; index < sortedOptions.size(); index++) { final SimpleOption option = sortedOptions.get(index); builder.pushIndentLevel(DEFAULT_PARAGRAPH_INDENT_LEVEL) .append(SimpleOptionAndArgumentParser.LONG_FORM_PREFIX + option.getLongForm(), TTYAttribute.BOLD) .popIndentation(); final OptionArgumentType argumentType = option.getArgumentType(); if (argumentType == OptionArgumentType.OPTIONAL) { builder.append("[" + SimpleOptionAndArgumentParser.OPTION_ARGUMENT_DELIMITER + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + "]"); } else if (argumentType == OptionArgumentType.REQUIRED) { builder.append(SimpleOptionAndArgumentParser.OPTION_ARGUMENT_DELIMITER + "<" + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + ">"); } if (option.getShortForm().isPresent()) { builder.append(", "); builder.append( SimpleOptionAndArgumentParser.SHORT_FORM_PREFIX + option.getShortForm() .orElseThrow(AtlasShellToolsException::new).toString(), TTYAttribute.BOLD); if (argumentType == OptionArgumentType.OPTIONAL) { builder.append("[" + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + "]"); } else if (argumentType == OptionArgumentType.REQUIRED) { builder.append("<" + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + ">"); } } builder.newline(); addParagraphWithLineWrapping(DEFAULT_INNER_PARAGRAPH_INDENT_LEVEL, maximumColumn, option.getDescription(), builder, true); builder.newline(); // Add an extra newline when we are not on the last element if (index < sortedOptions.size() - 1) { builder.newline(); } } return builder.toString(); } public static void generateTextForSynopsisSection(final String programName, // NOSONAR final int maximumColumn, final Map> optionsWithContext, final Set contexts, final Map> argumentArities, final Map> argumentOptionalities, final TTYStringBuilder builder) { builder.append("SYNOPSIS", TTYAttribute.BOLD).newline(); builder.clearIndentationStack(); builder.withLevelWidth(DEFAULT_PARAGRAPH_INDENT_WIDTH); for (final Integer context : contexts) { builder.pushIndentLevel(DEFAULT_PARAGRAPH_INDENT_LEVEL) .append(programName, TTYAttribute.UNDERLINE).popIndentation().append(" "); final StringBuilder paragraph = new StringBuilder(); // add all the options final List sortedOptions = new ArrayList<>( optionsWithContext.getOrDefault(context, new HashSet<>())); Collections.sort(sortedOptions); for (final SimpleOption option : sortedOptions) { if (option.getOptionality() == OptionOptionality.OPTIONAL) { paragraph.append("["); } paragraph.append( SimpleOptionAndArgumentParser.LONG_FORM_PREFIX + option.getLongForm()); final OptionArgumentType argumentType = option.getArgumentType(); if (argumentType == OptionArgumentType.OPTIONAL) { paragraph.append("[" + SimpleOptionAndArgumentParser.OPTION_ARGUMENT_DELIMITER + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + "]"); } else if (argumentType == OptionArgumentType.REQUIRED) { paragraph.append(SimpleOptionAndArgumentParser.OPTION_ARGUMENT_DELIMITER + "<" + option.getArgumentHint().orElseThrow(AtlasShellToolsException::new) + ">"); } if (option.getOptionality() == OptionOptionality.OPTIONAL) { paragraph.append("]"); } paragraph.append(" "); } // now add all the arguments for (final String hint : argumentArities.getOrDefault(context, new HashMap<>()) .keySet()) { if (argumentOptionalities.get(context).get(hint) == ArgumentOptionality.OPTIONAL) { paragraph.append("["); } else if (argumentOptionalities.get(context) .get(hint) == ArgumentOptionality.REQUIRED) { paragraph.append("<"); } paragraph.append(hint); if (argumentArities.get(context).get(hint) == ArgumentArity.VARIADIC) { paragraph.append("..."); } if (argumentOptionalities.get(context).get(hint) == ArgumentOptionality.OPTIONAL) { paragraph.append("] "); } else if (argumentOptionalities.get(context) .get(hint) == ArgumentOptionality.REQUIRED) { paragraph.append("> "); } } final int exactIndentation = DEFAULT_PARAGRAPH_INDENT_LEVEL * DEFAULT_PARAGRAPH_INDENT_WIDTH + programName.length() + " ".length(); addParagraphWithLineWrappingAtExactIndentation(exactIndentation, maximumColumn, paragraph.toString(), builder, false); builder.pushIndentLevel(0).newline(); } } private DocumentationFormatter() { } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/documentation/DocumentationRegistrar.java ================================================ package org.openstreetmap.atlas.utilities.command.documentation; import java.io.InputStream; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * @author lcram */ public class DocumentationRegistrar { private static final String DESCRIPTION_HEADER = "DESCRIPTION"; private final Map>> sections; public DocumentationRegistrar() { this.sections = new LinkedHashMap<>(); } public void addCodeLineToSection(final String section, final String codeLine) { final String capsSection = section.toUpperCase(); if (!this.sections.containsKey(capsSection)) { throw new CoreException("Section {} has not been added", capsSection); } final List> list = this.sections.get(capsSection); list.add(new Tuple<>(DocumentationFormatType.CODE, codeLine)); } public void addManualPageSection(final String section) { final String capsSection = section.toUpperCase(); if (this.sections.containsKey(capsSection)) { throw new CoreException("Manpage section {} was already added", capsSection); } this.sections.put(capsSection, new ArrayList<>()); } public void addManualPageSection(final String section, final InputStream sectionResourceFileStream) { final String capsSection = section.toUpperCase(); if (this.sections.containsKey(capsSection)) { throw new CoreException("Manpage section {} was already added", capsSection); } final StringResource resource = new StringResource(); resource.copyFrom(new InputStreamResource(() -> sectionResourceFileStream)); final String rawText = resource.all(); final List> sectionContents = new ArrayList<>(); StringBuilder paragraphBuilder = new StringBuilder(); final String[] split = rawText.split(System.getProperty("line.separator")); for (final String line : split) { if (line.isEmpty()) { sectionContents.add(new Tuple<>(DocumentationFormatType.PARAGRAPH, paragraphBuilder.toString())); paragraphBuilder = new StringBuilder(); } else if (line.startsWith("#")) { /* * Close and add any in-progress paragraph. If the user was not in the middle of * creating a paragraph, this is fine. Empty blocks will be filtered by downstream * code. */ sectionContents.add(new Tuple<>(DocumentationFormatType.PARAGRAPH, paragraphBuilder.toString())); paragraphBuilder = new StringBuilder(); // scrub the '#' off the line final String scrubbedLine = line.substring(1); sectionContents.add(new Tuple<>(DocumentationFormatType.CODE, scrubbedLine)); } else { paragraphBuilder.append(line + " "); } } // add last paragraph if one is left over sectionContents .add(new Tuple<>(DocumentationFormatType.PARAGRAPH, paragraphBuilder.toString())); this.sections.put(capsSection, sectionContents); } public void addParagraphToSection(final String section, final String paragraph) { final String capsSection = section.toUpperCase(); if (!this.sections.containsKey(capsSection)) { throw new CoreException("Section {} has not been added", capsSection); } final List> list = this.sections.get(capsSection); list.add(new Tuple<>(DocumentationFormatType.PARAGRAPH, paragraph)); } public String getDescriptionHeader() { return DESCRIPTION_HEADER; } public List> getSectionContents(final String section) { return this.sections.get(section); } public Set getSections() { return this.sections.keySet(); } public boolean hasDescriptionSection() { return this.sections.keySet().contains(DESCRIPTION_HEADER); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/documentation/PagerHelper.java ================================================ package org.openstreetmap.atlas.utilities.command.documentation; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; /** * @author lcram */ public class PagerHelper { private static final String PAGER_ENVIRONMENT_VARIABLE = "ATLAS_SHELL_TOOLS_PAGER"; private static final String PAGER_FALLBACK_VARIABLE = "PAGER"; private static final String DEFAULT_PAGER = "less -cSRMis"; /** * Page a given string using the system paging program. Will check PAGER environment variable * first and use that if possible. * * @param string * the string to page */ public void pageString(final String string) { final String pagerVariable = System.getenv(PAGER_ENVIRONMENT_VARIABLE); final String pagerFallbackVariable = System.getenv(PAGER_FALLBACK_VARIABLE); final Optional pagerProgram; final String[] pagerFlags; if (pagerVariable != null && !pagerVariable.isEmpty()) { pagerProgram = callWhichOnPager(pagerVariable.split("\\s+")[0]); pagerFlags = extractFlagsFromVariable(pagerVariable); } else if (pagerFallbackVariable != null && !pagerFallbackVariable.isEmpty()) { pagerProgram = callWhichOnPager(pagerFallbackVariable.split("\\s+")[0]); pagerFlags = extractFlagsFromVariable(pagerFallbackVariable); } else { pagerProgram = callWhichOnPager(DEFAULT_PAGER.split("\\s+")[0]); pagerFlags = extractFlagsFromVariable(DEFAULT_PAGER); } File temporaryFile = null; try { temporaryFile = File.temporary(); } catch (final Exception exception) { System.out.println(string); // NOSONAR return; } if (temporaryFile == null) { System.out.println(string); // NOSONAR return; } temporaryFile.writeAndClose(string); try { final String[] processBuilderArguments = new String[1 + pagerFlags.length + 1]; processBuilderArguments[0] = pagerProgram.orElseThrow(AtlasShellToolsException::new); if (pagerFlags.length > 0) { System.arraycopy(pagerFlags, 0, processBuilderArguments, 1, pagerFlags.length); } processBuilderArguments[processBuilderArguments.length - 1] = temporaryFile .getAbsolutePathString(); final ProcessBuilder processBuilder = new ProcessBuilder(processBuilderArguments); processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); final Process process = processBuilder.start(); process.waitFor(); } catch (final Exception exception) { System.out.println(string); // NOSONAR } finally { temporaryFile.delete(); } } private Optional callWhichOnPager(final String pager) { final String whichProgram = "which"; final Process process; try { process = new ProcessBuilder(whichProgram, pager).start(); } catch (final IOException exception) { return Optional.empty(); } final InputStream stream = process.getInputStream(); final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); String line = null; try { line = reader.readLine(); } catch (final IOException exception) { return Optional.empty(); } if (line == null || line.isEmpty()) { return Optional.empty(); } return Optional.of(line); } private String[] extractFlagsFromVariable(final String variable) { final String[] flags = variable.split("\\s+"); return Arrays.copyOfRange(flags, 1, flags.length); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/ArgumentArity.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing; /** * The arity of an argument. * * @see "https://en.wikipedia.org/wiki/Arity" * @author lcram */ public enum ArgumentArity { UNARY, VARIADIC } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/ArgumentOptionality.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing; /** * The optionality of a program argument. * * @author lcram */ public enum ArgumentOptionality { OPTIONAL, REQUIRED } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/OptionArgumentType.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing; /** * The optionality of an option's argument. * * @author lcram */ public enum OptionArgumentType { NONE, OPTIONAL, REQUIRED } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/OptionOptionality.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing; /** * The optionality of an option. * * @author lcram */ public enum OptionOptionality { OPTIONAL, REQUIRED } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/SimpleOptionAndArgumentParser.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.AmbiguousAbbreviationException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.ArgumentException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.OptionParseException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.UnknownOptionException; import org.openstreetmap.atlas.utilities.command.parsing.exceptions.UnparsableContextException; import org.openstreetmap.atlas.utilities.conversion.StringConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A simple option and argument parser, designed specifically to impose constraints on the format of * the arguments and options. Non-ambiguity is enforced at registration time. Once you have * successfully registered the parser, you can be sure it will parse any input command line as * expected, throwing errors where appropriate. Nothing about this class is thread safe, should you * decide to parse in one thread and read results in another. *

* Supports multiple types of arguments:
* OPTIONAL vs REQUIRED: if an argument marked REQUIRED is not supplied, the parser will throw an * error
* UNARY vs VARIADIC: a VARIADIC argument is one that can consist of an arbitrary number of * values
*
* Supports long and short options:
* --opt : a long option
* --opt-arg=my_argument : a long option with argument, supports optional or required arguments
* --opt-arg my_argument : alternate syntax for required long option arguments
* -a : a short option
* -abc : bundled short options (-a, -b, -c)
* -o arg : a short option (-o) that takes a required arg
* -oarg : alternate syntax, a short option (-o) that takes a required or optional arg
*
* If an option is specified multiple times with different arguments, the parser will use the * version in the highest ARGV position (ie. the furthest right on the command line). *

* This class supports both the POSIX short option spec as well as the GNU long option spec. See * included links for details.
*
* This class supports long option prefix abbreviations. This means that a long option "--option" * can be abbreviated on the command line as "--o" or "--op" or any non-ambiguous prefix. If an * abbreviation results in ambiguity, the parser will throw an error at parse-time.
*
* Note that this class also supports multiple parsing contexts, if desired. A parsing context * corresponds to certain usage case. For example, you can register a context with ID 3 that takes a * single argument and the option "--opt1". Then you can also define a context ID 4 that takes 2 * arguments and an option "--opt2". The parser will automatically figure out which context is * implied from the supplied command line. If more than one context matches, the context with the * lowest numerical ID is selected. If no matching contexts can be found, the parser throws an error * with a diagnostic message explaining what happened.
*
* * @see "https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html" * @see "http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap12.html" * @see "http://pubs.opengroup.org/onlinepubs/7908799/xbd/utilconv.html" * @author lcram */ public class SimpleOptionAndArgumentParser { /** * A simple option representation. Store the option long/short form as well as metadata about * the option. * * @author lcram */ public class SimpleOption implements Comparable { private final String longForm; private final Optional shortForm; private final String description; private final OptionOptionality optionality; // Default values for option argument fields private OptionArgumentType argumentType = OptionArgumentType.NONE; private Optional argumentHint = Optional.empty(); SimpleOption(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final OptionArgumentType argumentType, final String argumentHint) { if (longForm == null || longForm.isEmpty()) { throw new CoreException("Long option form cannot be null or empty"); } if (shortForm != null && !Character.isLetterOrDigit(shortForm)) { throw new CoreException("Invalid short option form {}: must be letter or digit", shortForm); } if (description == null || description.isEmpty()) { throw new CoreException("Description cannot be null or empty"); } this.longForm = longForm; this.shortForm = Optional.ofNullable(shortForm); this.description = description; this.optionality = optionality; this.argumentType = argumentType; if (this.argumentType != OptionArgumentType.NONE) { if (argumentHint != null && !argumentHint.isEmpty()) { final String[] split = argumentHint.split("\\s+"); if (split.length > 1) { throw new CoreException("Option argument hint cannot contain whitespace"); } this.argumentHint = Optional.of(argumentHint); } else { throw new CoreException("Option argument hint cannot be null or empty"); } } } @Override public int compareTo(final SimpleOption other) { final String otherCaps = other.longForm.toUpperCase(); final String thisCaps = this.longForm.toUpperCase(); return thisCaps.compareTo(otherCaps); } @Override public boolean equals(final Object other) { if (other instanceof SimpleOption) { if (this == other) { return true; } final SimpleOption that = (SimpleOption) other; return Objects.equals(this.longForm, that.longForm); } return false; } public Optional getArgumentHint() { return this.argumentHint; } public OptionArgumentType getArgumentType() { return this.argumentType; } public String getDescription() { return this.description; } public String getLongForm() { return this.longForm; } public OptionOptionality getOptionality() { return this.optionality; } public Optional getShortForm() { return this.shortForm; } @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; return hashSeed * initialPrime + Objects.hashCode(this.longForm); } @Override public String toString() { final StringBuilder builder = new StringBuilder(); builder.append(this.longForm); if (this.shortForm.isPresent()) { builder.append(", " + this.shortForm.get()); } return builder.toString(); } } public static final String LONG_FORM_PREFIX = "--"; public static final String SHORT_FORM_PREFIX = "-"; public static final String OPTION_ARGUMENT_DELIMITER = "="; public static final String END_OPTIONS_OPERATOR = "--"; public static final int NO_CONTEXT = 0; private static final String MUST_REGISTER_AT_LEAST_ONE_CONTEXT = "Must register at least one context."; private static final String PROVIDED_OPTION_LONG_FORM_WAS_AMBIGUOUS = "provided option long form {} was ambiguous"; private static final String CANNOT_GET_OPTIONS_BEFORE_PARSING = "Cannot get options before parsing!"; private static final Logger logger = LoggerFactory .getLogger(SimpleOptionAndArgumentParser.class); private final Map> contextToRegisteredOptions; private final Map> contextToArgumentHintToArity; private final Map> contextToArgumentHintToOptionality; private final Map contextToRegisteredVariadicArgument; private final Map contextToRegisteredOptionalArgument; private final SortedSet registeredContexts; private final Set longFormsSeen; private final Set shortFormsSeen; private final Set argumentHintsSeen; private final Map> parsedOptions; private final Map> parsedArguments; private int currentContext; private boolean parseStepRanAtLeastOnce; private boolean ignoreUnknownOptions; public SimpleOptionAndArgumentParser() { this.contextToRegisteredOptions = new HashMap<>(); this.contextToArgumentHintToArity = new HashMap<>(); this.contextToArgumentHintToOptionality = new HashMap<>(); this.contextToRegisteredVariadicArgument = new HashMap<>(); this.contextToRegisteredOptionalArgument = new HashMap<>(); this.registeredContexts = new TreeSet<>(); this.longFormsSeen = new HashSet<>(); this.shortFormsSeen = new HashSet<>(); this.argumentHintsSeen = new HashSet<>(); this.parsedOptions = new LinkedHashMap<>(); this.parsedArguments = new LinkedHashMap<>(); this.currentContext = NO_CONTEXT; this.parseStepRanAtLeastOnce = false; this.ignoreUnknownOptions = false; } /** * Get the mapping of registered argument hints to their arities. * * @return the mapping */ public Map> getArgumentHintToArity() { return this.contextToArgumentHintToArity; } /** * Get the mapping of registered argument hints to their optionalities * * @return the mapping */ public Map> getArgumentHintToOptionality() { return this.contextToArgumentHintToOptionality; } public int getContext() { return this.currentContext; } public Map> getContextToRegisteredOptions() { return this.contextToRegisteredOptions; } /** * Get the argument of a given option, if present. If the option is not a registered option, * this will throw an exception. * * @param longForm * the long form of the option * @return an {@link Optional} wrapping the argument * @throws CoreException * if longForm does not refer to a registered option */ public Optional getOptionArgument(final String longForm) { final Optional option; try { option = getParsedOptionFromLongForm(longForm); } catch (final UnknownOptionException exception) { throw new CoreException("{} not a registered option", longForm); } if (option.isPresent()) { return this.parsedOptions.get(option.get()); } return Optional.empty(); } /** * Get the argument of a given option, if present. Also, convert it using the supplied * converter. If the converter function returns null, then this method will return * {@link Optional#empty()}. If the option is not a registered option, this will throw an * exception. * * @param * the type to convert to * @param longForm * the long form of the option * @param converter * the conversion function * @return an {@link Optional} wrapping the argument * @throws CoreException * if longForm does not refer to a registered option */ public Optional getOptionArgument(final String longForm, final StringConverter converter) { final Optional option; try { option = getParsedOptionFromLongForm(longForm); } catch (final UnknownOptionException exception) { throw new CoreException("{} not a registered option", longForm); } if (option.isPresent()) { final Optional argument = this.parsedOptions.get(option.get()); if (argument.isPresent()) { final String argumentValue = argument.get(); return Optional.ofNullable(converter.convert(argumentValue)); } } return Optional.empty(); } /** * Get a mapping from option names to {@link SimpleOption}s. * * @return the mapping */ public Map getOptionNameToRegisteredOption() { final Set allOptions = getRegisteredOptions(); final Map map = new HashMap<>(); for (final SimpleOption option : allOptions) { map.put(option.getLongForm(), option); } return map; } /** * Get the registered contexts for this parser. * * @return the set */ public SortedSet getRegisteredContexts() { return this.registeredContexts; } /** * Get the set of registered {@link SimpleOption}s. * * @return the set */ public Set getRegisteredOptions() { final Set allOptions = new HashSet<>(); for (final Integer context : this.registeredContexts) { allOptions.addAll(this.contextToRegisteredOptions.get(context)); } return allOptions; } /** * Given a hint registered as a unary argument, return an optional wrapping the argument value * associated with that hint. * * @param hint * the hint to check * @return an optional wrapping the value * @throws CoreException * if the argument hint was not registered or is not unary */ public Optional getUnaryArgument(final String hint) { if (!this.parseStepRanAtLeastOnce) { throw new CoreException("Cannot get arguments before parsing!"); } if (!this.contextToArgumentHintToArity.get(this.currentContext).containsKey(hint)) { return Optional.empty(); } if (this.contextToArgumentHintToArity.get(this.currentContext) .get(hint) != ArgumentArity.UNARY) { throw new CoreException("hint \'{}\' does not correspond to a unary argument", hint); } final List arguments = this.parsedArguments.get(hint); if (arguments != null && arguments.size() == 1) { return Optional.of(arguments.get(0)); } logger.debug("No value found for unary argument {}, returning empty Optional", hint); return Optional.empty(); } /** * Given a hint registered as a variadic argument, return the argument values associated with * that hint. * * @param hint * the hint to check * @return a list of the values * @throws CoreException * if the argument hint was not registered or is not variadic */ public List getVariadicArgument(final String hint) { if (!this.parseStepRanAtLeastOnce) { throw new CoreException("Cannot get arguments before parsing!"); } if (!this.contextToArgumentHintToArity.containsKey(this.currentContext) || !this.contextToArgumentHintToArity.get(this.currentContext).containsKey(hint)) { throw new CoreException( "hint \'{}\' does not correspond to a registered argument in context {}", hint, this.currentContext); } if (this.contextToArgumentHintToArity.get(this.currentContext) .get(hint) != ArgumentArity.VARIADIC) { throw new CoreException("hint \'{}\' does not correspond to a variadic argument", hint); } final List arguments = this.parsedArguments.get(hint); if (arguments != null) { return arguments; } logger.debug("No value found for variadic argument {}, returning empty List", hint); return new ArrayList<>(); } /** * Check if a given long form option was supplied. This will return true even if only the short * form was actually present on the command line. If the option is not a registered option, this * will return false. * * @param longForm * the long form option * @return if the option was supplied */ public boolean hasOption(final String longForm) { Optional option; try { option = getParsedOptionFromLongForm(longForm); } catch (final UnknownOptionException exception) { option = Optional.empty(); } return option.isPresent(); } /** * Set this parser to ignore unknown options. * * @param ignore * true to ignore unknown option * @return this modified instance */ public SimpleOptionAndArgumentParser ignoreUnknownOptions(final boolean ignore) { this.ignoreUnknownOptions = ignore; return this; } public boolean isEmpty() { return this.parsedOptions.isEmpty() && this.parsedArguments.isEmpty(); } public void parse(final List allArguments) throws AmbiguousAbbreviationException, // NOSONAR UnknownOptionException, UnparsableContextException { this.parsedArguments.clear(); this.parsedOptions.clear(); this.currentContext = NO_CONTEXT; boolean seenEndOptionsOperator = false; final List modifiedArguments = new ArrayList<>(); /* * First, we pre-parse arguments to see if there are any ambiguous or unknown long options. * This will help generate better error message for the end user. This check must happen * independent of any parsing context, since you need to be able to disambiguate option * prefix abbreviations before a context is selected. Consider the following example: */ // Parser Context ID 3 has option --opt1 // Parser Context ID 4 has option --opt2 // User supplies option --opt /* * In this case we want to throw an error early, warning that the option is ambiguous. If we * didn't, the parser context selection code would choose context 3 (since it picks the * first context that does not throw a parse error). This is not intuitive behaviour for end * users, who need not know about the mechanics of parser contexts. */ for (final String argument : allArguments) { boolean addBackArg = true; if (END_OPTIONS_OPERATOR.equals(argument)) { if (!seenEndOptionsOperator) { seenEndOptionsOperator = true; } } else if (SHORT_FORM_PREFIX.equals(argument)) { continue; // NOSONAR } else if (argument.startsWith(LONG_FORM_PREFIX) && !seenEndOptionsOperator) { final String[] split = argument.substring(LONG_FORM_PREFIX.length()) .split(OPTION_ARGUMENT_DELIMITER, 2); final String optionName = split[0]; final Optional option = checkForLongOption(optionName, getRegisteredOptions(), true); if (!option.isPresent()) { if (this.ignoreUnknownOptions) { addBackArg = false; } else { throw new UnknownOptionException(optionName, getRegisteredOptions()); } } } else if (argument.startsWith(SHORT_FORM_PREFIX) && !seenEndOptionsOperator) { final Optional option = checkForShortOption(argument.charAt(1), getRegisteredOptions()); if (!option.isPresent()) { if (this.ignoreUnknownOptions) { addBackArg = false; } else { throw new UnknownOptionException(argument.charAt(1)); } } } if (addBackArg) { modifiedArguments.add(argument); } } final SortedSet exceptionMessagesWeSaw = new TreeSet<>(); // Now we actually parse the arguments, assigning a context. for (final Integer context : this.registeredContexts) // NOSONAR { try { this.parseOptionsAndArguments(modifiedArguments, context); } catch (final Exception exception) { exceptionMessagesWeSaw.add(String.format("%d: %s (context %d)", context, exception.getMessage(), context)); continue; } this.currentContext = context; break; } if (this.currentContext == NO_CONTEXT) { throw new UnparsableContextException(exceptionMessagesWeSaw); } this.parseStepRanAtLeastOnce = true; } /** * Register an argument with a given arity. The argument hint is used as a key to retrieve the * argument value(s) later. Additionally, documentation generators can use the hint to create * more accurate doc pages. * * @param argumentHint * the hint for the argument * @param arity * the argument arity * @param optionality * whether the argument is optional or required * @param contexts * the contexts for this argument, if not provided then uses a default context * @throws CoreException * if the argument could not be registered */ public void registerArgument(final String argumentHint, final ArgumentArity arity, final ArgumentOptionality optionality, final Integer... contexts) { throwIfArgumentHintSeen(argumentHint); this.argumentHintsSeen.add(argumentHint); if (argumentHint == null || argumentHint.isEmpty()) { throw new CoreException("Argument hint cannot be null or empty"); } final String[] split = argumentHint.split("\\s+"); if (split.length > 1) { throw new CoreException("Option argument hint cannot contain whitespace"); } if (contexts.length == 0) { throw new CoreException("Must provide at least one context."); } for (int i = 0; i < contexts.length; i++) { registerArgumentHelper(contexts[i], argumentHint, arity, optionality); } } /** * Register a given context with no options or arguments. If the context already exists, this * will noop. * * @param context * the context to register */ public void registerEmptyContext(final int context) { if (this.registeredContexts.contains(context)) { logger.info("Tried to register empty context {}, but {} is already registered", context, context); return; } this.registeredContexts.add(context); this.contextToRegisteredOptions.put(context, new HashSet<>()); this.contextToRegisteredOptionalArgument.put(context, false); this.contextToArgumentHintToArity.put(context, new HashMap<>()); this.contextToArgumentHintToOptionality.put(context, new HashMap<>()); this.contextToRegisteredVariadicArgument.put(context, false); } /** * Register an option with a given long and short form. The option will be a flag option, ie. it * can take no arguments. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOption(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final Integer... contexts) { if (longForm != null) { throwIfDuplicateLongForm(longForm); this.longFormsSeen.add(longForm); } if (contexts.length == 0) { throw new CoreException(MUST_REGISTER_AT_LEAST_ONE_CONTEXT); } for (int i = 0; i < contexts.length; i++) { registerOptionHelper(contexts[i], longForm, shortForm, description, optionality, OptionArgumentType.NONE, null); } } /** * Register an option with a given long form. The option will be a flag option, ie. it can take * no arguments. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOption(final String longForm, final String description, final OptionOptionality optionality, final Integer... contexts) { this.registerOption(longForm, null, description, optionality, contexts); } /** * Register an option with a given long and short form that takes an optional argument. The * provided argument hint can be used for generated documentation, and should be a single word * describing the argument. The parser will throw an exception at parse-time if the argument is * not supplied. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOptionWithOptionalArgument(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (longForm != null) { throwIfDuplicateLongForm(longForm); this.longFormsSeen.add(longForm); } if (shortForm != null) { throwIfDuplicateShortForm(shortForm); this.shortFormsSeen.add(shortForm); } if (contexts.length == 0) { throw new CoreException(MUST_REGISTER_AT_LEAST_ONE_CONTEXT); } for (int i = 0; i < contexts.length; i++) { registerOptionHelper(contexts[i], longForm, shortForm, description, optionality, OptionArgumentType.OPTIONAL, argumentHint); } } /** * Register an option with a given long form that takes an optional argument. The provided * argument hint can be used for generated documentation, and should be a single word describing * the argument. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOptionWithOptionalArgument(final String longForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (longForm != null) { throwIfDuplicateLongForm(longForm); this.longFormsSeen.add(longForm); } if (contexts.length == 0) { throw new CoreException(MUST_REGISTER_AT_LEAST_ONE_CONTEXT); } for (int i = 0; i < contexts.length; i++) { registerOptionHelper(contexts[i], longForm, null, description, optionality, OptionArgumentType.OPTIONAL, argumentHint); } } /** * Register an option with a given long and short form that takes a required argument. The * provided argument hint can be used for generated documentation, and should be a single word * describing the argument. The parser will throw an exception at parse-time if the argument is * not supplied. * * @param longForm * the long form of the option, eg. --option * @param shortForm * the short form of the option, eg. -o * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOptionWithRequiredArgument(final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (longForm != null) { throwIfDuplicateLongForm(longForm); this.longFormsSeen.add(longForm); } if (shortForm != null) { throwIfDuplicateShortForm(shortForm); this.shortFormsSeen.add(shortForm); } if (contexts.length == 0) { throw new CoreException(MUST_REGISTER_AT_LEAST_ONE_CONTEXT); } for (int i = 0; i < contexts.length; i++) { registerOptionHelper(contexts[i], longForm, shortForm, description, optionality, OptionArgumentType.REQUIRED, argumentHint); } } /** * Register an option with a given long form that takes a required argument. The provided * argument hint can be used for generated documentation, and should be a single word describing * the argument. The parser will throw an exception at parse-time if the argument is not * supplied. * * @param longForm * the long form of the option, eg. --option * @param description * a simple description * @param optionality * the optionality * @param argumentHint * the hint for the argument * @param contexts * the contexts for this option, if not provided then uses a default context * @throws CoreException * if the option could not be registered */ public void registerOptionWithRequiredArgument(final String longForm, final String description, final OptionOptionality optionality, final String argumentHint, final Integer... contexts) { if (longForm != null) { throwIfDuplicateLongForm(longForm); this.longFormsSeen.add(longForm); } if (contexts.length == 0) { throw new CoreException(MUST_REGISTER_AT_LEAST_ONE_CONTEXT); } for (int i = 0; i < contexts.length; i++) { registerOptionHelper(contexts[i], longForm, null, description, optionality, OptionArgumentType.REQUIRED, argumentHint); } } private Optional checkForLongOption(final String longForm, final Set setToCheck, final boolean usePrefixMatching) throws AmbiguousAbbreviationException { final Set matchedOptions = new HashSet<>(); for (final SimpleOption option : setToCheck) { if (option.getLongForm().startsWith(longForm)) { /* * Break out if we find an exact match. This handles the edge case where you have * two options like "--option" and "--optionSuffix". In this case, if "--option" is * supplied, we want to return the exact match instead of throwing an ambiguity * error. */ if (option.getLongForm().equals(longForm)) { return Optional.of(option); } if (usePrefixMatching) { matchedOptions.add(option); } } } if (matchedOptions.size() > 1) { final List ambiguousOptions = matchedOptions.stream() .map(SimpleOption::getLongForm).collect(Collectors.toList()); throw new AmbiguousAbbreviationException(longForm, new StringList(ambiguousOptions).join(", ")); } else if (matchedOptions.size() == 1) { final SimpleOption matchedOption = matchedOptions.toArray(new SimpleOption[0])[0]; return Optional.of(matchedOption); } return Optional.empty(); } private Optional checkForShortOption(final Character shortForm, final Set setToCheck) { for (final SimpleOption option : setToCheck) { final Optional optionalForm = option.getShortForm(); if (optionalForm.isPresent() && optionalForm.get().equals(shortForm)) { return Optional.of(option); } } return Optional.empty(); } private Optional getParsedOptionFromLongForm(final String longForm) throws UnknownOptionException { if (!this.parseStepRanAtLeastOnce) { throw new CoreException(CANNOT_GET_OPTIONS_BEFORE_PARSING); } final Optional option; try { if (!registeredOptionForLongForm(this.currentContext, longForm).isPresent()) { throw new UnknownOptionException(longForm); } option = checkForLongOption(longForm, this.parsedOptions.keySet(), false); } catch (final AmbiguousAbbreviationException exception) { throw new CoreException(PROVIDED_OPTION_LONG_FORM_WAS_AMBIGUOUS, longForm); } return option; } /* * This function returns a boolean value specifying whether or not it consumed the lookahead * value. */ private boolean parseLongFormOption(final int tryContext, final String argument, // NOSONAR final Optional lookahead) throws UnknownOptionException, OptionParseException, AmbiguousAbbreviationException { final String scrubbedPrefix = argument.substring(LONG_FORM_PREFIX.length()); final String[] split = scrubbedPrefix.split(OPTION_ARGUMENT_DELIMITER, 2); final String optionName = split[0]; final Optional option = registeredOptionForLongForm(tryContext, optionName); if (option.isPresent()) { // Split length is 1 if command line looks like "... --option anotherThing ..." // Split length is > 1 if command line looks like "... --option=arg anotherThing ..." // Split length will never be < 1 if (split.length == 1) { // Cases to handle here regarding the lookahead // 1) The option takes no argument or an optional argument -> do not use lookahead // 2) The option takes a required argument -> attempt to use lookahead // Once done, we return whether or not we used the lookahead switch (option.get().getArgumentType()) { case NONE: // fallthru intended case OPTIONAL: this.parsedOptions.put(option.get(), Optional.empty()); return false; case REQUIRED: if (lookahead.isPresent()) { this.parsedOptions.put(option.get(), lookahead); return true; } else { throw new OptionParseException("option \'" + option.get().getLongForm() // NOSONAR + "\' needs an argument"); } default: throw new CoreException("Unrecognized OptionArgumentType {}", option.get().getArgumentType()); } } else { // Cases to handle here // 1) The option takes no argument -> throw an error // 2) The option takes an optional or required argument -> use the split final String optionArgument = split[1]; switch (option.get().getArgumentType()) { case NONE: throw new OptionParseException( "option \'" + option.get().getLongForm() + "\' takes no argument"); case OPTIONAL: // fallthru intended case REQUIRED: this.parsedOptions.put(option.get(), Optional.ofNullable(optionArgument)); return false; default: throw new CoreException("Unrecognized OptionArgumentType {}", option.get().getArgumentType()); } } } else { throw new UnknownOptionException(optionName); } } /** * Perform a full scan and parse of the provided arguments list. This method will populate the * parser's internal data structures so they are ready to be queried for results. This method * tries to parse the arguments within a supplied context. * * @param allArguments * The provided arguments list * @param tryContext * the context to try * @throws UnknownOptionException * If an unknown option is detected * @throws OptionParseException * If another parsing error occurs * @throws ArgumentException * If supplied arguments do not match the registered argument hints * @throws AmbiguousAbbreviationException * If an ambiguous long option abbreviation was used */ private void parseOptionsAndArguments(final List allArguments, final int tryContext) // NOSONAR throws UnknownOptionException, OptionParseException, ArgumentException, AmbiguousAbbreviationException { final List regularArguments = new ArrayList<>(); boolean seenEndOptionsOperator = false; this.parsedArguments.clear(); this.parsedOptions.clear(); int regularArgumentCounter = 0; boolean skipNextArgument = false; for (int index = 0; index < allArguments.size(); index++) { if (skipNextArgument) { skipNextArgument = false; continue; } skipNextArgument = false; final String argument = allArguments.get(index); // We store a lookahead to use in case of an option with the argument specified like // "--option optarg". In this case we will need the lookahead value. Optional lookahead = Optional.empty(); if (index + 1 < allArguments.size()) { lookahead = Optional.ofNullable(allArguments.get(index + 1)); } // Five cases: // Argument is "--" -> stop parsing arguments as options // Argument is "-" -> treat as a regular argument // Argument starts with "--" -> long form option // Argument starts with "-" -> short form option // Anything else -> regular argument if (END_OPTIONS_OPERATOR.equals(argument)) { if (seenEndOptionsOperator) { regularArguments.add(argument); } else { seenEndOptionsOperator = true; } } else if (SHORT_FORM_PREFIX.equals(argument)) { regularArguments.add(argument); } else if (argument.startsWith(LONG_FORM_PREFIX) && !seenEndOptionsOperator) { final boolean consumedLookahead = parseLongFormOption(tryContext, argument, lookahead); if (consumedLookahead) { skipNextArgument = true; } } else if (argument.startsWith(SHORT_FORM_PREFIX) && !seenEndOptionsOperator) { final boolean consumedLookahead = parseShortFormOption(tryContext, argument, lookahead); if (consumedLookahead) { skipNextArgument = true; } } else { regularArguments.add(argument); } } // Check that any option registered as required is actually present. If not, throw an error. final Set registeredOptions = this.contextToRegisteredOptions.get(tryContext); if (registeredOptions != null) { for (final SimpleOption registeredOption : registeredOptions) { if (registeredOption.getOptionality() == OptionOptionality.REQUIRED && !this.parsedOptions.keySet().contains(registeredOption)) { throw new OptionParseException( "missing required option " + registeredOption.longForm); } } } if (this.contextToRegisteredOptionalArgument.getOrDefault(tryContext, false)) { if (this.contextToArgumentHintToArity.containsKey(tryContext) && regularArguments .size() < this.contextToArgumentHintToArity.get(tryContext).size() - 1) { throw new ArgumentException("missing required argument(s)"); } } else { if (this.contextToArgumentHintToArity.containsKey(tryContext) && regularArguments .size() < this.contextToArgumentHintToArity.get(tryContext).size()) { throw new ArgumentException("missing required argument(s)"); } } // Now handle the regular arguments for (final String regularArgument : regularArguments) { regularArgumentCounter = parseRegularArgument(tryContext, regularArgument, regularArguments.size(), regularArgumentCounter); } this.parseStepRanAtLeastOnce = true; } private int parseRegularArgument(final int context, final String argument, final int regularArgumentSize, final int regularArgumentCounter) throws ArgumentException { int argumentCounter = regularArgumentCounter; if (!this.contextToArgumentHintToArity.containsKey(context)) { throw new ArgumentException("too many arguments"); } if (this.contextToArgumentHintToArity.containsKey(context) && argumentCounter >= this.contextToArgumentHintToArity.get(context).size()) { throw new ArgumentException("too many arguments"); } final String argumentHint = (String) this.contextToArgumentHintToArity.get(context).keySet() .toArray()[argumentCounter]; final ArgumentArity currentArity = this.contextToArgumentHintToArity.get(context) .get(argumentHint); switch (currentArity) { case UNARY: logger.debug("parsed unary argument hint => {} : value => {}", argumentHint, argument); this.parsedArguments.put(argumentHint, Arrays.asList(argument)); argumentCounter++; break; case VARIADIC: List multiArgumentList = this.parsedArguments.get(argumentHint); multiArgumentList = multiArgumentList == null ? new ArrayList<>() : multiArgumentList; multiArgumentList.add(argument); logger.debug("parsed variadic argument hint => {} : value => {}", argumentHint, argument); this.parsedArguments.put(argumentHint, multiArgumentList); // Two cases: // Case 1 -> [UNARY...] VARIADIC if (argumentCounter == this.contextToArgumentHintToArity.get(context).size() - 1) { // do nothing, we can consume the rest of the arguments } // Case 2 -> [UNARY...] VARIADIC UNARY [UNARY...] else { // cutoff point, be sure to save arguments for consumption by subsequent hints if (multiArgumentList.size() == regularArgumentSize - this.contextToArgumentHintToArity.get(context).size() + 1) { argumentCounter++; break; } } break; default: throw new CoreException("Unrecognized ArgumentArity {}", currentArity); } return argumentCounter; } /* * This function returns a boolean value specifying whether or not it consumed the lookahead * value. */ private boolean parseShortFormOption(final int context, final String argument, // NOSONAR final Optional lookahead) throws OptionParseException, UnknownOptionException { final String scrubbedPrefix = argument.substring(SHORT_FORM_PREFIX.length()); // Two cases // 1) command line looks like "... -o arg ..." // 2) command line looks like "... -oarg ..." // scrubbedPrefix length will never be < 1 // Case 1) "... -o arg ..." if (scrubbedPrefix.length() == 1) { final Optional option = registeredOptionForShortForm(context, scrubbedPrefix.charAt(0)); if (!option.isPresent()) { throw new UnknownOptionException(scrubbedPrefix.charAt(0)); } // 3 cases to handle here regarding the option argument type // a) The option takes no argument -> do not use lookahead // b) The option takes an optional argument -> do not use lookahead // c) The option takes a required argument -> attempt to use lookahead // Once done, we return whether or not we used the lookahead switch (option.get().getArgumentType()) { case NONE: // fallthru intended case OPTIONAL: this.parsedOptions.put(option.get(), Optional.empty()); return false; case REQUIRED: if (lookahead.isPresent()) { this.parsedOptions.put(option.get(), lookahead); return true; } else { throw new OptionParseException("option \'" + option.get().getShortForm().get() + "\' needs an argument"); // NOSONAR } default: throw new CoreException("Bad OptionArgumentType {}", option.get().getArgumentType()); } } // Case 2) "... -oarg ..." else { // Cases to handle here // a) The option is using bundling, ie. ("-oarg" meaning "-o -a -r -g") // b) The option is using an argument, ie. ("-oarg" where "arg" is an argument to "-o") // Check for case a) determine if valid bundle boolean isValidBundle = true; for (int index = 0; index < scrubbedPrefix.length(); index++) // NOSONAR { final char optionCharacter = scrubbedPrefix.charAt(index); final Optional option = registeredOptionForShortForm(context, optionCharacter); if (option.isPresent()) { if (option.get().getArgumentType() != OptionArgumentType.NONE) { isValidBundle = false; break; } } else { isValidBundle = false; break; } } if (isValidBundle) { // Bundle was valid, so loop over again and add all options for (int index = 0; index < scrubbedPrefix.length(); index++) { final char optionCharacter = scrubbedPrefix.charAt(index); final Optional option = registeredOptionForShortForm(context, optionCharacter); this.parsedOptions.put(option.get(), Optional.empty()); // NOSONAR } } else { // Bundle was not valid, so treat remaining chars as an option arg final char optionCharacter = scrubbedPrefix.charAt(0); final Optional option = registeredOptionForShortForm(context, optionCharacter); if (!option.isPresent()) { throw new UnknownOptionException(String.valueOf(optionCharacter).charAt(0)); } if (option.get().getArgumentType() == OptionArgumentType.NONE) { throw new OptionParseException("option \'" + option.get().getShortForm().get() // NOSONAR + "\' takes no argument"); } final String optionArgument = scrubbedPrefix.substring(1); this.parsedOptions.put(option.get(), Optional.ofNullable(optionArgument)); } return false; } } private void registerArgumentHelper(final int context, final String argumentHint, final ArgumentArity arity, final ArgumentOptionality optionality) { if (context < 0) { throw new CoreException("Context ID must be a positive integer"); } if (this.contextToRegisteredOptionalArgument.getOrDefault(context, false)) { throw new CoreException("Optional argument must be the last registered argument"); } if (arity == ArgumentArity.VARIADIC && this.contextToRegisteredVariadicArgument.getOrDefault(context, false)) { throw new CoreException("Cannot register more than one variadic argument"); } if (optionality == ArgumentOptionality.OPTIONAL) { if (this.contextToRegisteredOptionalArgument.getOrDefault(context, false)) { throw new CoreException("Cannot register more than one optional argument"); } if (this.contextToRegisteredVariadicArgument.getOrDefault(context, false)) { throw new CoreException( "Cannot register both an optional argument and a variadic argument"); } this.contextToRegisteredOptionalArgument.put(context, true); } if (arity == ArgumentArity.VARIADIC) { this.contextToRegisteredVariadicArgument.put(context, true); } final Map argumentHintToArity = this.contextToArgumentHintToArity .get(context) == null ? new LinkedHashMap<>() : this.contextToArgumentHintToArity.get(context); argumentHintToArity.put(argumentHint, arity); this.contextToArgumentHintToArity.put(context, argumentHintToArity); final Map argumentHintToOptionality = this.contextToArgumentHintToOptionality .get(context) == null ? new LinkedHashMap<>() : this.contextToArgumentHintToOptionality.get(context); argumentHintToOptionality.put(argumentHint, optionality); this.contextToArgumentHintToOptionality.put(context, argumentHintToOptionality); this.registeredContexts.add(context); } private void registerOptionHelper(final int context, final String longForm, final Character shortForm, final String description, final OptionOptionality optionality, final OptionArgumentType type, final String argumentHint) { if (context <= 0) { throw new CoreException("Context ID must be a positive integer (>= 1)"); } final Set registeredOptionsForContext = this.contextToRegisteredOptions .get(context) == null ? new HashSet<>() : this.contextToRegisteredOptions.get(context); registeredOptionsForContext.add(new SimpleOption(longForm, shortForm, description, optionality, type, argumentHint)); this.contextToRegisteredOptions.put(context, registeredOptionsForContext); this.registeredContexts.add(context); } private Optional registeredOptionForLongForm(final int context, final String longForm) throws AmbiguousAbbreviationException { return checkForLongOption(longForm, this.contextToRegisteredOptions.get(context), true); } private Optional registeredOptionForShortForm(final int context, final Character shortForm) { return checkForShortOption(shortForm, this.contextToRegisteredOptions.get(context)); } private void throwIfArgumentHintSeen(final String hint) { if (this.argumentHintsSeen.contains(hint)) { throw new CoreException("Cannot register argument hint {} more than once!", hint); } } private void throwIfDuplicateLongForm(final String longForm) { if (this.longFormsSeen.contains(longForm)) { throw new CoreException("Cannot register option {} more than once!", longForm); } } private void throwIfDuplicateShortForm(final Character shortForm) { if (this.shortFormsSeen.contains(shortForm)) { throw new CoreException("Cannot register option {} more than once!", shortForm); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/exceptions/AmbiguousAbbreviationException.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing.exceptions; /** * @author lcram */ public class AmbiguousAbbreviationException extends Exception { private static final long serialVersionUID = 8506034533362610699L; public AmbiguousAbbreviationException(final String option, final String ambiguousOptions) { super("long option \'" + option + "\' is ambiguous (" + ambiguousOptions + ")"); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/exceptions/ArgumentException.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing.exceptions; /** * @author lcram */ public class ArgumentException extends Exception { private static final long serialVersionUID = 8506034533362610699L; public ArgumentException(final String message) { super(message); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/exceptions/OptionParseException.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing.exceptions; /** * @author lcram */ public class OptionParseException extends Exception { private static final long serialVersionUID = 2471393426772482019L; public OptionParseException(final String message) { super(message); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/exceptions/UnknownOptionException.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing.exceptions; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.utilities.command.parsing.SimpleOptionAndArgumentParser.SimpleOption; /** * @author lcram */ public class UnknownOptionException extends Exception { private static final long serialVersionUID = 8506034533362610699L; private static Optional closestMatchMessage(final String option, final Set validOptions) { final Set optionNames = validOptions.stream().map(SimpleOption::getLongForm) .collect(Collectors.toSet()); String closestOption = null; int minimumDistance = Integer.MAX_VALUE; for (final String optionName : optionNames) { final int distance = StringUtils.getLevenshteinDistance(option, optionName); if (distance < minimumDistance) { closestOption = optionName; minimumDistance = distance; } } if (closestOption == null) { return Optional.empty(); } return Optional.of(", did you mean \'" + closestOption + "\'?"); } public UnknownOptionException(final Character option) { super("unknown short option \'" + option + "\'"); } public UnknownOptionException(final String option) { super("unknown long option \'" + option + "\'"); } public UnknownOptionException(final String option, final Set validOptions) { super("unknown long option \'" + option + "\'" + closestMatchMessage(option, validOptions).orElse("")); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/parsing/exceptions/UnparsableContextException.java ================================================ package org.openstreetmap.atlas.utilities.command.parsing.exceptions; import java.util.SortedSet; import org.openstreetmap.atlas.utilities.collections.StringList; /** * @author lcram */ public class UnparsableContextException extends Exception { private static final long serialVersionUID = 8204676424116770097L; public UnparsableContextException(final SortedSet exceptionMessagesWeSaw) { super("could not match command line to a usage context: " + System.getProperty("line.separator") + new StringList(exceptionMessagesWeSaw) .join(System.getProperty("line.separator"))); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AnyToGeoJsonCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.boundary.converters.CountryBoundaryMapGeoJsonConverter; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.CountryBoundaryMapTemplate; import org.openstreetmap.atlas.utilities.command.subcommands.templates.OutputDirectoryTemplate; import org.openstreetmap.atlas.utilities.command.subcommands.templates.ShardingTemplate; import com.google.gson.GsonBuilder; /** * This command converts our many different file formats to a GeoJSON representation. This may be * useful for various visualization software. * * @author lcram */ public class AnyToGeoJsonCommand extends AbstractAtlasShellToolsCommand { private static final String ATLAS_OPTION_LONG = "atlas"; private static final String ATLAS_OPTION_DESCRIPTION = "The path to an atlas file to be converted."; private static final String ATLAS_OPTION_HINT = "atlas-file"; private static final String COUNTRIES_OPTION_LONG = "countries"; private static final Character COUNTRIES_OPTION_SHORT = 'c'; private static final String COUNTRIES_OPTION_DESCRIPTION = "A comma separated list of allowlist country codes to exclusively include. Defaults to all."; private static final String COUNTRIES_OPTION_HINT = "included-countries"; private static final String COUNTRIES_DENY_LIST_OPTION_LONG = "countries-denylist"; private static final Character COUNTRIES_DENY_LIST_OPTION_SHORT = 'C'; private static final String COUNTRIES_DENY_LIST_OPTION_DESCRIPTION = "A comma separated denylist of country codes to explicitly exclude. Defaults to none."; private static final String COUNTRIES_DENY_LIST_OPTION_HINT = "excluded-countries"; private static final String POLYGONS_OPTION_LONG = "use-polygons"; private static final Character POLYGONS_OPTION_SHORT = 'p'; private static final String POLYGONS_OPTION_DESCRIPTION = "Use polygons instead of linestrings for the boundary GeoJSON. This may be better for certain visualization software."; private static final Integer ATLAS_CONTEXT = 3; private static final Integer SHARDING_CONTEXT = 4; private static final Integer BOUNDARY_CONTEXT = 5; private static final String OUTPUT_FILE = "output"; private static final String ATLAS_FILE = OUTPUT_FILE + "-" + ATLAS_OPTION_LONG + FileSuffix.GEO_JSON; private static final String SHARDING_FILE = OUTPUT_FILE + "-sharding" + FileSuffix.GEO_JSON; private static final String BOUNDARY_FILE = OUTPUT_FILE + "-country-boundary" + FileSuffix.GEO_JSON; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AnyToGeoJsonCommand().runSubcommandAndExit(args); } public AnyToGeoJsonCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { if (this.optionAndArgumentDelegate.getParserContext() == ATLAS_CONTEXT) { return executeAtlasContext(); } else if (this.optionAndArgumentDelegate.getParserContext() == SHARDING_CONTEXT) { return executeShardingContext(); } else if (this.optionAndArgumentDelegate.getParserContext() == BOUNDARY_CONTEXT) { return executeBoundaryContext(); } else { throw new AtlasShellToolsException(); } } @Override public String getCommandName() { return "any2geojson"; } @Override public String getSimpleDescription() { return "convert a custom file format to GeoJSON"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasSearchCommand.class .getResourceAsStream("AnyToGeoJsonCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasSearchCommand.class .getResourceAsStream("AnyToGeoJsonCommandExamplesSection.txt")); registerManualPageSectionsFromTemplate(new ShardingTemplate()); registerManualPageSectionsFromTemplate(new CountryBoundaryMapTemplate()); registerManualPageSectionsFromTemplate(new OutputDirectoryTemplate()); } @Override public void registerOptionsAndArguments() { registerOptionWithRequiredArgument(ATLAS_OPTION_LONG, ATLAS_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, ATLAS_OPTION_HINT, ATLAS_CONTEXT); registerOptionWithRequiredArgument(COUNTRIES_OPTION_LONG, COUNTRIES_OPTION_SHORT, COUNTRIES_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, COUNTRIES_OPTION_HINT, BOUNDARY_CONTEXT); registerOptionWithRequiredArgument(COUNTRIES_DENY_LIST_OPTION_LONG, COUNTRIES_DENY_LIST_OPTION_SHORT, COUNTRIES_DENY_LIST_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, COUNTRIES_DENY_LIST_OPTION_HINT, BOUNDARY_CONTEXT); registerOption(POLYGONS_OPTION_LONG, POLYGONS_OPTION_SHORT, POLYGONS_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, BOUNDARY_CONTEXT); registerOptionsAndArgumentsFromTemplate(new ShardingTemplate(SHARDING_CONTEXT)); registerOptionsAndArgumentsFromTemplate(new CountryBoundaryMapTemplate(BOUNDARY_CONTEXT)); registerOptionsAndArgumentsFromTemplate( new OutputDirectoryTemplate(ATLAS_CONTEXT, SHARDING_CONTEXT, BOUNDARY_CONTEXT)); super.registerOptionsAndArguments(); } private int executeAtlasContext() { final File atlasFile = new File(this.optionAndArgumentDelegate .getOptionArgument(ATLAS_OPTION_LONG).orElseThrow(AtlasShellToolsException::new), this.getFileSystem()); if (!atlasFile.exists()) { this.outputDelegate .printlnErrorMessage("file not found: " + atlasFile.getAbsolutePathString()); return 1; } final Atlas atlas = new AtlasResourceLoader().load(atlasFile); final Optional pathOptional = OutputDirectoryTemplate.getOutputPath(this); if (pathOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("could not save atlas file"); return 1; } final Path concatenatedPath = Paths.get(pathOptional.get().toAbsolutePath().toString(), ATLAS_FILE); final File outputFile = new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage( "writing the atlas geojson file to " + outputFile.toAbsolutePath().toString()); } atlas.saveAsLineDelimitedGeoJsonFeatures(outputFile, (entity, json) -> { // Dummy consumer, we don't need to mutate the JSON }); return 0; } private int executeBoundaryContext() { Set countries = new HashSet<>(); if (this.optionAndArgumentDelegate.hasOption(COUNTRIES_OPTION_LONG)) { countries = this.optionAndArgumentDelegate .getOptionArgument(COUNTRIES_OPTION_LONG, this::parseCommaSeparatedCountries) .orElse(new HashSet<>()); } final boolean usePolygons = this.optionAndArgumentDelegate.hasOption(POLYGONS_OPTION_LONG); Set countriesDenyList = new HashSet<>(); if (this.optionAndArgumentDelegate.hasOption(COUNTRIES_DENY_LIST_OPTION_LONG)) { countriesDenyList = this.optionAndArgumentDelegate .getOptionArgument(COUNTRIES_DENY_LIST_OPTION_LONG, this::parseCommaSeparatedCountries) .orElse(new HashSet<>()); } CountryBoundaryMap map = null; final Optional mapOptional = CountryBoundaryMapTemplate .getCountryBoundaryMap(this); if (mapOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("failed to load country boundary"); return 1; } map = mapOptional.get(); final String boundaryJson; if (countries.isEmpty()) { boundaryJson = new CountryBoundaryMapGeoJsonConverter().prettyPrint(true) .withCountryDenyList(countriesDenyList).usePolygons(usePolygons) .convertToString(map); } else { boundaryJson = new CountryBoundaryMapGeoJsonConverter().withCountryAllowList(countries) .withCountryDenyList(countriesDenyList).prettyPrint(true) .usePolygons(usePolygons).convertToString(map); } if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("converting boundary file to GeoJSON..."); } final Optional pathOptional = OutputDirectoryTemplate.getOutputPath(this); if (pathOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("could not save boundary file"); return 1; } final Path concatenatedPath = Paths.get(pathOptional.get().toAbsolutePath().toString(), BOUNDARY_FILE); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("writing the boundary geojson file to " + concatenatedPath.toAbsolutePath().toString()); } new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()) .writeAndClose(boundaryJson); return 0; } private int executeShardingContext() { final Sharding sharding = ShardingTemplate.getSharding(this); final String shardingJson = new GsonBuilder().setPrettyPrinting().create() .toJson(sharding.asGeoJson()); final Optional pathOptional = OutputDirectoryTemplate.getOutputPath(this); if (pathOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("could not save sharding tree"); return 1; } final Path concatenatedPath = Paths.get(pathOptional.get().toAbsolutePath().toString(), SHARDING_FILE); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("writing the sharding geojson file to " + concatenatedPath.toAbsolutePath().toString()); } new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()) .writeAndClose(shardingJson); return 0; } private Set parseCommaSeparatedCountries(final String countryString) { final Set countrySet = new HashSet<>(); if (countryString.isEmpty()) { return countrySet; } countrySet.addAll(Arrays.stream(countryString.split(",")).collect(Collectors.toSet())); return countrySet; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasDiffCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.change.Change; import org.openstreetmap.atlas.geography.atlas.change.ChangeBuilder; import org.openstreetmap.atlas.geography.atlas.change.FeatureChange; import org.openstreetmap.atlas.geography.atlas.change.diff.AtlasDiff; import org.openstreetmap.atlas.geography.atlas.change.serializer.ChangeGeoJsonSerializer; import org.openstreetmap.atlas.geography.atlas.complete.PrettifyStringFormat; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import com.google.common.collect.Sets; /** * @author lcram */ public class AtlasDiffCommand extends AbstractAtlasShellToolsCommand { /** * @author matthieun */ private static class AtlasDiffCommandContext { private final File beforeAtlasFile; private final File afterAtlasFile; private final boolean useGeoJson; private final boolean useLdGeoJson; private final boolean fullText; private final Long selectedIdentifier; private final ItemType selectedType; private final boolean recursive; AtlasDiffCommandContext(final File beforeAtlasFile, final File afterAtlasFile, // NOSONAR final boolean useGeoJson, final boolean useLdGeoJson, final boolean fullText, final Long selectedIdentifier, final ItemType selectedType, final boolean recursive) { this.beforeAtlasFile = beforeAtlasFile; this.afterAtlasFile = afterAtlasFile; this.useGeoJson = useGeoJson; this.useLdGeoJson = useLdGeoJson; this.fullText = fullText; this.selectedIdentifier = selectedIdentifier; this.selectedType = selectedType; this.recursive = recursive; } public File getAfterAtlasFile() { return this.afterAtlasFile; } public File getBeforeAtlasFile() { return this.beforeAtlasFile; } public Long getSelectedIdentifier() { return this.selectedIdentifier; } public ItemType getSelectedType() { return this.selectedType; } public boolean isFullText() { return this.fullText; } public boolean isRecursive() { return this.recursive; } public boolean isUseGeoJson() { return this.useGeoJson; } public boolean isUseLdGeoJson() { return this.useLdGeoJson; } } static final String NO_CHANGE = "atlases are effectively identical"; private static final String BEFORE_ATLAS_ARGUMENT = "before-atlas(es)"; private static final String AFTER_ATLAS_ARGUMENT = "after-atlas(es)"; private static final List FORMAT_TYPE_STRINGS = Arrays .stream(PrettifyStringFormat.values()).map(PrettifyStringFormat::toString) .collect(Collectors.toList()); private static final PrettifyStringFormat DEFAULT_PRETTY_FEATURE_CHANGE_FORMAT = PrettifyStringFormat.MINIMAL_MULTI_LINE; private static final PrettifyStringFormat DEFAULT_PRETTY_COMPLETE_ENTITY_FORMAT = PrettifyStringFormat.MINIMAL_SINGLE_LINE; private static final String FEATURE_CHANGE_FORMAT_OPTION_LONG = "feature-change-format"; private static final String COMPLETE_ENTITY_FORMAT_OPTION_LONG = "complete-entity-format"; private static final String FEATURE_CHANGE_FORMAT_OPTION_HINT = "format"; private static final String COMPLETE_ENTITY_FORMAT_OPTION_HINT = "format"; private static final String FEATURE_CHANGE_FORMAT_OPTION_DESCRIPTION = "The format type for the constituent FeatureChanges. Valid settings are: " + new StringList(FORMAT_TYPE_STRINGS).join(", ") + ". Defaults to " + DEFAULT_PRETTY_FEATURE_CHANGE_FORMAT.toString() + "."; private static final String COMPLETE_ENTITY_FORMAT_OPTION_DESCRIPTION = "The format type for the CompleteEntities within the constituent FeatureChanges. Valid settings are: " + new StringList(FORMAT_TYPE_STRINGS).join(", ") + ". Defaults to " + DEFAULT_PRETTY_COMPLETE_ENTITY_FORMAT.toString() + "."; private static final String LDGEOJSON_OPTION_LONG = "ldgeojson"; private static final String LDGEOJSON_OPTION_DESCRIPTION = "Use the line-delimited geoJSON format for output."; private static final String GEOJSON_OPTION_LONG = "geojson"; private static final String GEOJSON_OPTION_DESCRIPTION = "Use the pretty geoJSON format for output."; private static final String FULL_OPTION_LONG = "full"; private static final String FULL_OPTION_DESCRIPTION = "Show the full FeatureChange instead of just the ChangeDescription."; private static final String FOLDER_SEARCH_RECURSIVE_OPTION_LONG = "recursive"; private static final String FOLDER_SEARCH_RECURSIVE_OPTION_DESCRIPTION = "When comparing Atlas folders, search sub-folders too."; private static final List ITEM_TYPE_STRINGS = Arrays.stream(ItemType.values()) .map(ItemType::toString).collect(Collectors.toList()); private static final String TYPE_OPTION_LONG = "type"; private static final String TYPE_OPTION_DESCRIPTION = "The ItemType of the desired feature. Valid types are: " + new StringList(ITEM_TYPE_STRINGS).join(", ") + "."; private static final String TYPE_OPTION_HINT = "type"; private static final String ID_OPTION_LONG = "id"; private static final String ID_OPTION_DESCRIPTION = "The identifier of the desired feature."; private static final String ID_OPTION_HINT = "id"; private static final Integer LDGEOJSON_CONTEXT = 4; private static final Integer GEOJSON_CONTEXT = 5; private static final Integer FULL_CONTEXT = 6; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AtlasDiffCommand().runSubcommandAndExit(args); } public AtlasDiffCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() // NOSONAR { final String beforeAtlasPath = this.optionAndArgumentDelegate .getUnaryArgument(BEFORE_ATLAS_ARGUMENT).orElseThrow(AtlasShellToolsException::new); final String afterAtlasPath = this.optionAndArgumentDelegate .getUnaryArgument(AFTER_ATLAS_ARGUMENT).orElseThrow(AtlasShellToolsException::new); final File beforeAtlasFile = new File(beforeAtlasPath, this.getFileSystem()); final File afterAtlasFile = new File(afterAtlasPath, this.getFileSystem()); boolean useGeoJson = false; boolean useLdGeoJson = false; boolean fullText = false; Long selectedIdentifier = null; ItemType selectedType = null; boolean recursive = false; if (this.optionAndArgumentDelegate.hasOption( ID_OPTION_LONG) != this.optionAndArgumentDelegate.hasOption(TYPE_OPTION_LONG)) { this.outputDelegate.printlnErrorMessage("options --" + ID_OPTION_LONG + " and --" + TYPE_OPTION_LONG + " must be supplied together or not at all"); return 2; } if (this.optionAndArgumentDelegate.hasOption(ID_OPTION_LONG)) { final String idString = this.optionAndArgumentDelegate.getOptionArgument(ID_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new); try { selectedIdentifier = Long.parseLong(idString); } catch (final Exception exception) { this.outputDelegate.printlnErrorMessage("could not parse id " + idString); return 2; } } if (this.optionAndArgumentDelegate.hasOption(TYPE_OPTION_LONG)) { final String typeString = this.optionAndArgumentDelegate .getOptionArgument(TYPE_OPTION_LONG).orElseThrow(AtlasShellToolsException::new) .toUpperCase(); try { selectedType = ItemType.valueOf(typeString); } catch (final Exception exception) { this.outputDelegate.printlnErrorMessage("could not parse id " + typeString); return 2; } } if (this.optionAndArgumentDelegate.getParserContext() == GEOJSON_CONTEXT) { useGeoJson = true; } if (this.optionAndArgumentDelegate.getParserContext() == LDGEOJSON_CONTEXT) { useLdGeoJson = true; } if (this.optionAndArgumentDelegate.getParserContext() == FULL_CONTEXT) { fullText = true; } if (this.optionAndArgumentDelegate.hasOption(FOLDER_SEARCH_RECURSIVE_OPTION_LONG)) { recursive = true; } if (!beforeAtlasFile.exists()) { this.outputDelegate.printlnWarnMessage("file not found: " + beforeAtlasPath); return 2; } if (!afterAtlasFile.exists()) { this.outputDelegate.printlnWarnMessage("file not found: " + afterAtlasPath); return 2; } final AtlasDiffCommandContext context = new AtlasDiffCommandContext(beforeAtlasFile, afterAtlasFile, useGeoJson, useLdGeoJson, fullText, selectedIdentifier, selectedType, recursive); if (beforeAtlasFile.isDirectory() && afterAtlasFile.isDirectory()) { final int result = this.compute(context, beforeAtlasFile, afterAtlasFile); return result > 0 ? 1 : 0; } else if (!beforeAtlasFile.isDirectory() && !afterAtlasFile.isDirectory()) { final Atlas beforeAtlas = load(beforeAtlasFile); final Atlas afterAtlas = load(afterAtlasFile); final int result = this.compute(context, beforeAtlas, afterAtlas); return result > 0 ? 1 : 0; } else { this.outputDelegate.printlnErrorMessage("Cannot compare a file and a directory."); return 1; } } @Override public String getCommandName() { return "atlas-diff"; } @Override public String getSimpleDescription() { return "compare two atlas files"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasDiffCommand.class .getResourceAsStream("AtlasDiffCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasDiffCommand.class.getResourceAsStream("AtlasDiffCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerOptionWithRequiredArgument(TYPE_OPTION_LONG, TYPE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, TYPE_OPTION_HINT, DEFAULT_CONTEXT, GEOJSON_CONTEXT, LDGEOJSON_CONTEXT, FULL_CONTEXT); registerOptionWithRequiredArgument(ID_OPTION_LONG, ID_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, ID_OPTION_HINT, DEFAULT_CONTEXT, GEOJSON_CONTEXT, LDGEOJSON_CONTEXT, FULL_CONTEXT); registerOptionWithRequiredArgument(FEATURE_CHANGE_FORMAT_OPTION_LONG, FEATURE_CHANGE_FORMAT_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, FEATURE_CHANGE_FORMAT_OPTION_HINT, FULL_CONTEXT); registerOptionWithRequiredArgument(COMPLETE_ENTITY_FORMAT_OPTION_LONG, COMPLETE_ENTITY_FORMAT_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, COMPLETE_ENTITY_FORMAT_OPTION_HINT, FULL_CONTEXT); registerOption(LDGEOJSON_OPTION_LONG, LDGEOJSON_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, LDGEOJSON_CONTEXT); registerOption(GEOJSON_OPTION_LONG, GEOJSON_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, GEOJSON_CONTEXT); registerOption(FULL_OPTION_LONG, FULL_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, FULL_CONTEXT); registerOption(FOLDER_SEARCH_RECURSIVE_OPTION_LONG, FOLDER_SEARCH_RECURSIVE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, DEFAULT_CONTEXT, FULL_CONTEXT, GEOJSON_CONTEXT, LDGEOJSON_CONTEXT); registerArgument(BEFORE_ATLAS_ARGUMENT, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED, DEFAULT_CONTEXT, LDGEOJSON_CONTEXT, GEOJSON_CONTEXT, FULL_CONTEXT); registerArgument(AFTER_ATLAS_ARGUMENT, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED, DEFAULT_CONTEXT, LDGEOJSON_CONTEXT, GEOJSON_CONTEXT, FULL_CONTEXT); super.registerOptionsAndArguments(); } int compute(final AtlasDiffCommandContext context, final File beforeAtlasFile, final File afterAtlasFile) { int result = 0; final Map beforeNamesToFiles = new HashMap<>(); final Map afterNamesToFiles = new HashMap<>(); final List beforeFilesToConsider = context.isRecursive() ? beforeAtlasFile.listFilesRecursively(false) : beforeAtlasFile.listFiles(false); final List afterFilesToConsider = context.isRecursive() ? afterAtlasFile.listFilesRecursively(false) : afterAtlasFile.listFiles(false); beforeFilesToConsider.stream().filter(this::checkAtlas).forEach( file -> beforeNamesToFiles.put(getRelativeFileName(beforeAtlasFile, file), file)); afterFilesToConsider.stream().filter(this::checkAtlas).forEach( file -> afterNamesToFiles.put(getRelativeFileName(afterAtlasFile, file), file)); final Set filesOnlyInBefore = Sets.difference(beforeNamesToFiles.keySet(), afterNamesToFiles.keySet()); final Set filesOnlyInAfter = Sets.difference(afterNamesToFiles.keySet(), beforeNamesToFiles.keySet()); final Set filesInBoth = Sets.intersection(beforeNamesToFiles.keySet(), afterNamesToFiles.keySet()); if (!filesOnlyInBefore.isEmpty()) { final String warnMessage = "Files only in Before Atlas folder:"; this.outputDelegate.printlnWarnMessage(warnMessage); filesOnlyInBefore.stream().sorted().forEach(this.outputDelegate::printlnWarnMessage); result += filesOnlyInBefore.size(); } if (!filesOnlyInAfter.isEmpty()) { final String warnMessage = "Files only in After Atlas folder:"; this.outputDelegate.printlnWarnMessage(warnMessage); filesOnlyInAfter.stream().sorted().forEach(this.outputDelegate::printlnWarnMessage); result += filesOnlyInAfter.size(); } for (final String name : filesInBoth.stream().sorted().collect(Collectors.toList())) { final Atlas beforeAtlas = load(beforeNamesToFiles.get(name)); final Atlas afterAtlas = load(afterNamesToFiles.get(name)); this.outputDelegate.printlnStdout(name, TTYAttribute.BOLD, TTYAttribute.GREEN); result += compute(context, beforeAtlas, afterAtlas); } return result > 0 ? 1 : 0; } /** * @param context * The context to apply * @param beforeAtlas * The before Atlas to compare * @param afterAtlas * The after Atlas to compare * @return 1 if there are differences */ int compute(final AtlasDiffCommandContext context, final Atlas beforeAtlas, final Atlas afterAtlas) { final AtlasDiff diff = new AtlasDiff(beforeAtlas, afterAtlas).saveAllGeometries(false); final Optional changeOptional = diff.generateChange(); if (changeOptional.isPresent()) { final Optional trimmedChangeOption = trimChange(context, changeOptional.get()); final Change change; if (trimmedChangeOption.isPresent()) { change = trimmedChangeOption.get(); } else { return 0; } final String serializedString; if (context.isUseGeoJson()) { serializedString = new ChangeGeoJsonSerializer().convert(change); } else if (context.isUseLdGeoJson()) { serializedString = change.toLineDelimitedFeatureChanges(true); } else if (context.isFullText()) { final PrettifyStringFormat featureChangeFormat = this.optionAndArgumentDelegate .getOptionArgument(FEATURE_CHANGE_FORMAT_OPTION_LONG, PrettifyStringFormat::valueOf) .orElse(DEFAULT_PRETTY_FEATURE_CHANGE_FORMAT); final PrettifyStringFormat completeEntityFormat = this.optionAndArgumentDelegate .getOptionArgument(COMPLETE_ENTITY_FORMAT_OPTION_LONG, PrettifyStringFormat::valueOf) .orElse(DEFAULT_PRETTY_COMPLETE_ENTITY_FORMAT); serializedString = change.prettify(featureChangeFormat, completeEntityFormat, false) + "\n"; } else { final StringBuilder builder = new StringBuilder(); change.changes().forEach(featureChange -> { builder.append(featureChange.explain()); builder.append("\n"); }); serializedString = builder.toString(); } this.outputDelegate.printlnStdout(serializedString); return change.changeCount() > 0 ? 1 : 0; } else { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage(NO_CHANGE); } return 0; } } private boolean checkAtlas(final Resource resource) { return AtlasResourceLoader.HAS_ATLAS_EXTENSION.test(resource) || AtlasResourceLoader.HAS_TEXT_ATLAS_EXTENSION.test(resource); } private String getRelativeFileName(final File parent, final File file) { return file.getAbsolutePathString().substring(parent.getAbsolutePathString().length() + 1); } private Atlas load(final File file) { return new AtlasResourceLoader() .load(new InputStreamResource(file::read).withName(file.getAbsolutePathString())); } private Optional trimChange(final AtlasDiffCommandContext context, final Change change) { if (this.optionAndArgumentDelegate.hasOption(ID_OPTION_LONG) && this.optionAndArgumentDelegate.hasOption(TYPE_OPTION_LONG)) { final Optional featureChangeOptional = change .changeFor(context.getSelectedType(), context.getSelectedIdentifier()); if (featureChangeOptional.isPresent()) { return Optional.of(new ChangeBuilder().add(featureChangeOptional.get()).get()); } else { final String stdoutMessage = "No change found for " + context.getSelectedType() + " " + context.getSelectedIdentifier(); this.outputDelegate.printlnWarnMessage(stdoutMessage); return Optional.empty(); } } return Optional.of(change); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasMetadataReaderCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderCommand; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; /** * @author lcram */ public class AtlasMetadataReaderCommand extends AtlasLoaderCommand { private static final String SIZE_OPTION_LONG = "size"; private static final String SIZE_OPTION_DESCRIPTION = "Show feature array sizes."; private static final String ORIGINAL_OPTION_LONG = "original"; private static final String ORIGINAL_OPTION_DESCRIPTION = "Show value of 'original' field."; private static final String CODE_VERSION_OPTION_LONG = "code-version"; private static final String CODE_VERSION_OPTION_DESCRIPTION = "Show the code version."; private static final String DATA_VERSION_OPTION_LONG = "data-version"; private static final String DATA_VERSION_OPTION_DESCRIPTION = "Show the data version."; private static final String COUNTRY_OPTION_LONG = "country"; private static final String COUNTRY_OPTION_DESCRIPTION = "Show country(s)."; private static final String SHARD_OPTION_LONG = "shard"; private static final String SHARD_OPTION_DESCRIPTION = "Show shard(s)."; private static final String TAGS_OPTION_LONG = "tags"; private static final String TAGS_OPTION_DESCRIPTION = "Show metadata tags."; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AtlasMetadataReaderCommand().runSubcommandAndExit(args); } public AtlasMetadataReaderCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public String getCommandName() { return "atlas-metadata-reader"; } @Override public String getSimpleDescription() { return "read selected fields from the atlas metadata"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasMetadataReaderCommand.class .getResourceAsStream("AtlasMetadataReaderCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasMetadataReaderCommand.class .getResourceAsStream("AtlasMetadataReaderCommandExamplesSection.txt")); super.registerManualPageSections(); } @Override public void registerOptionsAndArguments() { registerOption(SIZE_OPTION_LONG, SIZE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(ORIGINAL_OPTION_LONG, ORIGINAL_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(CODE_VERSION_OPTION_LONG, CODE_VERSION_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(DATA_VERSION_OPTION_LONG, DATA_VERSION_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(COUNTRY_OPTION_LONG, COUNTRY_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(SHARD_OPTION_LONG, SHARD_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(TAGS_OPTION_LONG, TAGS_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); super.registerOptionsAndArguments(); } @Override protected void processAtlas(final Atlas atlas, final String atlasFileName, final File atlasResource) { this.outputDelegate.printlnStdout(atlasResource.getPathString() + " metadata:", TTYAttribute.BOLD); if (this.optionAndArgumentDelegate.hasOption(SIZE_OPTION_LONG)) { final StringBuilder builder = new StringBuilder(); builder.append("Size: "); builder.append("\n\tNodes: "); builder.append(atlas.metaData().getSize().getNodeNumber()); builder.append("\n\tEdges: "); builder.append(atlas.metaData().getSize().getEdgeNumber()); builder.append("\n\tAreas: "); builder.append(atlas.metaData().getSize().getAreaNumber()); builder.append("\n\tLines: "); builder.append(atlas.metaData().getSize().getLineNumber()); builder.append("\n\tPoints: "); builder.append(atlas.metaData().getSize().getPointNumber()); builder.append("\n\tRelations: "); builder.append(atlas.metaData().getSize().getRelationNumber()); this.outputDelegate.printlnStdout(builder.toString(), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(ORIGINAL_OPTION_LONG)) { this.outputDelegate.printlnStdout("Original: " + atlas.metaData().isOriginal(), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(CODE_VERSION_OPTION_LONG)) { this.outputDelegate.printlnStdout( "Code Version: " + atlas.metaData().getCodeVersion().orElse("null"), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(DATA_VERSION_OPTION_LONG)) { this.outputDelegate.printlnStdout( "Data Version: " + atlas.metaData().getDataVersion().orElse("null"), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(COUNTRY_OPTION_LONG)) { this.outputDelegate.printlnStdout( "Country: " + atlas.metaData().getCountry().orElse("null"), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(SHARD_OPTION_LONG)) { this.outputDelegate.printlnStdout( "Shard: " + atlas.metaData().getShardName().orElse("null"), TTYAttribute.GREEN); } if (this.optionAndArgumentDelegate.hasOption(TAGS_OPTION_LONG)) { final StringBuilder builder = new StringBuilder(); final SortedSet sortedTags = atlas.metaData().getTags().entrySet().stream() .map(entry -> entry.getKey() + " -> " + entry.getValue()) .collect(Collectors.toCollection(TreeSet::new)); builder.append(new StringList(sortedTags).join("\n\t")); this.outputDelegate.printlnStdout("Tags:\n\t" + builder.toString(), TTYAttribute.GREEN); } // If none of the specific options are supplied, print everything if (!this.optionAndArgumentDelegate.hasOption(SIZE_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(ORIGINAL_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(CODE_VERSION_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(DATA_VERSION_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(COUNTRY_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(SHARD_OPTION_LONG) && !this.optionAndArgumentDelegate.hasOption(TAGS_OPTION_LONG)) { this.outputDelegate.printlnStdout(atlas.metaData().toReadableString(), TTYAttribute.GREEN); } else { this.outputDelegate.printlnStdout(""); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasSearchCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.Point; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.complete.CompleteEntity; import org.openstreetmap.atlas.geography.atlas.complete.PrettifyStringFormat; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.LineItem; import org.openstreetmap.atlas.geography.atlas.items.LocationItem; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Relation; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.utilities.collections.Sets; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderCommand; import org.openstreetmap.atlas.utilities.command.subcommands.templates.PredicateTemplate; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.openstreetmap.atlas.utilities.tuples.Tuple; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; /** * Search atlases for some given feature identifiers or properties, with various options and * restrictions. Draws some inspiration from similar identifier locator commands by cstaylor and * bbreithaupt. * * @author lcram */ // Some future improvements // + fix RelationMember delimiting so that roles with ';' won't break everything? public class AtlasSearchCommand extends AtlasLoaderCommand { /** * @author lcram */ private static class RelationMemberSearchConstraint { private ItemType type = null; private Long identifier = null; private String role = null; RelationMemberSearchConstraint() { } /** * Check if this constraint matches at least one member of a given {@link Relation}'s member * list. * * @param relation * the {@link Relation} against which to check * @return if there is a match */ boolean matches(final Relation relation) { boolean constraintMatchedAnItem = false; for (final RelationBean.RelationBeanItem item : relation.getBean()) { boolean constraintMatchedType = true; if (this.type != null && !this.type.equals(item.getType())) { constraintMatchedType = false; } boolean constraintMatchedId = true; if (this.identifier != null && !this.identifier.equals(item.getIdentifier())) { constraintMatchedId = false; } boolean constraintMatchedRole = true; if (this.role != null && !this.role.equals(item.getRole())) { constraintMatchedRole = false; } constraintMatchedAnItem = constraintMatchedType && constraintMatchedId && constraintMatchedRole; /* * Break early if we match an item, since we don't need to check any more. */ if (constraintMatchedAnItem) { break; } } return constraintMatchedAnItem; } RelationMemberSearchConstraint withId(final Long identifier) { this.identifier = identifier; return this; } RelationMemberSearchConstraint withRole(final String role) { this.role = role; return this; } RelationMemberSearchConstraint withType(final ItemType type) { this.type = type; return this; } } private static final List ITEM_TYPE_STRINGS = Arrays.stream(ItemType.values()) .map(ItemType::toString).collect(Collectors.toList()); private static final String TYPES_OPTION_LONG = "type"; private static final String TYPES_OPTION_DESCRIPTION = "A comma separated list of ItemTypes by which to narrow the search. Valid types are: " + new StringList(ITEM_TYPE_STRINGS).join(", ") + ". Defaults to including all values, unless another option (e.g. --startNode) automatically narrows the search space."; private static final String TYPES_OPTION_HINT = "types"; private static final String BOUNDING_POLYGON_OPTION_LONG = "bounding-polygons"; private static final String BOUNDING_POLYGON_OPTION_DESCRIPTION = "Match all features within at least one member of a given colon separated list of bounding polygons."; private static final String BOUNDING_POLYGON_OPTION_HINT = "wkt-polygons"; private static final String GEOMETRY_OPTION_LONG = "geometry"; private static final String GEOMETRY_OPTION_DESCRIPTION = "A colon separated list of exact geometry WKTs for which to search."; private static final String GEOMETRY_OPTION_HINT = "wkt-geometry"; private static final String SUB_GEOMETRY_OPTION_LONG = "sub-geometry"; private static final String SUB_GEOMETRY_OPTION_DESCRIPTION = "Like --geometry, but can match against contained geometry. E.g. POINT(2 2) would match LINESTRING(1 1, 2 2, 3 3)."; private static final String SUB_GEOMETRY_OPTION_HINT = "wkt-geometry"; private static final String TAGGABLEFILTER_OPTION_LONG = "tag-filter"; private static final String TAGGABLEFILTER_OPTION_DESCRIPTION = "A TaggableFilter by which to filter the search space."; private static final String TAGGABLEFILTER_OPTION_HINT = "filter"; private static final String TAGGABLEMATCHER_OPTION_LONG = "tag-matcher"; private static final String TAGGABLEMATCHER_OPTION_DESCRIPTION = "A TaggableMatcher by which to filter the search space."; private static final String TAGGABLEMATCHER_OPTION_HINT = "matcher"; private static final String STARTNODE_OPTION_LONG = "start-nodes"; private static final String STARTNODE_OPTION_DESCRIPTION = "A comma separated list of start node identifiers for which to search."; private static final String STARTNODE_OPTION_HINT = "ids"; private static final String ENDNODE_OPTION_LONG = "end-nodes"; private static final String ENDNODE_OPTION_DESCRIPTION = "A comma separated list of end node identifiers for which to search."; private static final String ENDNODE_OPTION_HINT = "ids"; private static final String INEDGE_OPTION_LONG = "in-edges"; private static final String INEDGE_OPTION_DESCRIPTION = "A comma separated list of in edge identifiers for which to search."; private static final String INEDGE_OPTION_HINT = "ids"; private static final String OUTEDGE_OPTION_LONG = "out-edges"; private static final String OUTEDGE_OPTION_DESCRIPTION = "A comma separated list of out edge identifiers for which to search."; private static final String OUTEDGE_OPTION_HINT = "ids"; private static final String PARENT_RELATIONS_OPTION_LONG = "parent-relations"; private static final String PARENT_RELATIONS_OPTION_DESCRIPTION = "A comma separated list of parent relation identifiers for which to search."; private static final String PARENT_RELATIONS_OPTION_HINT = "ids"; private static final String RELATION_MEMBERS_OR_OPTION_LONG = "relation-members"; private static final String RELATION_MEMBERS_OR_OPTION_DESCRIPTION = "Filter to relations that contain at least one of the given semicolon separated members." + " Members can be specified like e.g. `AREA,1234,myrole;EDGE,4567,*;*,9012,*'." + " Here you can see you may optionally supply a `*' wildcard to broaden the search constraints." + " This example string filters for relations that contain an Area with ID 1234 and role `myrole', OR an Edge with ID 4567 and any role, OR any feature type with ID 9012 and any role."; private static final String RELATION_MEMBERS_OR_OPTION_HINT = "member[;member]..."; private static final String RELATION_MEMBERS_AND_OPTION_LONG = "and-relation-members"; private static final String RELATION_MEMBERS_AND_OPTION_DESCRIPTION = "Filter to relations that contain all of the given semicolon separated members." + " Members can be specified like e.g. `EDGE,*,to;EDGE,*,from;NODE,1234,via'." + " Here you can see you may optionally supply a `*' wildcard to broaden the search constraints." + " This example string filters for relations that contain a `to' AND `from' Edge with any ID, as well as a Node 1234 with role `via'."; private static final String RELATION_MEMBERS_AND_OPTION_HINT = "member[;member]..."; private static final String ID_OPTION_LONG = "id"; private static final String ID_OPTION_DESCRIPTION = "A comma separated list of Atlas ids for which to search."; private static final String ID_OPTION_HINT = "ids"; private static final String OSMID_OPTION_LONG = "osmid"; private static final String OSMID_OPTION_DESCRIPTION = "A comma separated list of OSM ids for which to search."; private static final String OSMID_OPTION_HINT = "osmids"; private static final String ALL_OPTION_LONG = "all"; private static final String ALL_OPTION_DESCRIPTION = "Ignore all other criteria and just print all entities."; private static final String JSON_OPTION_LONG = "json"; private static final String JSON_OPTION_DESCRIPTION = "Print matches in a parsable JSON format. For e.g., try chaining output into `jq' for more flexibility."; private static final String OUTPUT_ATLAS = "collected-multi.atlas"; private static final String COLLECT_OPTION_LONG = "collect-matching"; private static final String COLLECT_OPTION_DESCRIPTION = "Collect all matching atlas files and save to a file using the MultiAtlas."; private static final Integer ALL_TYPES_CONTEXT = 3; private static final Integer EDGE_ONLY_CONTEXT = 4; private static final Integer NODE_ONLY_CONTEXT = 5; private static final Integer RELATION_ONLY_CONTEXT = 6; private static final Integer SHOW_ALL_CONTEXT = 7; private static final String COULD_NOT_PARSE = "could not parse %s '%s'"; private static final List IMPORTS_ALLOW_LIST = Arrays.asList( "org.openstreetmap.atlas.geography.atlas.items", "org.openstreetmap.atlas.tags.annotations", "org.openstreetmap.atlas.tags.annotations.validation", "org.openstreetmap.atlas.tags.annotations.extraction", "org.openstreetmap.atlas.tags", "org.openstreetmap.atlas.tags.names", "org.openstreetmap.atlas.geography", "org.openstreetmap.atlas.utilities.collections"); private static final String WILDCARD = "*"; private Set geometryWkts; private Set subGeometryWkts; private Set boundingWkts; private TaggableFilter taggableFilter; private TaggableMatcher taggableMatcher; private Set startNodeIds; private Set endNodeIds; private Set inEdgeIds; private Set outEdgeIds; private Set parentRelations; private Set relationMemberConstraintsOR; private Set relationMemberConstraintsAND; private Predicate predicate; private Set ids; private Set osmIds; private Set typesToCheckFromOption; private final Set impliedTypesToCheck; private final Set matchingAtlases; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AtlasSearchCommand().runSubcommandAndExit(args); } public AtlasSearchCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); this.geometryWkts = new HashSet<>(); this.subGeometryWkts = new HashSet<>(); this.boundingWkts = new HashSet<>(); this.startNodeIds = new HashSet<>(); this.endNodeIds = new HashSet<>(); this.inEdgeIds = new HashSet<>(); this.outEdgeIds = new HashSet<>(); this.parentRelations = new HashSet<>(); this.relationMemberConstraintsOR = new HashSet<>(); this.relationMemberConstraintsAND = new HashSet<>(); this.ids = new HashSet<>(); this.osmIds = new HashSet<>(); this.typesToCheckFromOption = new HashSet<>(); this.impliedTypesToCheck = new HashSet<>(); this.matchingAtlases = new HashSet<>(); } @Override public int finish() { if (this.optionAndArgumentDelegate.hasOption(COLLECT_OPTION_LONG) && !this.matchingAtlases.isEmpty()) { final Path concatenatedPath = this.getFileSystem() .getPath(getOutputPath().toAbsolutePath().toString(), OUTPUT_ATLAS); final File outputFile = new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()); final Atlas outputAtlas; if (this.matchingAtlases.size() == 1) { outputAtlas = new ArrayList<>(this.matchingAtlases).get(0); outputAtlas.save(outputFile); } else { outputAtlas = new MultiAtlas(this.matchingAtlases); new PackedAtlasCloner().cloneFrom(outputAtlas).save(outputFile); } if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate .printlnCommandMessage("saved to " + concatenatedPath.toString()); } } if (this.matchingAtlases.isEmpty()) { return 1; } return 0; } @Override public String getCommandName() { return "find"; } @Override public String getSimpleDescription() { return "find features with given identifiers or properties in given atlas(es)"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasSearchCommand.class .getResourceAsStream("AtlasSearchCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasSearchCommand.class .getResourceAsStream("AtlasSearchCommandExamplesSection.txt")); registerManualPageSectionsFromTemplate(new PredicateTemplate()); super.registerManualPageSections(); } @Override public void registerOptionsAndArguments() { registerOptionWithRequiredArgument(TYPES_OPTION_LONG, TYPES_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, TYPES_OPTION_HINT, ALL_TYPES_CONTEXT); registerOptionWithRequiredArgument(BOUNDING_POLYGON_OPTION_LONG, BOUNDING_POLYGON_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, BOUNDING_POLYGON_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(GEOMETRY_OPTION_LONG, GEOMETRY_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, GEOMETRY_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(SUB_GEOMETRY_OPTION_LONG, SUB_GEOMETRY_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, SUB_GEOMETRY_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(TAGGABLEFILTER_OPTION_LONG, TAGGABLEFILTER_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, TAGGABLEFILTER_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(TAGGABLEMATCHER_OPTION_LONG, TAGGABLEMATCHER_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, TAGGABLEMATCHER_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(STARTNODE_OPTION_LONG, STARTNODE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, STARTNODE_OPTION_HINT, EDGE_ONLY_CONTEXT); registerOptionWithRequiredArgument(ENDNODE_OPTION_LONG, ENDNODE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, ENDNODE_OPTION_HINT, EDGE_ONLY_CONTEXT); registerOptionWithRequiredArgument(INEDGE_OPTION_LONG, INEDGE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, INEDGE_OPTION_HINT, NODE_ONLY_CONTEXT); registerOptionWithRequiredArgument(OUTEDGE_OPTION_LONG, OUTEDGE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, OUTEDGE_OPTION_HINT, NODE_ONLY_CONTEXT); registerOptionWithRequiredArgument(PARENT_RELATIONS_OPTION_LONG, PARENT_RELATIONS_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, PARENT_RELATIONS_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(RELATION_MEMBERS_OR_OPTION_LONG, RELATION_MEMBERS_OR_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, RELATION_MEMBERS_OR_OPTION_HINT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(RELATION_MEMBERS_AND_OPTION_LONG, RELATION_MEMBERS_AND_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, RELATION_MEMBERS_AND_OPTION_HINT, RELATION_ONLY_CONTEXT); registerOptionsAndArgumentsFromTemplate(new PredicateTemplate(ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT)); registerOptionWithRequiredArgument(ID_OPTION_LONG, ID_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, ID_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOptionWithRequiredArgument(OSMID_OPTION_LONG, OSMID_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, OSMID_OPTION_HINT, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT); registerOption(ALL_OPTION_LONG, ALL_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, SHOW_ALL_CONTEXT); registerOption(JSON_OPTION_LONG, JSON_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, ALL_TYPES_CONTEXT, EDGE_ONLY_CONTEXT, NODE_ONLY_CONTEXT, RELATION_ONLY_CONTEXT, SHOW_ALL_CONTEXT); registerOption(COLLECT_OPTION_LONG, COLLECT_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); super.registerOptionsAndArguments(); } @Override public int start() { /* * Get the types we will need to search for, implied by the option context. So for example, * if the user supplied an option that indicates they are only interested in Relations (e.g. * --relation-members), then we can set the impliedTypesToCheck to contain only RELATION. */ if (this.optionAndArgumentDelegate.getParserContext() == ALL_TYPES_CONTEXT) { this.impliedTypesToCheck.addAll(Sets.hashSet(ItemType.values())); } else if (this.optionAndArgumentDelegate.getParserContext() == NODE_ONLY_CONTEXT) { this.impliedTypesToCheck.add(ItemType.NODE); } else if (this.optionAndArgumentDelegate.getParserContext() == EDGE_ONLY_CONTEXT) { this.impliedTypesToCheck.add(ItemType.EDGE); } else if (this.optionAndArgumentDelegate.getParserContext() == RELATION_ONLY_CONTEXT) { this.impliedTypesToCheck.add(ItemType.RELATION); } else if (this.optionAndArgumentDelegate.getParserContext() == SHOW_ALL_CONTEXT) { this.impliedTypesToCheck.addAll(Sets.hashSet(ItemType.values())); /* * If our context is from --all, we don't need to do anything else. */ return 0; } /* * Handle the various search properties. */ if (this.optionAndArgumentDelegate.getParserContext() == ALL_TYPES_CONTEXT) { this.typesToCheckFromOption = this.optionAndArgumentDelegate .getOptionArgument(TYPES_OPTION_LONG, this::parseCommaSeparatedItemTypes) .orElse(new HashSet<>()); } this.boundingWkts = this.optionAndArgumentDelegate .getOptionArgument(BOUNDING_POLYGON_OPTION_LONG, this::parseColonSeparatedWkts) .orElse(new HashSet<>()); this.geometryWkts = this.optionAndArgumentDelegate .getOptionArgument(GEOMETRY_OPTION_LONG, this::parseColonSeparatedWkts) .orElse(new HashSet<>()); this.subGeometryWkts = this.optionAndArgumentDelegate .getOptionArgument(SUB_GEOMETRY_OPTION_LONG, this::parseColonSeparatedWkts) .orElse(new HashSet<>()); this.taggableFilter = this.optionAndArgumentDelegate .getOptionArgument(TAGGABLEFILTER_OPTION_LONG, TaggableFilter::forDefinition) .orElse(null); this.taggableMatcher = this.optionAndArgumentDelegate .getOptionArgument(TAGGABLEMATCHER_OPTION_LONG, TaggableMatcher::from).orElse(null); if (this.optionAndArgumentDelegate.getParserContext() == EDGE_ONLY_CONTEXT) { this.startNodeIds = this.optionAndArgumentDelegate .getOptionArgument(STARTNODE_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); this.endNodeIds = this.optionAndArgumentDelegate .getOptionArgument(ENDNODE_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); } if (this.optionAndArgumentDelegate.getParserContext() == NODE_ONLY_CONTEXT) { this.inEdgeIds = this.optionAndArgumentDelegate .getOptionArgument(INEDGE_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); this.outEdgeIds = this.optionAndArgumentDelegate .getOptionArgument(OUTEDGE_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); } if (this.optionAndArgumentDelegate.getParserContext() == RELATION_ONLY_CONTEXT) { this.relationMemberConstraintsOR = this.optionAndArgumentDelegate .getOptionArgument(RELATION_MEMBERS_OR_OPTION_LONG, this::parseSemicolonSeparatedRelationMembers) .orElse(new HashSet<>()); this.relationMemberConstraintsAND = this.optionAndArgumentDelegate .getOptionArgument(RELATION_MEMBERS_AND_OPTION_LONG, this::parseSemicolonSeparatedRelationMembers) .orElse(new HashSet<>()); } this.parentRelations = this.optionAndArgumentDelegate .getOptionArgument(PARENT_RELATIONS_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); this.predicate = PredicateTemplate.getPredicate(AtlasEntity.class, IMPORTS_ALLOW_LIST, this) .orElse(null); /* * Handle identifier searches. */ this.ids = this.optionAndArgumentDelegate .getOptionArgument(ID_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); this.osmIds = this.optionAndArgumentDelegate .getOptionArgument(OSMID_OPTION_LONG, this::parseCommaSeparatedLongs) .orElse(new HashSet<>()); if (this.typesToCheckFromOption.isEmpty() && this.boundingWkts.isEmpty() && this.geometryWkts.isEmpty() && this.subGeometryWkts.isEmpty() && this.taggableFilter == null && this.taggableMatcher == null && this.startNodeIds.isEmpty() && this.endNodeIds.isEmpty() && this.inEdgeIds.isEmpty() && this.outEdgeIds.isEmpty() && this.relationMemberConstraintsOR.isEmpty() && this.relationMemberConstraintsAND.isEmpty() && this.parentRelations.isEmpty() && this.predicate == null && this.ids.isEmpty() && this.osmIds.isEmpty()) { this.outputDelegate .printlnErrorMessage("no filtering objects were successfully constructed"); return 1; } return 0; } @Override protected void processAtlas(final Atlas atlas, final String atlasFileName, // NOSONAR final File atlasResource) { List boundedEntities = null; if (!this.boundingWkts.isEmpty()) { boundedEntities = entitiesBoundedByWktGeometry(this.boundingWkts, atlas); } Iterable entitiesWeAreChecking = atlas.entities(); if (boundedEntities != null) { entitiesWeAreChecking = boundedEntities; } /* * This loop is O(N) (where N is the number of atlas entities), assuming the lists of * provided evaluation properties are much smaller than the size of the entity set. We try * every condition to see if we can falsify this entity as a match candidate. */ for (final AtlasEntity entity : entitiesWeAreChecking) // NOSONAR { boolean entityMatchesAllCriteriaSoFar = true; if (!this.impliedTypesToCheck.contains(entity.getType())) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && !this.typesToCheckFromOption.isEmpty() && !this.typesToCheckFromOption.contains(entity.getType())) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && this.taggableFilter != null && !this.taggableFilter.test(entity)) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && this.taggableMatcher != null && !this.taggableMatcher.test(entity)) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && !this.ids.isEmpty() && !this.ids.contains(entity.getIdentifier())) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && !this.osmIds.isEmpty() && !this.osmIds.contains(entity.getOsmIdentifier())) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && !this.geometryWkts.isEmpty()) { boolean matchedAtLeastOneWktGeometry = false; for (final String wkt : this.geometryWkts) { if (entityMatchesWktGeometry(entity, wkt)) { matchedAtLeastOneWktGeometry = true; } } if (!matchedAtLeastOneWktGeometry) { entityMatchesAllCriteriaSoFar = false; } } if (entityMatchesAllCriteriaSoFar && !this.subGeometryWkts.isEmpty()) { boolean containedAtLeastOneWktGeometry = false; for (final String wkt : this.subGeometryWkts) { if (entityContainsWktGeometry(entity, wkt)) { containedAtLeastOneWktGeometry = true; } } if (!containedAtLeastOneWktGeometry) { entityMatchesAllCriteriaSoFar = false; } } if (entityMatchesAllCriteriaSoFar && this.predicate != null && !this.predicate.test(entity)) { entityMatchesAllCriteriaSoFar = false; } if (entityMatchesAllCriteriaSoFar && this.optionAndArgumentDelegate.getParserContext() == NODE_ONLY_CONTEXT) { final Node node = (Node) entity; final Set intersectingInEdgeIdentifiers = com.google.common.collect.Sets .intersection(node.inEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toSet()), this.inEdgeIds); final Set intersectingOutEdgeIdentifiers = com.google.common.collect.Sets .intersection(node.outEdges().stream().map(Edge::getIdentifier) .collect(Collectors.toSet()), this.outEdgeIds); if (!this.inEdgeIds.isEmpty() && intersectingInEdgeIdentifiers.isEmpty()) { entityMatchesAllCriteriaSoFar = false; } if (!this.outEdgeIds.isEmpty() && intersectingOutEdgeIdentifiers.isEmpty()) { entityMatchesAllCriteriaSoFar = false; } } if (entityMatchesAllCriteriaSoFar && this.optionAndArgumentDelegate.getParserContext() == EDGE_ONLY_CONTEXT) { final Edge edge = (Edge) entity; if (!this.startNodeIds.isEmpty() && !this.startNodeIds.contains(edge.start().getIdentifier())) { entityMatchesAllCriteriaSoFar = false; } if (!this.endNodeIds.isEmpty() && !this.endNodeIds.contains(edge.end().getIdentifier())) { entityMatchesAllCriteriaSoFar = false; } } if (entityMatchesAllCriteriaSoFar && this.optionAndArgumentDelegate.getParserContext() == RELATION_ONLY_CONTEXT && !this.relationMemberConstraintsAND.isEmpty()) { final Relation relation = (Relation) entity; for (final RelationMemberSearchConstraint constraint : this.relationMemberConstraintsAND) { if (!constraint.matches(relation)) { entityMatchesAllCriteriaSoFar = false; break; } } } if (entityMatchesAllCriteriaSoFar && this.optionAndArgumentDelegate.getParserContext() == RELATION_ONLY_CONTEXT && !this.relationMemberConstraintsOR.isEmpty()) { final Relation relation = (Relation) entity; boolean foundMemberMatch = false; for (final RelationMemberSearchConstraint constraint : this.relationMemberConstraintsOR) { if (constraint.matches(relation)) { foundMemberMatch = true; break; } } if (!foundMemberMatch) { entityMatchesAllCriteriaSoFar = false; } } if (entityMatchesAllCriteriaSoFar && !this.parentRelations.isEmpty()) { final Set intersectingParentRelationIdentifiers = com.google.common.collect.Sets .intersection(entity.relations().stream().map(Relation::getIdentifier) .collect(Collectors.toSet()), this.parentRelations); if (intersectingParentRelationIdentifiers.isEmpty()) { entityMatchesAllCriteriaSoFar = false; } } /* * If we made it here while matching all criteria, then we can print a diagnostic * detailing the find. */ if (entityMatchesAllCriteriaSoFar) { this.matchingAtlases.add(atlas); if (this.optionAndArgumentDelegate.hasOption(JSON_OPTION_LONG)) { printEntityWithJSONFormat(entity, atlasResource, atlas); } else { printEntityWithHumanReadableFormat(entity, atlasResource); } } } } private List entitiesBoundedByWktGeometry(final Iterable wkts, final Atlas atlas) // NOSONAR { final List entities = new ArrayList<>(); for (final String wkt : wkts) // NOSONAR { final Geometry geometry = parseWkt(wkt); if (geometry == null) { continue; } Polygon inputPolygon = null; if (geometry instanceof org.locationtech.jts.geom.Polygon) { inputPolygon = new JtsPolygonConverter() .backwardConvert((org.locationtech.jts.geom.Polygon) geometry); } else { this.outputDelegate.printlnErrorMessage("--" + BOUNDING_POLYGON_OPTION_LONG + " only supports POLYGON, found " + geometry.getClass().getName()); continue; } for (final AtlasEntity withinEntity : atlas.entitiesWithin(inputPolygon)) { entities.add(withinEntity); } } return entities; } private boolean entityContainsWktGeometry(final AtlasEntity entity, final String wkt) // NOSONAR { if (entity.getType() == ItemType.RELATION) { return false; } final Geometry geometry = parseWkt(wkt); if (geometry == null) { return false; } Location inputLocation = null; PolyLine inputPolyline = null; if (geometry instanceof Point) { inputLocation = new JtsPointConverter().backwardConvert((Point) geometry); } else if (geometry instanceof LineString) { inputPolyline = new JtsPolyLineConverter().backwardConvert((LineString) geometry); } else { this.outputDelegate.printlnErrorMessage( "--" + SUB_GEOMETRY_OPTION_LONG + " only supports POINT and LINESTRING, found " + geometry.getClass().getName()); return false; } boolean matchedSomething; if (entity.getType() == ItemType.POINT || entity.getType() == ItemType.NODE) { final Location location = ((LocationItem) entity).getLocation(); if (inputLocation != null) { matchedSomething = location.equals(inputLocation); return matchedSomething; } } else if (entity.getType() == ItemType.LINE || entity.getType() == ItemType.EDGE) { final PolyLine line = ((LineItem) entity).asPolyLine(); if (inputLocation != null) { matchedSomething = line.contains(inputLocation); if (matchedSomething) { return true; } } if (inputPolyline != null) { matchedSomething = line.overlapsShapeOf(inputPolyline); return matchedSomething; } } else if (entity.getType() == ItemType.AREA) { final Polygon polygon = ((Area) entity).asPolygon(); if (inputLocation != null) { matchedSomething = polygon.contains(inputLocation); if (matchedSomething) { return true; } } if (inputPolyline != null) { matchedSomething = polygon.overlapsShapeOf(inputPolyline); return matchedSomething; } } return false; } private boolean entityMatchesWktGeometry(final AtlasEntity entity, final String wkt) // NOSONAR { if (entity.getType() == ItemType.RELATION) { return false; } final Geometry geometry = parseWkt(wkt); if (geometry == null) { return false; } Location inputLocation = null; PolyLine inputPolyLine = null; Polygon inputPolygon = null; if (geometry instanceof Point) { inputLocation = new JtsPointConverter().backwardConvert((Point) geometry); } else if (geometry instanceof LineString) { inputPolyLine = new JtsPolyLineConverter().backwardConvert((LineString) geometry); } else if (geometry instanceof org.locationtech.jts.geom.Polygon) { inputPolygon = new JtsPolygonConverter() .backwardConvert((org.locationtech.jts.geom.Polygon) geometry); } else { this.outputDelegate.printlnErrorMessage("--" + GEOMETRY_OPTION_LONG + " only supports POINT, LINESTRING, and POLYGON, found " + geometry.getClass().getName()); return false; } final boolean matchedSomething; if (entity.getType() == ItemType.POINT || entity.getType() == ItemType.NODE) { final Location location = ((LocationItem) entity).getLocation(); if (inputLocation != null) { matchedSomething = location.equals(inputLocation); return matchedSomething; } } else if (entity.getType() == ItemType.LINE || entity.getType() == ItemType.EDGE) { final PolyLine line = ((LineItem) entity).asPolyLine(); if (inputPolyLine != null) { matchedSomething = line.equals(inputPolyLine); return matchedSomething; } } else if (entity.getType() == ItemType.AREA) { final Polygon polygon = ((Area) entity).asPolygon(); if (inputPolygon != null) { matchedSomething = polygon.equals(inputPolygon); return matchedSomething; } } return false; } private Optional> extractIdFromMemberElement(final String element, final String member) { if (WILDCARD.equals(element)) { return Optional.of(new Tuple<>(WILDCARD, null)); } final long identifier; try { identifier = Long.parseLong(element); return Optional.of(new Tuple<>(element, identifier)); } catch (final NumberFormatException exception) { this.outputDelegate.printlnErrorMessage( "could not parse ID `" + element + "' from member `" + member + "'"); return Optional.empty(); } } private Optional> extractItemTypeFromMemberElement(final String element, final String member) { if (WILDCARD.equals(element)) { return Optional.of(new Tuple<>(WILDCARD, null)); } final ItemType type; try { type = ItemType.valueOf(element.toUpperCase()); return Optional.of(new Tuple<>(element, type)); } catch (final IllegalArgumentException exception) { this.outputDelegate.printlnErrorMessage( "could not parse ItemType `" + element + "' from member `" + member + "'"); return Optional.empty(); } } private Optional> extractRoleFromMemberElement(final String element) { if (WILDCARD.equals(element)) { return Optional.of(new Tuple<>(WILDCARD, null)); } /* * Allow users to specify escape sequences. This way, they may escape the '*', or escape the * '\' if they want to include these characters literally. */ final StringBuilder builder = new StringBuilder(); for (final int codePoint : element.codePoints().toArray()) { /* * Only check for backslash when we see a BMP code point. This way, our role string will * support emojis and other non-BMP characters. */ if (Character.isBmpCodePoint(codePoint) && ((char) codePoint) == '\\') { continue; } builder.appendCodePoint(codePoint); } return Optional.of(new Tuple<>(element, builder.toString())); } private Set parseColonSeparatedWkts(final String wktString) { final Set wktSet = new HashSet<>(); if (wktString.isEmpty()) { return wktSet; } final WKTReader reader = new WKTReader(); final String[] wktStringSplit = wktString.split(":"); for (final String wkt : wktStringSplit) { try { reader.read(wkt); wktSet.add(wkt); } catch (final ParseException exception) { this.outputDelegate.printlnErrorMessage(String.format(COULD_NOT_PARSE, "wkt", wkt)); return new HashSet<>(); } } return wktSet; } private Set parseCommaSeparatedItemTypes(final String typeString) { final Set typeSet = new HashSet<>(); if (typeString.isEmpty()) { return typeSet; } final String[] typeStringSplit = typeString.split(","); for (final String typeElement : typeStringSplit) { final ItemType type; try { type = ItemType.valueOf(typeElement.toUpperCase()); typeSet.add(type); } catch (final IllegalArgumentException exception) { this.outputDelegate.printlnErrorMessage( String.format(COULD_NOT_PARSE, "ItemType", typeElement)); return new HashSet<>(); } } return typeSet; } private Set parseCommaSeparatedLongs(final String idString) { final Set idSet = new HashSet<>(); if (idString.isEmpty()) { return idSet; } final String[] idStringSplit = idString.split(","); for (final String idElement : idStringSplit) { final long identifier; try { identifier = Long.parseLong(idElement); idSet.add(identifier); } catch (final NumberFormatException exception) { this.outputDelegate .printlnErrorMessage(String.format(COULD_NOT_PARSE, "id", idElement)); return new HashSet<>(); } } return idSet; } private Set parseSemicolonSeparatedRelationMembers( final String memberString) { final Set constraints = new HashSet<>(); if (memberString.isEmpty()) { return constraints; } final String[] memberStringSplit = memberString.split(";"); for (final String member : memberStringSplit) { final int expectedElementLength = 3; final String[] memberElements = member.split(","); if (memberElements.length != expectedElementLength) { this.outputDelegate.printlnErrorMessage( "invalid syntax for member string `" + memberString + "'"); return new HashSet<>(); } final Optional> itemType = extractItemTypeFromMemberElement( memberElements[0], member); final Optional> identifier = extractIdFromMemberElement( memberElements[1], member); final Optional> role = extractRoleFromMemberElement( memberElements[2]); if (itemType.isEmpty() || identifier.isEmpty() || role.isEmpty()) { return new HashSet<>(); } final RelationMemberSearchConstraint constraint = new RelationMemberSearchConstraint(); constraint.withType(itemType.get().getSecond()).withId(identifier.get().getSecond()) .withRole(role.get().getSecond()); constraints.add(constraint); } return constraints; } private Geometry parseWkt(final String wkt) { final WKTReader reader = new WKTReader(); final Geometry geometry; try { geometry = reader.read(wkt); return geometry; } catch (final ParseException exception) { this.outputDelegate.printlnErrorMessage("unable to parse `" + wkt + "' as WKT"); return null; } } private void printEntityWithHumanReadableFormat(final AtlasEntity entity, final File atlasResource) { this.outputDelegate.printlnStdout( "Found entity matching criteria in " + atlasResource.getPathString() + ":", TTYAttribute.BOLD); this.outputDelegate.printlnStdout(((CompleteEntity) CompleteEntity.from(entity)) .prettify(PrettifyStringFormat.MINIMAL_MULTI_LINE, false), TTYAttribute.GREEN); this.outputDelegate.printlnStdout(""); } private void printEntityWithJSONFormat(final AtlasEntity entity, final File atlasResource, final Atlas atlas) { final JsonObject outputObject = new JsonObject(); outputObject.addProperty("shard", atlas.metaData().getCountry().orElse("XUK") + "_" + atlas.metaData().getShardName().orElse("0-0-0")); outputObject.addProperty("path", atlasResource.getPathString()); outputObject.add("entity", ((CompleteEntity) CompleteEntity.from(entity)).toJson()); this.outputDelegate.printlnStdout( new GsonBuilder().disableHtmlEscaping().create().toJson(outputObject), TTYAttribute.GREEN); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasShardingConverterCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.collections.Iterables; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; /** * This command provides an easy way to change the sharding in which a folder of atlas files is * described. * * @author matthieun */ public class AtlasShardingConverterCommand extends AbstractAtlasShellToolsCommand { private static final String INPUT = "input"; private static final String INPUT_DESCRIPTION = "The input folder containing XXX_.atlas files"; private static final String OUTPUT = "output"; private static final String OUTPUT_DESCRIPTION = "The output folder where XXX_.atlas files will be saved"; private static final String INPUT_SHARDING = "inputSharding"; private static final String INPUT_SHARDING_DESCRIPTION = "The input sharding"; private static final String OUTPUT_SHARDING = "outputSharding"; private static final String OUTPUT_SHARDING_DESCRIPTION = "The output sharding"; private static final Pattern FILE_MATCHER = Pattern .compile("^[A-Za-z0-9]+_{1}([A-Za-z0-9]|-)+\\.atlas$"); private static final String EXCEPTION_MESSAGE = "{} needs to be specified."; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AtlasShardingConverterCommand().runSubcommandAndExit(args); } public AtlasShardingConverterCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { final File inputFolder = new File(this.optionAndArgumentDelegate.getOptionArgument(INPUT) .orElseThrow(() -> new CoreException(EXCEPTION_MESSAGE, INPUT)), this.getFileSystem()); final File outputFolder = new File(this.optionAndArgumentDelegate.getOptionArgument(OUTPUT) .orElseThrow(() -> new CoreException(EXCEPTION_MESSAGE, OUTPUT)), this.getFileSystem()); if (!inputFolder.exists()) { throw new CoreException("{} does not exist.", inputFolder); } if (outputFolder.exists()) { throw new CoreException("{} already exists.", outputFolder); } final Sharding inputSharding = Sharding .forString(this.optionAndArgumentDelegate.getOptionArgument(INPUT_SHARDING) .orElseThrow(() -> new CoreException(EXCEPTION_MESSAGE, INPUT_SHARDING))); final Sharding outputSharding = Sharding .forString(this.optionAndArgumentDelegate.getOptionArgument(OUTPUT_SHARDING) .orElseThrow(() -> new CoreException(EXCEPTION_MESSAGE, OUTPUT_SHARDING))); final List inputFiles = inputFolder.listFilesRecursively().stream() .filter(file -> FILE_MATCHER.matcher(file.getName()).matches()) .collect(Collectors.toList()); this.outputDelegate.printlnCommandMessage("Found input files: " + inputFiles); final Map inputShardToAtlas = new HashMap<>(); final Set countries = new HashSet<>(); final Set inputShards = inputFiles.stream().map(file -> { final StringList split = StringList.split(file.getName(), "_"); String shardName = split.get(1); shardName = shardName.substring(0, shardName.indexOf(FileSuffix.ATLAS.toString())); final Shard inputShard = inputSharding.shardForName(shardName); inputShardToAtlas.put(inputShard, file); countries.add(split.get(0)); return inputShard; }).collect(Collectors.toSet()); if (countries.size() > 1) { throw new CoreException("Found more than one country in the folder: {}", countries); } this.outputDelegate.printlnCommandMessage( "Found " + inputShards.size() + " input shards: " + inputShards); this.outputDelegate.printlnCommandMessage("Found country: " + countries.iterator().next()); final Set outputShards = inputShards.stream().flatMap( inputShard -> Iterables.asList(outputSharding.shards(inputShard.bounds())).stream()) .collect(Collectors.toSet()); if (outputShards.isEmpty()) { throw new CoreException("There are no resulting output shards."); } else { outputFolder.mkdirs(); } this.outputDelegate.printlnCommandMessage( "Found " + outputShards.size() + " output shards: " + outputShards); for (final Shard outputShard : outputShards) { this.outputDelegate.printlnCommandMessage("Processing output shard " + outputShard); final List inputAtlases = new ArrayList<>(); Iterables.stream(inputSharding.shards(outputShard.bounds())) .filter(inputShardToAtlas::containsKey) .forEach(inputShard -> inputAtlases.add(inputShardToAtlas.get(inputShard))); this.outputDelegate.printlnCommandMessage("Loading Atlas with " + inputAtlases); final Atlas combined = new AtlasResourceLoader().load(inputAtlases); final Optional result = combined.subAtlas(outputShard.bounds(), AtlasCutType.SOFT_CUT); final File outputFile = outputFolder.child( countries.iterator().next() + "_" + outputShard.getName() + FileSuffix.ATLAS); this.outputDelegate.printlnCommandMessage("Saving Atlas to " + outputFile); result.ifPresent(atlas -> atlas.save(outputFile)); } return 0; } @Override public String getCommandName() { return "sharding-converter"; } @Override public String getSimpleDescription() { return "translate Atlas files from one Sharding to another"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasShardingConverterCommand.class .getResourceAsStream("AtlasShardingConverterCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasShardingConverterCommand.class .getResourceAsStream("AtlasShardingConverterCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerOptionWithRequiredArgument(INPUT, INPUT_DESCRIPTION, OptionOptionality.REQUIRED, "/path/to/atlases"); registerOptionWithRequiredArgument(OUTPUT, OUTPUT_DESCRIPTION, OptionOptionality.REQUIRED, "/path/to/output"); registerOptionWithRequiredArgument(INPUT_SHARDING, INPUT_SHARDING_DESCRIPTION, OptionOptionality.REQUIRED, "type@parameter"); registerOptionWithRequiredArgument(OUTPUT_SHARDING, OUTPUT_SHARDING_DESCRIPTION, OptionOptionality.REQUIRED, "type@parameter"); super.registerOptionsAndArguments(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/AtlasShellToolsDemoCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.List; import java.util.Scanner; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; /** * @author lcram */ public class AtlasShellToolsDemoCommand extends AbstractAtlasShellToolsCommand { private static final int BREAKFAST_CONTEXT = 4; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new AtlasShellToolsDemoCommand().runSubcommandAndExit(args); } public AtlasShellToolsDemoCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { // Check if the parser context detected the breakfast usage if (this.optionAndArgumentDelegate.getParserContext() == BREAKFAST_CONTEXT) { executeBreakfastContext(); } else { executeLunchDinnerContext(); } return 0; } @Override public String getCommandName() { return "ast-demo"; } @Override public String getSimpleDescription() { return "a demo of the Atlas Shell Tools subcommand API and features"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasShellToolsDemoCommand.class .getResourceAsStream("AtlasShellToolsDemoCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasShellToolsDemoCommand.class .getResourceAsStream("AtlasShellToolsDemoCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { /* * Generally, it's better practice to declare option forms, descriptions, and hints in * static final Strings at the top of your class. However, this demo command declares them * using literals just for ease of tutorial. */ final String beerDescription = "Brand of your favorite beer. " + "Currently making this option description really long in" + " order to test out the autoformatting capabilities of" + " the DocumentationFormatter class."; setVersion("0.0.1"); // Register options/arguments for default lunch/dinner context registerOption("capitalize", 'c', "Capitalize the foods list.", OptionOptionality.OPTIONAL); registerOptionWithRequiredArgument("beer", beerDescription, OptionOptionality.OPTIONAL, "brand"); registerOptionWithOptionalArgument("cheese", 'C', // NOSONAR "Use cheese. Defaults to cheddar, but will accept a supplied alternative.", OptionOptionality.OPTIONAL, "type"); registerOptionWithRequiredArgument("repeat", 'R', "Repeat the food list N times.", OptionOptionality.OPTIONAL, "N"); registerArgument("favoriteMeal", ArgumentArity.UNARY, ArgumentOptionality.REQUIRED); registerArgument("favoriteFoods", ArgumentArity.VARIADIC, ArgumentOptionality.OPTIONAL); // Register options/arguments for an alternate breakfast use case registerOption("breakfast", 'b', "Use breakfast mode", OptionOptionality.REQUIRED, BREAKFAST_CONTEXT); registerArgument("favoriteBreakfastFood", ArgumentArity.UNARY, ArgumentOptionality.OPTIONAL, BREAKFAST_CONTEXT); /* * Always call super.registerOptionsAndArguments last. Some superclasses will attempt to * register options to all available parser contexts. You want to ensure that each super * class has access to the full set of parser contexts within its class hierarchy. For * example, the global superclass AbstractAtlasShellToolsCommand registers a '--verbose' * option to all parser contexts. By calling super.register... last, you ensure that the * '--verbose' registry code runs for every context you registered above. Had you called * super.register... first, the '--verbose' registry code would have missed any additional * contexts you registered here. */ super.registerOptionsAndArguments(); } private void executeBreakfastContext() { this.outputDelegate.printlnCommandMessage( "value of HOME environment variable: " + this.getEnvironmentValue("user.home")); final String breakfast = this.optionAndArgumentDelegate .getUnaryArgument("favoriteBreakfastFood").orElse("Default waffles :("); this.outputDelegate.printlnStdout("Using special breakfast mode:"); this.outputDelegate.printlnStdout(breakfast, TTYAttribute.BOLD); this.outputDelegate.printlnStdout("Now say something!"); this.outputDelegate.printStdout("> "); try (Scanner scanner = new Scanner(this.getInStream())) { final String input = scanner.nextLine(); this.outputDelegate.printlnStdout("You said: " + input); } } private void executeLunchDinnerContext() { // We registered favoriteFoods as variadic so it comes back as a List. final List foods = this.optionAndArgumentDelegate .getVariadicArgument("favoriteFoods"); // We registered favoriteMeal as REQUIRED so it is safe to unwrap the Optional. // The orElseThrow is just there to stop Sonar from complaining. final String meal = this.optionAndArgumentDelegate.getUnaryArgument("favoriteMeal") .orElseThrow(AtlasShellToolsException::new); this.outputDelegate.printStdout("I like meal "); this.outputDelegate.printStdout(meal, TTYAttribute.MAGENTA, TTYAttribute.BOLD, TTYAttribute.BLINK); this.outputDelegate.printlnStdout(" the best"); final int repeatDefault = 1; final int repeat = this.optionAndArgumentDelegate.getOptionArgument("repeat", value -> { final int parsed; try { parsed = Integer.parseInt(value); } catch (final Exception exception) { this.outputDelegate .printlnWarnMessage("failed to parse repeat argument, using default"); return null; } return parsed; }).orElse(repeatDefault); this.outputDelegate.printlnStdout("Favorite foods are:"); for (int index = 0; index < repeat; index++) { for (final String food : foods) { String mutableFood = food; if (this.optionAndArgumentDelegate.hasOption("capitalize")) { mutableFood = mutableFood.toUpperCase(); } this.outputDelegate.printlnStdout(mutableFood, TTYAttribute.BOLD); } } if (this.optionAndArgumentDelegate.hasOption("cheese")) { this.outputDelegate.printlnStdout("Using " + this.optionAndArgumentDelegate.getOptionArgument("cheese").orElse("cheddar") + " cheese"); } if (this.optionAndArgumentDelegate.hasOption("beer")) { this.outputDelegate .printlnStdout("Also ordering a beer, " + this.optionAndArgumentDelegate .getOptionArgument("beer").orElseThrow(AtlasShellToolsException::new)); } else { this.outputDelegate.printlnWarnMessage("beer skipped"); } this.outputDelegate.printStderr("Here is a closing stderr message\n", TTYAttribute.UNDERLINE); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/ConcatenateAtlasCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderCommand; /** * @author lcram */ public class ConcatenateAtlasCommand extends AtlasLoaderCommand { private static final String OUTPUT_ATLAS = "output.atlas"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; private final List atlases = new ArrayList<>(); public static void main(final String[] args) { new ConcatenateAtlasCommand().runSubcommandAndExit(args); } public ConcatenateAtlasCommand() { super(); this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public String getCommandName() { return "fatlas"; } @Override public String getSimpleDescription() { return "create and save a fatlas using the MultiAtlas"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", ConcatenateAtlasCommand.class .getResourceAsStream("ConcatenateAtlasCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", ConcatenateAtlasCommand.class .getResourceAsStream("ConcatenateAtlasCommandExamplesSection.txt")); super.registerManualPageSections(); } @Override protected int finish() { if (this.atlases.isEmpty()) { this.outputDelegate.printlnErrorMessage("could not load atlas(es)"); return 1; } final Atlas atlas = new MultiAtlas(this.atlases); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("cloning..."); } final PackedAtlas outputAtlas = new PackedAtlasCloner().cloneFrom(atlas); final Path concatenatedPath = Paths.get(getOutputPath().toAbsolutePath().toString(), OUTPUT_ATLAS); final File outputFile = new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()); outputAtlas.save(outputFile); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("saved to " + concatenatedPath.toString()); } return 0; } @Override protected void processAtlas(final Atlas atlas, final String atlasFileName, final File atlasResource) { this.atlases.add(atlas); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/CountryBoundaryMapPrinterCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.io.BufferedWriter; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Rectangle; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.jts.JtsMultiPolygonToMultiPolygonConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.subcommands.templates.CountryBoundaryMapTemplate; import org.openstreetmap.atlas.utilities.time.Time; /** * @author matthieun */ public class CountryBoundaryMapPrinterCommand extends AbstractAtlasShellToolsCommand { public static final String BOUNDARY_OPTION_LONG = "country-boundary"; public static void main(final String[] args) { new CountryBoundaryMapPrinterCommand().runSubcommandAndExit(args); } @Override public int execute() { final File boundaryFile = getBoundaryFile(); String boundaryFileName = boundaryFile.getName(); boundaryFileName = boundaryFileName.substring(0, boundaryFileName.indexOf('.')); final Optional boundariesOption = CountryBoundaryMapTemplate .getCountryBoundaryMap(this); final File outputFolder = boundaryFile.parent(); final File geojson = outputFolder.child(boundaryFileName + "-geojson"); geojson.mkdirs(); final File wkt = outputFolder.child(boundaryFileName + "-wkt"); wkt.mkdirs(); if (boundariesOption.isEmpty()) { getCommandOutputDelegate().printlnErrorMessage("Could not read boundary file!"); return 1; } final CountryBoundaryMap map = boundariesOption.get(); final Set countrySet = map.countryCodesOverlappingWith(Rectangle.MAXIMUM).stream() .collect(Collectors.toSet()); final GeometryFactory geometryFactory = new GeometryFactory(); for (final String country : countrySet) { final Time start = Time.now(); final Polygon[] polygons = map.countryBoundary(country).toArray(new Polygon[0]); final MultiPolygon multiPolygon = new MultiPolygon(polygons, geometryFactory); saveGeometry(wkt, geojson, country, multiPolygon); if (getOptionAndArgumentDelegate().hasVerboseOption()) { getCommandOutputDelegate() .printlnCommandMessage("Saved " + country + " in " + start.elapsedSince()); } } return 0; } @Override public String getCommandName() { return "boundary-itemizer"; } @Override public String getSimpleDescription() { return "Read a CountryBoundaryMap file and print each country to geojson and wkt"; } @Override public void registerManualPageSections() { registerManualPageSectionsFromTemplate(new CountryBoundaryMapTemplate()); addManualPageSection("DESCRIPTION", CountryBoundaryMapPrinterCommand.class .getResourceAsStream("CountryBoundaryMapPrinterCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", CountryBoundaryMapPrinterCommand.class .getResourceAsStream("CountryBoundaryMapPrinterCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerOptionsAndArgumentsFromTemplate(new CountryBoundaryMapTemplate()); super.registerOptionsAndArguments(); } private File getBoundaryFile() { return new File(getOptionAndArgumentDelegate() .getOptionArgument(CountryBoundaryMapTemplate.COUNTRY_BOUNDARY_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new), this.getFileSystem()); } private void save(final WritableResource output, final String string) { try (BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(output.write(), StandardCharsets.UTF_8))) { writer.write(string); } catch (final Exception e) { throw new CoreException("Could not save file {}", output.getName(), e); } } private void saveGeometry(final File wkt, final File geojson, final String name, final MultiPolygon multiPolygon) { save(wkt.child(name + FileSuffix.WKT), multiPolygon.toText()); final File countryFile = geojson.child(name + FileSuffix.GEO_JSON); new JtsMultiPolygonToMultiPolygonConverter().convert(multiPolygon) .saveAsGeoJson(countryFile); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/CountryShardToBoundsCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.List; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.WktPolygonConverter; import org.openstreetmap.atlas.geography.sharding.GeoHashSharding; import org.openstreetmap.atlas.geography.sharding.GeoHashTile; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.geography.sharding.SlippyTileSharding; import org.openstreetmap.atlas.geography.sharding.converters.StringToShardConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author lcram */ public class CountryShardToBoundsCommand extends AbstractAtlasShellToolsCommand { private static final Logger logger = LoggerFactory.getLogger(CountryShardToBoundsCommand.class); private static final String REVERSE_OPTION_LONG = "reverse"; private static final String REVERSE_OPTION_DESCRIPTION = "Convert given WKT bound(s) to SlippyTile/GeoHashTile shard(s) if possible. Supports up to slippy zoom level " + Sharding.SLIPPY_ZOOM_MAXIMUM + " and geohash precision " + GeoHashTile.MAXIMUM_PRECISION + "."; private static final String COUNTRY_BOUNDARY_OPTION_LONG = "country-boundary"; private static final String COUNTRY_BOUNDARY_OPTION_DESCRIPTION = "A boundary file to use as a source. See DESCRIPTION section for details."; private static final String COUNTRY_BOUNDARY_OPTION_HINT = "boundary-file"; private static final String SHARD = "shard"; private static final String COUNTRY = "ISO3-country-code"; private static final Integer SHARD_CONTEXT = 3; private static final Integer COUNTRY_CONTEXT = 4; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new CountryShardToBoundsCommand().runSubcommandAndExit(args); } public CountryShardToBoundsCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { if (this.optionAndArgumentDelegate.getParserContext() == SHARD_CONTEXT) { return executeShardContext(); } else if (this.optionAndArgumentDelegate.getParserContext() == COUNTRY_CONTEXT) { return executeCountryContext(); } else { throw new AtlasShellToolsException(); } } @Override public String getCommandName() { return "country-shard-bounds"; } @Override public String getSimpleDescription() { return "get the WKT bounds of given shards or countries"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", CountryShardToBoundsCommand.class .getResourceAsStream("CountryShardToBoundsCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", CountryShardToBoundsCommand.class .getResourceAsStream("CountryShardToBoundsCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerOption(REVERSE_OPTION_LONG, REVERSE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOptionWithRequiredArgument(COUNTRY_BOUNDARY_OPTION_LONG, COUNTRY_BOUNDARY_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, COUNTRY_BOUNDARY_OPTION_HINT, COUNTRY_CONTEXT); registerArgument(SHARD, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, SHARD_CONTEXT); registerArgument(COUNTRY, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, COUNTRY_CONTEXT); super.registerOptionsAndArguments(); } private int executeCountryContext() { final CountryBoundaryMap countryBoundaryMap; final File boundaryMapFile = new File( this.optionAndArgumentDelegate.getOptionArgument(COUNTRY_BOUNDARY_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new), this.getFileSystem()); if (!boundaryMapFile.exists()) { this.outputDelegate.printlnErrorMessage( "boundary file " + boundaryMapFile.getAbsolutePathString() + " does not exist"); return 1; } if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("loading country boundary map..."); } countryBoundaryMap = CountryBoundaryMap.fromPlainText(boundaryMapFile); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("loaded boundary map"); } final List countryCodes = this.optionAndArgumentDelegate .getVariadicArgument(COUNTRY); for (int i = 0; i < countryCodes.size(); i++) { final String countryCode = countryCodes.get(i).toUpperCase(); this.outputDelegate.printlnStdout(countryCode + " boundary:", TTYAttribute.BOLD); final List boundaries = countryBoundaryMap .countryBoundary(countryCode); if (boundaries == null || boundaries.isEmpty()) { this.outputDelegate.printlnWarnMessage("no boundaries found for " + countryCode); } else { for (final org.locationtech.jts.geom.Polygon boundary : boundaries) { this.outputDelegate.printlnStdout(boundary.toText(), TTYAttribute.GREEN); } } if (i < countryCodes.size() - 1) { this.outputDelegate.printlnStdout(""); } } return 0; } private int executeShardContext() { if (this.optionAndArgumentDelegate.hasOption(REVERSE_OPTION_LONG)) { final List wkts = this.optionAndArgumentDelegate.getVariadicArgument(SHARD); for (int i = 0; i < wkts.size(); i++) { final String wkt = wkts.get(i); parseWktAndPrintOutput(wkt); // Only print a separating newline if there were multiple entries if (i < wkts.size() - 1) { this.outputDelegate.printlnStdout(""); } } } else { final List shards = this.optionAndArgumentDelegate.getVariadicArgument(SHARD); for (int i = 0; i < shards.size(); i++) { final String shard = shards.get(i); parseShardAndPrintOutput(shard); // Only print a separating newline if there were multiple entries if (i < shards.size() - 1) { this.outputDelegate.printlnStdout(""); } } } return 0; } private void parseShardAndPrintOutput(final String shardName) { this.outputDelegate.printlnStdout(shardName + " bounds:", TTYAttribute.BOLD); final Shard shard; try { shard = new StringToShardConverter().convert(shardName); } catch (final Exception exception) { logger.error("unable to parse {}", shardName, exception); return; } this.outputDelegate.printlnStdout(shard.bounds().toWkt(), TTYAttribute.GREEN); } private void parseWktAndPrintOutput(final String wkt) { final Polygon polygon; try { polygon = new WktPolygonConverter().backwardConvert(wkt); } catch (final Exception exception) { logger.error("unable to parse WKT polygon {}", wkt, exception); return; } for (int zoom = 1; zoom <= Sharding.SLIPPY_ZOOM_MAXIMUM; zoom++) { final SlippyTileSharding sharding = new SlippyTileSharding(zoom); for (final Shard shard : sharding.shardsIntersecting(polygon)) { if (shard.toWkt().equals(wkt)) { this.outputDelegate.printlnStdout(wkt + " exactly matched shard:", TTYAttribute.BOLD); this.outputDelegate.printlnStdout(shard.toString(), TTYAttribute.GREEN); return; } } } for (int precision = 1; precision <= GeoHashTile.MAXIMUM_PRECISION; precision++) { final GeoHashSharding sharding = new GeoHashSharding(precision); for (final Shard shard : sharding.shardsIntersecting(polygon)) { if (shard.toWkt().equals(wkt)) { this.outputDelegate.printlnStdout(wkt + " exactly matched shard:", TTYAttribute.BOLD); this.outputDelegate.printlnStdout(shard.toString(), TTYAttribute.GREEN); return; } } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/HelloWorldCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; /** * @author lcram */ public class HelloWorldCommand extends AbstractAtlasShellToolsCommand { private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new HelloWorldCommand().runSubcommandAndExit(args); } public HelloWorldCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { this.outputDelegate.printStdout("Hello, " + this.optionAndArgumentDelegate.getOptionArgument("name").orElse("world") + "!\n"); return 0; } @Override public String getCommandName() { return "hello-world"; } @Override public String getSimpleDescription() { return "a simple subcommand that prints \"Hello, world!\" and exits"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", HelloWorldCommand.class .getResourceAsStream("HelloWorldCommandDescriptionSection.txt")); } @Override public void registerOptionsAndArguments() { registerOptionWithRequiredArgument("name", "Your name for the greeting.", OptionOptionality.OPTIONAL, "name"); super.registerOptionsAndArguments(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/IsoCountryCodeCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.locale.IsoCountry; import org.openstreetmap.atlas.locale.IsoCountryFuzzyMatcher; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; /** * @author lcram */ public class IsoCountryCodeCommand extends AbstractAtlasShellToolsCommand { private static final int DEFAULT_MATCH_NUMBER = 10; private static final char NUMBER_OPTION_SHORT = 'n'; private static final String NUMBER_OPTION_LONG = "number"; private static final String NUMBER_OPTION_DESCRIPTION = "The number of matches to display. Defaults to " + DEFAULT_MATCH_NUMBER + "."; private static final String NUMBER_OPTION_HINT = "n"; private static final char ALL_OPTION_SHORT = 'a'; private static final String ALL_OPTION_LONG = "all"; private static final String ALL_OPTION_DESCRIPTION = "Show the entire ISO country listing."; private static final Integer ALL_OPTION_CONTEXT = 4; private static final String QUERY_HINT = "query"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new IsoCountryCodeCommand().runSubcommandAndExit(args); } public IsoCountryCodeCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { if (this.optionAndArgumentDelegate.getParserContext() == ALL_OPTION_CONTEXT) { return allExecute(); } final List queries = this.optionAndArgumentDelegate.getVariadicArgument(QUERY_HINT); for (int i = 0; i < queries.size(); i++) { final String query = queries.get(i); final Optional forIsoCode = IsoCountry.forCountryCode(query); final Optional forDisplayNameExact = IsoCountry.forDisplayCountry(query); final List forDisplayNameTopMatches = IsoCountryFuzzyMatcher .forDisplayCountryTopMatches(this.optionAndArgumentDelegate .getOptionArgument(NUMBER_OPTION_LONG, Integer::parseInt) .orElse(DEFAULT_MATCH_NUMBER), query.toLowerCase()); if (forIsoCode.isEmpty() && IsoCountry.forCountryCode(query.toUpperCase()).isPresent()) { this.outputDelegate.printlnWarnMessage( "did you mean case-sensitive ISO code '" + query.toUpperCase() + "'?"); } // check for exact country code first if (forIsoCode.isPresent()) { this.outputDelegate.printlnStdout("ISO code '" + query + "' matched: ", TTYAttribute.BOLD); printCountry(forIsoCode.get()); } else if (forDisplayNameExact.isPresent()) { this.outputDelegate.printlnStdout("Display country name '" + query + "' matched: ", TTYAttribute.BOLD); printCountry(forDisplayNameExact.get()); } else if (!forDisplayNameTopMatches.isEmpty()) { this.outputDelegate.printlnStdout( "Display country name '" + query + "' had no exact matches. " + forDisplayNameTopMatches.size() + " closest matches are:", TTYAttribute.BOLD); for (final IsoCountry country : forDisplayNameTopMatches) { printCountry(country); } } else { this.outputDelegate.printlnErrorMessage("unmatchable query " + query); } if (i < queries.size() - 1) { this.outputDelegate.printlnStdout(""); } } return 0; } @Override public String getCommandName() { return "iso-country-code"; } @Override public String getSimpleDescription() { return "convert ISO country codes to countries and back again"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", IsoCountryCodeCommand.class .getResourceAsStream("IsoCountryCodeCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", IsoCountryCodeCommand.class .getResourceAsStream("IsoCountryCodeCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerArgument(QUERY_HINT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT); registerOptionWithRequiredArgument(NUMBER_OPTION_LONG, NUMBER_OPTION_SHORT, NUMBER_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, NUMBER_OPTION_HINT, AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT); registerOption(ALL_OPTION_LONG, ALL_OPTION_SHORT, ALL_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, ALL_OPTION_CONTEXT); super.registerOptionsAndArguments(); } private int allExecute() { final List countries = new ArrayList<>(IsoCountry.allCountryCodes()); Collections.sort(countries); this.outputDelegate.printlnStdout("Displaying all countries:", TTYAttribute.BOLD); for (final String country : countries) { final Optional forCode = IsoCountry.forCountryCode(country); if (forCode.isEmpty()) { throw new AtlasShellToolsException(); } printCountry(forCode.get()); } return 0; } private void printCountry(final IsoCountry country) { this.outputDelegate.printlnStdout(country.getCountryCode() + " " + country.getIso3CountryCode() + " " + country.toString(), TTYAttribute.GREEN); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/JavaToProtoSerializationCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas.AtlasSerializationFormat; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasCloner; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderTemplate; import org.openstreetmap.atlas.utilities.command.subcommands.templates.OutputDirectoryTemplate; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; /** * @author lcram */ public class JavaToProtoSerializationCommand extends AbstractAtlasShellToolsCommand { private static final String CHECK_OPTION_LONG = "check"; private static final Character CHECK_OPTION_SHORT = 'c'; private static final String CHECK_OPTION_DESCRIPTION = "Check the serialization format of the atlas(es) without converting."; private static final String REVERSE_OPTION_LONG = "reverse"; private static final Character REVERSE_OPTION_SHORT = 'R'; private static final String REVERSE_OPTION_DESCRIPTION = "Convert Protocol Buffers atlas(es) back to Java serialization."; private static final Integer CHECK_CONTEXT = 4; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new JavaToProtoSerializationCommand().runSubcommandAndExit(args); } public JavaToProtoSerializationCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { return AtlasLoaderTemplate.execute(this, null, this::processAtlas, null); } @Override public String getCommandName() { return "java2proto"; } @Override public String getSimpleDescription() { return "convert Java-serialized atlases to Protocol Buffers format"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", JavaToProtoSerializationCommand.class .getResourceAsStream("JavaToProtoSerializationCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", JavaToProtoSerializationCommand.class .getResourceAsStream("JavaToProtoSerializationCommandExamplesSection.txt")); registerManualPageSectionsFromTemplate(new AtlasLoaderTemplate()); registerManualPageSectionsFromTemplate(new OutputDirectoryTemplate()); } @Override public void registerOptionsAndArguments() { registerOption(REVERSE_OPTION_LONG, REVERSE_OPTION_SHORT, REVERSE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(CHECK_OPTION_LONG, CHECK_OPTION_SHORT, CHECK_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, CHECK_CONTEXT); registerOptionsAndArgumentsFromTemplate(new AtlasLoaderTemplate( AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT, CHECK_CONTEXT)); registerOptionsAndArgumentsFromTemplate( new OutputDirectoryTemplate(AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT)); super.registerOptionsAndArguments(); } private void processAtlas(final Atlas atlas, final String atlasFileName, final File atlasResource) { PackedAtlas outputAtlas; try { outputAtlas = (PackedAtlas) atlas; } catch (final ClassCastException exception) { outputAtlas = new PackedAtlasCloner().cloneFrom(atlas); } if (this.optionAndArgumentDelegate.getParserContext() == CHECK_CONTEXT) { this.outputDelegate.printStdout("atlas "); this.outputDelegate.printStdout(atlasResource.getPathString(), TTYAttribute.BOLD); this.outputDelegate.printStdout(" format: "); this.outputDelegate.printlnStdout(outputAtlas.getSerializationFormat().toString(), TTYAttribute.BOLD); } else { if (this.optionAndArgumentDelegate.hasOption(REVERSE_OPTION_LONG)) { outputAtlas.setSaveSerializationFormat(AtlasSerializationFormat.JAVA); } else { outputAtlas.setSaveSerializationFormat(AtlasSerializationFormat.PROTOBUF); } final Optional outputPathOptional = OutputDirectoryTemplate.getOutputPath(this); if (outputPathOptional.isEmpty()) { this.outputDelegate .printlnWarnMessage("could not save " + atlasFileName + ", skipping..."); return; } final Path concatenatedPath = outputPathOptional .map(path -> Paths.get(path.toAbsolutePath().toString(), atlasFileName)) .orElseThrow(AtlasShellToolsException::new); final File outputFile = new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()); outputAtlas.save(outputFile); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnStdout("Saved to " + concatenatedPath.toString()); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/OsmFileParserCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.testing.OsmFileParser; /** * @author matthieun */ public class OsmFileParserCommand extends AbstractAtlasShellToolsCommand { private static final String JOSM_OSM_FILE = "josm"; private static final String OSM_FILE = "osm"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; public static void main(final String[] args) { new OsmFileParserCommand().runSubcommandAndExit(args); } public OsmFileParserCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); } @Override public int execute() { final String josmOsmFile = this.optionAndArgumentDelegate.getUnaryArgument(JOSM_OSM_FILE) .orElseThrow(AtlasShellToolsException::new); final String osmFile = this.optionAndArgumentDelegate.getUnaryArgument(OSM_FILE) .orElseThrow(AtlasShellToolsException::new); new OsmFileParser().update(new File(josmOsmFile, this.getFileSystem()), new File(osmFile, this.getFileSystem())); return 0; } @Override public String getCommandName() { return "josm2osm"; } @Override public String getSimpleDescription() { return "transform a JOSM OSM file into a real OSM file."; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", OsmFileParserCommand.class .getResourceAsStream("OsmFileParserCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", OsmFileParserCommand.class .getResourceAsStream("OsmFileParserCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerArgument(JOSM_OSM_FILE, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED); registerArgument(OSM_FILE, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED); super.registerOptionsAndArguments(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/OsmToAtlasCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.InputStreamResource; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.testing.TestAtlasHandler; /** * Convert an .osm file of various types to an atlas. * * @author jklamer */ public class OsmToAtlasCommand extends AbstractAtlasShellToolsCommand { private static final String INPUT_OSM_FILE_ARGUMENT = "input-osm-file"; private static final String OUTPUT_ATLAS_FILE_ARGUMENT = "output-atlas-file"; private static final String JOSM_OPTION_LONG = "josm"; private static final String JOSM_OPTION_DESCRIPTION = "Specify if the OSM file is in JOSM format."; private static final String COUNTRY_OPTION_LONG = "country"; private static final String COUNTRY_OPTION_DESCRIPTION = "Specify an ISO3 country code to use for slicing."; private static final String COUNTRY_OPTION_HINT = "ISO3"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; public static void main(final String[] args) { new OsmToAtlasCommand().runSubcommandAndExit(args); } public OsmToAtlasCommand() { super(); this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); } @Override public int execute() { final File osmFile = new File( this.optionAndArgumentDelegate.getUnaryArgument(INPUT_OSM_FILE_ARGUMENT) .orElseThrow(AtlasShellToolsException::new), this.getFileSystem()); final File atlasFile = new File( this.optionAndArgumentDelegate.getUnaryArgument(OUTPUT_ATLAS_FILE_ARGUMENT) .orElseThrow(AtlasShellToolsException::new), this.getFileSystem()); final boolean useJosmFormat = this.optionAndArgumentDelegate.hasOption(JOSM_OPTION_LONG); final Atlas atlas = TestAtlasHandler.getAtlasFromJosmOsmResource(useJosmFormat, new InputStreamResource(osmFile::read), osmFile.getName(), this.optionAndArgumentDelegate.getOptionArgument(COUNTRY_OPTION_LONG)); atlas.save(atlasFile); return 0; } @Override public String getCommandName() { return "osm2atlas"; } @Override public String getSimpleDescription() { return "convert a .osm file into an Atlas file"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", OsmToAtlasCommand.class .getResourceAsStream("OsmToAtlasCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", OsmToAtlasCommand.class .getResourceAsStream("OsmToAtlasCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerArgument(INPUT_OSM_FILE_ARGUMENT, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED); registerArgument(OUTPUT_ATLAS_FILE_ARGUMENT, ArgumentArity.UNARY, ArgumentOptionality.REQUIRED); registerOption(JOSM_OPTION_LONG, JOSM_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOptionWithRequiredArgument(COUNTRY_OPTION_LONG, COUNTRY_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, COUNTRY_OPTION_HINT); super.registerOptionsAndArguments(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/PackedToTextAtlasCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderCommand; /** * @author lcram */ public class PackedToTextAtlasCommand extends AtlasLoaderCommand { private static final String SAVED_TO = "saved to "; private static final String REVERSE_OPTION_LONG = "reverse"; private static final Character REVERSE_OPTION_SHORT = 'R'; private static final String REVERSE_OPTION_DESCRIPTION = "Convert a text atlas in TextAtlas format (i.e. not GeoJSON) back into a PackedAtlas."; private static final String GEOJSON_OPTION_LONG = "geojson"; private static final Character GEOJSON_OPTION_SHORT = 'g'; private static final String GEOJSON_OPTION_DESCRIPTION = "Save atlas as GeoJSON."; private static final String LDGEOJSON_OPTION_LONG = "ldgeojson"; private static final Character LDGEOJSON_OPTION_SHORT = 'l'; private static final String LDGEOJSON_OPTION_DESCRIPTION = "Save atlas as line-delimited GeoJSON."; private static final Integer GEOJSON_CONTEXT = 4; private static final Integer LDGEOJSON_CONTEXT = 5; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new PackedToTextAtlasCommand().runSubcommandAndExit(args); } public PackedToTextAtlasCommand() { super(); this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public String getCommandName() { return "packed2text"; } @Override public String getSimpleDescription() { return "transform a PackedAtlas into a human-readable format"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", PackedToTextAtlasCommand.class .getResourceAsStream("PackedToTextAtlasCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", PackedToTextAtlasCommand.class .getResourceAsStream("PackedToTextAtlasCommandExamplesSection.txt")); super.registerManualPageSections(); } @Override public void registerOptionsAndArguments() { registerOption(REVERSE_OPTION_LONG, REVERSE_OPTION_SHORT, REVERSE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); registerOption(GEOJSON_OPTION_LONG, GEOJSON_OPTION_SHORT, GEOJSON_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, GEOJSON_CONTEXT); registerOption(LDGEOJSON_OPTION_LONG, LDGEOJSON_OPTION_SHORT, LDGEOJSON_OPTION_DESCRIPTION, OptionOptionality.REQUIRED, LDGEOJSON_CONTEXT); super.registerOptionsAndArguments(); } @Override protected void processAtlas(final Atlas atlas, final String atlasFileName, final File atlasResource) { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate .printlnCommandMessage("converting " + atlasResource.getPathString() + "..."); } try { writeOutput(atlasFileName, atlas); } catch (final Exception exception) { this.outputDelegate.printlnErrorMessage("failed to save file for " + atlasResource.getPathString() + ": " + exception.getMessage()); } } private void writeOutput(final String atlasFileName, final Atlas outputAtlas) { final String fileName = AtlasLoaderCommand.removeSuffixFromFileName(atlasFileName); final Path concatenatedPath = Paths.get(getOutputPath().toAbsolutePath().toString(), fileName); File outputFile = null; if (this.optionAndArgumentDelegate .getParserContext() == AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT) { if (this.optionAndArgumentDelegate.hasOption(REVERSE_OPTION_LONG)) { outputFile = new File( concatenatedPath.toAbsolutePath().toString() + FileSuffix.ATLAS, this.getFileSystem()); outputAtlas.save(outputFile); } else { outputFile = new File(concatenatedPath.toAbsolutePath().toString() + FileSuffix.ATLAS + FileSuffix.TEXT, this.getFileSystem()); outputAtlas.saveAsText(outputFile); } } else if (this.optionAndArgumentDelegate.getParserContext() == GEOJSON_CONTEXT) { outputFile = new File( concatenatedPath.toAbsolutePath().toString() + FileSuffix.GEO_JSON, this.getFileSystem()); outputAtlas.saveAsGeoJson(outputFile); } else if (this.optionAndArgumentDelegate.getParserContext() == LDGEOJSON_CONTEXT) { outputFile = new File( concatenatedPath.toAbsolutePath().toString() + FileSuffix.GEO_JSON, this.getFileSystem()); outputAtlas.saveAsLineDelimitedGeoJsonFeatures(outputFile, (entity, json) -> { // Dummy consumer, we don't need to mutate the JSON }); } else { throw new AtlasShellToolsException(); } if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate .printlnCommandMessage(SAVED_TO + outputFile.toAbsolutePath().toString()); } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/PbfToAtlasCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas.AtlasSerializationFormat; import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption; import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator; import org.openstreetmap.atlas.geography.atlas.raw.sectioning.AtlasSectionProcessor; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.streaming.compression.Decompressor; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.MultipleOutputCommand; /** * @author samgass * @author matthieun */ public class PbfToAtlasCommand extends MultipleOutputCommand { // The hint for the input path for the PBF file(s) to convert private static final String PBF_PATH_HINT = "pbf"; // The country name for the country shards Atlas file(s) to output private static final String COUNTRY_NAME = "countryName"; // The file containing the WKT polygon to constrain the loading private static final String BOUNDS = "bounds"; // Whether or not to stop at the raw atlas private static final String RAW = "raw"; private static final String COUNTRY_NAME_DESCRIPTION = "The country for the shard to build"; private static final String BOUNDS_DESCRIPTION = "The file containing WKT bounds to restrain the loading."; private static final String RAW_DESCRIPTION = "Whether or not to stop at the raw atlas. If this is enabled, way-sectioning will not happen"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; private List pbfs; public static void main(final String[] args) { new PbfToAtlasCommand().runSubcommandAndExit(args); } public PbfToAtlasCommand() { super(); this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { // set up the output path from the parent class final int code = super.execute(); if (code != 0) { return code; } getInputPBFs(); final String countryName = this.optionAndArgumentDelegate.getOptionArgument(COUNTRY_NAME) .orElseThrow(AtlasShellToolsException::new); this.pbfs.forEach(pbf -> { PackedAtlas atlas = (PackedAtlas) new RawAtlasGenerator(pbf, AtlasLoadingOption.createOptionWithOnlySectioning(), getBounds()).build(); final String pbfName = pbf.getName().replace(FileSuffix.PBF.toString(), ""); final String rawAtlasFilename = String.format("%s%s%s%s", countryName, Shard.SHARD_DATA_SEPARATOR, pbfName, FileSuffix.ATLAS); if (!stopAtRaw()) { final AtlasSectionProcessor waySectionProcessor = new AtlasSectionProcessor(atlas, AtlasLoadingOption.createOptionWithNoSlicing()); atlas = (PackedAtlas) waySectionProcessor.run(); } atlas.setSaveSerializationFormat(AtlasSerializationFormat.PROTOBUF); final Path concatenatedPath; if (this.optionAndArgumentDelegate .hasOption(MultipleOutputCommand.OUTPUT_DIRECTORY_OPTION_LONG)) { // save atlas to user specified output directory concatenatedPath = Paths.get(getOutputPath().toAbsolutePath().toString(), rawAtlasFilename); } else { // save atlas in place concatenatedPath = Paths.get(Paths.get(pbf.getAbsolutePathString()).getParent() .toAbsolutePath().toString(), rawAtlasFilename); } this.outputDelegate.printlnStdout(concatenatedPath.toAbsolutePath().toString()); final File outputFile = new File(concatenatedPath.toAbsolutePath().toString(), this.getFileSystem()); atlas.save(outputFile); }); return 0; } @Override public String getCommandName() { return "pbf2atlas"; } @Override public String getSimpleDescription() { return "generate way-sectioned Atlas file(s) from the given PBF shard(s)"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", PbfToAtlasCommand.class.getResourceAsStream("PbfToAtlasDescriptionSection.txt")); addManualPageSection("EXAMPLES", PbfToAtlasCommand.class .getResourceAsStream("PbfToAtlasCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { final Integer[] contexts = this.optionAndArgumentDelegate.getFilteredRegisteredContexts() .toArray(new Integer[0]); this.registerArgument(PBF_PATH_HINT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, contexts); this.registerOptionWithRequiredArgument(COUNTRY_NAME, COUNTRY_NAME_DESCRIPTION, OptionOptionality.REQUIRED, COUNTRY_NAME); this.registerOptionWithRequiredArgument(BOUNDS, BOUNDS_DESCRIPTION, OptionOptionality.OPTIONAL, BOUNDS); this.registerOption(RAW, RAW_DESCRIPTION, OptionOptionality.OPTIONAL); super.registerOptionsAndArguments(); } private MultiPolygon getBounds() { final Optional boundsFilePathOption = this.optionAndArgumentDelegate .getOptionArgument(BOUNDS); if (boundsFilePathOption.isPresent()) { final String wktFileName = boundsFilePathOption.get(); final File wktFile = new File(wktFileName, this.getFileSystem()); if (wktFileName.endsWith(FileSuffix.GZIP.toString())) { wktFile.setDecompressor(Decompressor.GZIP); } final String wkt = wktFile.firstLine(); return MultiPolygon.wkt(wkt); } else { return MultiPolygon.MAXIMUM; } } /** * Get a list of input PBF resources from the input switch */ private void getInputPBFs() { if (this.pbfs == null) { this.pbfs = new ArrayList<>(); } else { return; } final List inputPbfPaths = this.optionAndArgumentDelegate .getVariadicArgument(PBF_PATH_HINT); inputPbfPaths.forEach(path -> { final File file = new File(path, this.getFileSystem(), false); if (!file.exists()) { this.outputDelegate.printlnWarnMessage("file not found: " + path); } else if (file.isDirectory()) { this.outputDelegate.printlnWarnMessage("skipping directory: " + path); } else { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("loading " + path); } this.pbfs.add(file); } }); } private boolean stopAtRaw() { return this.optionAndArgumentDelegate.hasOption(RAW); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/SubAtlasCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.FileSuffix; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.AtlasLoaderCommand; import org.openstreetmap.atlas.utilities.command.subcommands.templates.PredicateTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author lcram */ public class SubAtlasCommand extends AtlasLoaderCommand { private static final Logger logger = LoggerFactory.getLogger(SubAtlasCommand.class); private static final String POLYGON_OPTION_LONG = "polygon"; private static final String POLYGON_OPTION_DESCRIPTION = "The WKT of the polygon with which to cut."; private static final String POLYGON_OPTION_HINT = "wkt"; private static final List CUT_TYPE_STRINGS = Arrays.stream(AtlasCutType.values()) .map(AtlasCutType::toString).collect(Collectors.toList()); private static final String CUT_TYPE_OPTION_LONG = "cut-type"; private static final String CUT_TYPE_OPTION_DESCRIPTION = "The cut-type of this subatlas. Valid settings are: " + new StringList(CUT_TYPE_STRINGS).join(", ") + ". Defaults to SOFT_CUT."; private static final String CUT_TYPE_OPTION_HINT = "type"; private static final String SLICE_FIRST_OPTION_LONG = "slice-first"; private static final String SLICE_FIRST_OPTION_DESCRIPTION = "Cut with supplied geometry before applying the predicate."; private static final List IMPORTS_ALLOW_LIST = Arrays.asList( "org.openstreetmap.atlas.geography.atlas.items", "org.openstreetmap.atlas.tags.annotations", "org.openstreetmap.atlas.tags.annotations.validation", "org.openstreetmap.atlas.tags.annotations.extraction", "org.openstreetmap.atlas.tags", "org.openstreetmap.atlas.tags.names", "org.openstreetmap.atlas.geography", "org.openstreetmap.atlas.utilities.collections"); private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; private AtlasCutType cutType; private Polygon polygon; private Predicate matcher; public static void main(final String[] args) { new SubAtlasCommand().runSubcommandAndExit(args); } public SubAtlasCommand() { super(); this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); this.cutType = AtlasCutType.SOFT_CUT; } @Override public String getCommandName() { return "subatlas"; } @Override public String getSimpleDescription() { return "cut subatlases according to given parameters"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", SubAtlasCommand.class.getResourceAsStream("SubAtlasCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", SubAtlasCommand.class.getResourceAsStream("SubAtlasCommandExamplesSection.txt")); registerManualPageSectionsFromTemplate(new PredicateTemplate()); super.registerManualPageSections(); } @Override public void registerOptionsAndArguments() { this.registerOptionWithRequiredArgument(POLYGON_OPTION_LONG, POLYGON_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, POLYGON_OPTION_HINT); this.registerOptionsAndArgumentsFromTemplate(new PredicateTemplate()); this.registerOptionWithRequiredArgument(CUT_TYPE_OPTION_LONG, CUT_TYPE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, CUT_TYPE_OPTION_HINT); this.registerOption(SLICE_FIRST_OPTION_LONG, SLICE_FIRST_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL); super.registerOptionsAndArguments(); } @Override protected void processAtlas(final Atlas atlas, final String atlasFileName, final File atlasResource) { Optional subbedAtlas = Optional.empty(); if (this.optionAndArgumentDelegate.hasOption(SLICE_FIRST_OPTION_LONG)) { if (this.polygon != null) { subbedAtlas = atlas.subAtlas(this.polygon, this.cutType); } if (this.matcher != null) { subbedAtlas = subbedAtlas.orElse(atlas).subAtlas(this.matcher, this.cutType); } } else { if (this.matcher != null) { subbedAtlas = atlas.subAtlas(this.matcher, this.cutType); } if (this.polygon != null) { subbedAtlas = subbedAtlas.orElse(atlas).subAtlas(this.polygon, this.cutType); } } if (subbedAtlas.isPresent()) { final String fileName = "sub_" + AtlasLoaderCommand.removeSuffixFromFileName(atlasFileName); final Path concatenatedPath = Paths.get(getOutputPath().toAbsolutePath().toString(), fileName); final File outputFile = new File( concatenatedPath.toAbsolutePath().toString() + FileSuffix.ATLAS, this.getFileSystem()); subbedAtlas.get().save(outputFile); if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate .printlnCommandMessage("saved to " + outputFile.getAbsolutePathString()); } } else { this.outputDelegate.printlnWarnMessage( "skipping save of empty subatlas cut from " + atlasResource.getPathString()); } } @Override protected int start() { final Optional cutTypeParameter = this.optionAndArgumentDelegate .getOptionArgument(CUT_TYPE_OPTION_LONG); final Optional wktParameter = this.optionAndArgumentDelegate .getOptionArgument(POLYGON_OPTION_LONG); if (cutTypeParameter.isPresent()) { try { this.cutType = AtlasCutType.valueOf(cutTypeParameter.get().toUpperCase()); } catch (final IllegalArgumentException exception) { this.outputDelegate .printlnErrorMessage("invalid cut type " + cutTypeParameter.get()); this.outputDelegate .printlnStderr("Try " + new StringList(CUT_TYPE_STRINGS).join(", ")); return 1; } } if (wktParameter.isPresent()) { final WKTReader reader = new WKTReader(); Geometry geometry = null; try { geometry = reader.read(wktParameter.get()); } catch (final ParseException exception) { logger.error("unable to parse {}", wktParameter.get(), exception); return 1; } if (geometry instanceof org.locationtech.jts.geom.Polygon) { this.polygon = new JtsPolygonConverter() .backwardConvert((org.locationtech.jts.geom.Polygon) geometry); } else { this.outputDelegate .printlnErrorMessage("unsupported geometry type " + wktParameter.get()); return 1; } } this.matcher = PredicateTemplate.getPredicate(AtlasEntity.class, IMPORTS_ALLOW_LIST, this) .orElse(null); return 0; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/TaggableMatcherPrinterCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.List; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; /** * @author lcram */ public class TaggableMatcherPrinterCommand extends AbstractAtlasShellToolsCommand { private static final String REVERSE_OPTION_LONG = "reverse"; private static final String REVERSE_OPTION_DESCRIPTION = "Convert an old-style TaggableFilter into a TaggableMatcher."; private static final String FILTERS_ARGUMENT = "filters"; private static final String MATCHERS_ARGUMENT = "matchers"; private static final int REVERSE_CONTEXT = 4; public static void main(final String[] args) { new TaggableMatcherPrinterCommand().runSubcommandAndExit(args); } @Override public int execute() { if (this.getOptionAndArgumentDelegate().getParserContext() == REVERSE_CONTEXT) { executeReverseContext(); } else { executeDefaultContext(); } return 0; } @Override public String getCommandName() { return "print-matcher"; } @Override public String getSimpleDescription() { return "print a TaggableMatcher as a tree"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", AtlasSearchCommand.class .getResourceAsStream("TaggableMatcherPrinterCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", AtlasSearchCommand.class .getResourceAsStream("TaggableMatcherPrinterCommandExamplesSection.txt")); } @Override public void registerOptionsAndArguments() { registerArgument(MATCHERS_ARGUMENT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED); registerArgument(FILTERS_ARGUMENT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, REVERSE_CONTEXT); registerOption(REVERSE_OPTION_LONG, REVERSE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, REVERSE_CONTEXT); super.registerOptionsAndArguments(); } private void executeDefaultContext() { final List definitions = this.getOptionAndArgumentDelegate() .getVariadicArgument(MATCHERS_ARGUMENT); for (int i = 0; i < definitions.size(); i++) { final String definition = definitions.get(i); try { final TaggableMatcher matcher = TaggableMatcher.from(definition); this.getCommandOutputDelegate().printStdout(matcher.prettyPrintTree()); if (matcher.lengthOfLongestLineForPrintedTree() > this.getMaximumColumn()) { this.getCommandOutputDelegate() .printlnWarnMessage("tree was too big for detected terminal width"); this.getCommandOutputDelegate().printlnCommandMessage( "try piping into `less -S' to disable line-wrapping"); } } catch (final CoreException exception) { if (exception.getMessage().contains("invalid nested equality operators")) { this.getCommandOutputDelegate().printlnErrorMessage( "definition `" + definition + "' contained nested equality operators"); } else { throw exception; } } // Print an extra newline between trees, but not for the last tree if (i < definitions.size() - 1) { this.getCommandOutputDelegate().printlnStdout(""); } } } private void executeReverseContext() { final List definitions = this.getOptionAndArgumentDelegate() .getVariadicArgument(FILTERS_ARGUMENT); for (final String definition : definitions) { try { this.getCommandOutputDelegate().printlnStdout(TaggableFilter .forDefinition(definition).convertToTaggableMatcher().getDefinition()); } catch (final Exception exception) { this.getCommandOutputDelegate().printlnErrorMessage(exception.getMessage()); } } } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/TemplateTestCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.util.List; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.ListOfNumbersTemplate; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; /** * A command to test the {@link AtlasShellToolsCommandTemplate} feature. * * @author lcram */ public class TemplateTestCommand extends AbstractAtlasShellToolsCommand { private static final int REVERSE_CONTEXT = 4; public static void main(final String[] args) { new TemplateTestCommand().runSubcommandAndExit(args); } @Override public int execute() { if (this.getOptionAndArgumentDelegate() .getParserContext() == AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT) { final List listOfNumbers = ListOfNumbersTemplate.getListOfNumbers(this); if (listOfNumbers.isEmpty()) { this.getCommandOutputDelegate().printlnErrorMessage("failed to parse number list!"); return 1; } this.getCommandOutputDelegate().printlnStdout(listOfNumbers.toString(), TTYAttribute.GREEN); } else { this.getCommandOutputDelegate().printlnStdout("Using reverse context!", TTYAttribute.GREEN); } return 0; } @Override public String getCommandName() { return "template-test"; } @Override public String getSimpleDescription() { return "test the Atlas Shell Tools template feature"; } @Override public void registerManualPageSections() { registerManualPageSectionsFromTemplate(new ListOfNumbersTemplate()); } @Override public void registerOptionsAndArguments() { registerOptionsAndArgumentsFromTemplate( new ListOfNumbersTemplate(AbstractAtlasShellToolsCommand.DEFAULT_CONTEXT)); registerOption("reverse", "Perform this operation in reverse.", OptionOptionality.REQUIRED, REVERSE_CONTEXT); super.registerOptionsAndArguments(); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/WKTShardCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.prep.PreparedPolygon; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.geography.converters.jts.JtsPointConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolyLineConverter; import org.openstreetmap.atlas.geography.converters.jts.JtsPolygonConverter; import org.openstreetmap.atlas.geography.sharding.Shard; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.geography.sharding.converters.StringToShardConverter; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.tags.ISOCountryTag; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.templates.CountryBoundaryMapTemplate; import org.openstreetmap.atlas.utilities.command.subcommands.templates.ShardingTemplate; import org.openstreetmap.atlas.utilities.command.terminal.TTYAttribute; import org.openstreetmap.atlas.utilities.maps.MultiMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author lcram */ public class WKTShardCommand extends AbstractAtlasShellToolsCommand { private static final Logger logger = LoggerFactory.getLogger(WKTShardCommand.class); private static final String INPUT_FILE_OPTION_LONG = "input"; private static final String INPUT_FILE_OPTION_DESCRIPTION = "An input file from which to source the WKT entities. See DESCRIPTION section for details."; private static final String INPUT_FILE_OPTION_HINT = "file"; private static final Integer SHARDING_CONTEXT = 3; private static final Integer COUNTRY_BOUNDARY_CONTEXT = 4; private static final String INPUT_WKT_SHARD = "wkt|shard"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; public static void main(final String[] args) { new WKTShardCommand().runSubcommandAndExit(args); } public WKTShardCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); } @Override public int execute() { final List inputWktOrShard = new ArrayList<>(); if (this.optionAndArgumentDelegate.hasOption(INPUT_FILE_OPTION_LONG)) { inputWktOrShard.addAll(readInputsFromFile(this.optionAndArgumentDelegate .getOptionArgument(INPUT_FILE_OPTION_LONG).orElse(null))); } inputWktOrShard.addAll(this.optionAndArgumentDelegate.getVariadicArgument(INPUT_WKT_SHARD)); if (inputWktOrShard.isEmpty()) { this.outputDelegate.printlnWarnMessage("no input WKTs were found"); return 0; } Sharding sharding = null; CountryBoundaryMap countryBoundaryMap = null; if (this.optionAndArgumentDelegate.getParserContext() == SHARDING_CONTEXT) { sharding = ShardingTemplate.getSharding(this); } else if (this.optionAndArgumentDelegate.getParserContext() == COUNTRY_BOUNDARY_CONTEXT) { final Optional mapOptional = CountryBoundaryMapTemplate .getCountryBoundaryMap(this); if (mapOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("failed to load country boundary"); return 1; } countryBoundaryMap = mapOptional.get(); } else { throw new AtlasShellToolsException(); } for (int i = 0; i < inputWktOrShard.size(); i++) { final String wktOrShard = inputWktOrShard.get(i); parseWktOrShardAndPrintOutput(wktOrShard, sharding, countryBoundaryMap); // Only print a separating newline if there were multiple entries if (i < inputWktOrShard.size() - 1) { this.outputDelegate.printlnStdout(""); } } return 0; } @Override public String getCommandName() { return "wkt-shard"; } @Override public String getSimpleDescription() { return "perform various intersection lookups"; } @Override public void registerManualPageSections() { addManualPageSection("DESCRIPTION", WKTShardCommand.class.getResourceAsStream("WKTShardCommandDescriptionSection.txt")); addManualPageSection("EXAMPLES", WKTShardCommand.class.getResourceAsStream("WKTShardCommandExamplesSection.txt")); registerManualPageSectionsFromTemplate(new ShardingTemplate()); registerManualPageSectionsFromTemplate(new CountryBoundaryMapTemplate()); } @Override public void registerOptionsAndArguments() { registerArgument(INPUT_WKT_SHARD, ArgumentArity.VARIADIC, ArgumentOptionality.OPTIONAL, SHARDING_CONTEXT, COUNTRY_BOUNDARY_CONTEXT); registerOptionWithRequiredArgument(INPUT_FILE_OPTION_LONG, INPUT_FILE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, INPUT_FILE_OPTION_HINT, SHARDING_CONTEXT, COUNTRY_BOUNDARY_CONTEXT); registerOptionsAndArgumentsFromTemplate(new ShardingTemplate(SHARDING_CONTEXT)); registerOptionsAndArgumentsFromTemplate( new CountryBoundaryMapTemplate(COUNTRY_BOUNDARY_CONTEXT)); super.registerOptionsAndArguments(); } private void parseWktOrShardAndPrintOutput(final String wktOrShard, final Sharding sharding, final CountryBoundaryMap countryBoundaryMap) { final Optional geometryOptional = parseWktOrShardString(wktOrShard); if (geometryOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage( "unable to parse '" + wktOrShard + "' as WKT or shard string"); return; } final Geometry geometry = geometryOptional.get(); if (geometry instanceof Point) { printPointOutput(wktOrShard, geometry, sharding, countryBoundaryMap); } else if (geometry instanceof LineString) { printLineStringOutput(wktOrShard, geometry, sharding, countryBoundaryMap); } else if (geometry instanceof Polygon) { printPolygonOutput(wktOrShard, geometry, sharding, countryBoundaryMap); } /* * TODO handle more geometry types? e.g. MultiPoint, MultiLineString, and MultiPolygon? */ else { this.outputDelegate.printlnErrorMessage("unsupported geometry type " + wktOrShard); } } private Optional parseWktOrShardString(final String wktOrShard) { final WKTReader reader = new WKTReader(); try { return Optional.of(reader.read(wktOrShard)); } catch (final ParseException exception) { logger.warn("unable to parse {} as wkt", wktOrShard, exception); // input String was not a WKT, so try parsing it as a shard string try { final StringToShardConverter converter = new StringToShardConverter(); final Shard shard = converter.convert(wktOrShard); return Optional.of(new WKTReader().read(shard.toWkt())); } catch (final Exception exception2) { logger.warn("unable to parse {} as shard", wktOrShard, exception2); } } return Optional.empty(); } private void printLineStringOutput(final String wktOrShard, final Geometry geometry, final Sharding sharding, final CountryBoundaryMap countryBoundaryMap) { this.outputDelegate.printlnStdout(wktOrShard + " intersects:", TTYAttribute.BOLD); final PolyLine polyline = new JtsPolyLineConverter().backwardConvert((LineString) geometry); if (sharding != null) { final Iterable shards = sharding.shardsIntersecting(polyline); for (final Shard shard : shards) { this.outputDelegate.printlnStdout(shard.toString(), TTYAttribute.GREEN); } } if (countryBoundaryMap != null) { final MultiMap boundaries = countryBoundaryMap.boundaries(polyline); for (final String country : boundaries.keySet()) { this.outputDelegate.printlnStdout(country, TTYAttribute.GREEN); } } } private void printPointOutput(final String wktOrShard, final Geometry geometry, final Sharding sharding, final CountryBoundaryMap countryBoundaryMap) { this.outputDelegate.printlnStdout(wktOrShard + " covered by:", TTYAttribute.BOLD); final Location location = new JtsPointConverter().backwardConvert((Point) geometry); if (sharding != null) { final Iterable shards = sharding.shardsCovering(location); for (final Shard shard : shards) { this.outputDelegate.printlnStdout(shard.toString(), TTYAttribute.GREEN); } } if (countryBoundaryMap != null) { final MultiMap boundaries = countryBoundaryMap.boundaries(location); for (final String country : boundaries.keySet()) { this.outputDelegate.printlnStdout(country, TTYAttribute.GREEN); } } } private void printPolygonOutput(final String wktOrShard, final Geometry geometry, final Sharding sharding, final CountryBoundaryMap countryBoundaryMap) { this.outputDelegate.printlnStdout(wktOrShard + " contains or intersects:", TTYAttribute.BOLD); final org.openstreetmap.atlas.geography.Polygon polygon = new JtsPolygonConverter() .backwardConvert((Polygon) geometry); if (sharding != null) { final Iterable shards = sharding.shards(polygon); for (final Shard shard : shards) { this.outputDelegate.printlnStdout(shard.toString(), TTYAttribute.GREEN); } } if (countryBoundaryMap != null) { /* * This is handled a little differently here than in the other printXOutput methods. * This is because the CountryBoundaryMap#boundaries method does not handle certain * cases the way we want it to, e.g. like when a shard boundary or other large polygon * completely encloses a country boundary (think large ocean shard totally containing a * small island country). We still want to report those enclosed boundaries in this * case. In the future, we may want to fix CountryBoundaryMap to handle this case in * some way. */ final List polygons = countryBoundaryMap .query(geometry.getEnvelopeInternal()).stream().distinct() .collect(Collectors.toList()); final Set countries = new HashSet<>(); polygons.forEach(polygon2 -> countries.add(CountryBoundaryMap .getGeometryProperty(polygon2.getGeometry(), ISOCountryTag.KEY))); for (final String country : countries) { this.outputDelegate.printlnStdout(country, TTYAttribute.GREEN); } } } private List readInputsFromFile(final String path) { if (path == null) { throw new AtlasShellToolsException(); } final Path inputPath = this.getFileSystem().getPath(path); if (inputPath.toString().startsWith("~")) { this.outputDelegate.printlnWarnMessage("the '~' was not expanded by your shell"); } if (!Files.isReadable(inputPath) || !Files.isRegularFile(inputPath)) { this.outputDelegate.printlnErrorMessage( inputPath.toAbsolutePath().toString() + " is not a readable file"); return new ArrayList<>(); } final List wktOrShardList = new ArrayList<>(); final StringResource resource = new StringResource(); resource.copyFrom(new File(inputPath.toAbsolutePath().toString(), this.getFileSystem())); final String rawText = resource.all(); final String[] split = rawText.split(System.getProperty("line.separator")); for (final String line : split) { if (!line.isEmpty()) { wktOrShardList.add(line); } } return wktOrShardList; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/AtlasLoaderCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.command.AbstractAtlasSubCommand; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.JavaToProtoSerializationCommand; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * A helper super class for any command that wants to load atlas files from disk. Provides a builtin * variadic input argument, as well as automatic conversion from paths to resources with various * options for increased flexibility. Subclasses can override the start(), processAtlas(), and * finish() methods, which provide a way to operate on the input atlases without having to deal with * resource loading and iterating.
* This class is based off the {@link AbstractAtlasSubCommand} by cstaylor. * * @author lcram * @author cstaylor * @deprecated Please use {@link AtlasLoaderTemplate} instead. See * {@link JavaToProtoSerializationCommand } for an example. */ @Deprecated public abstract class AtlasLoaderCommand extends MultipleOutputCommand { private static final String COMBINED_ATLAS_NAME = "combined.atlas"; private static final String INPUT_HINT = "input-atlases"; private static final String COMBINE_OPTION_LONG = "combine"; private static final String COMBINE_OPTION_DESCRIPTION = "Combine all input atlases into a MultiAtlas before processing."; private static final String STRICT_OPTION_LONG = "strict"; private static final String STRICT_OPTION_DESCRIPTION = "Fail fast if any input atlases are missing."; private static final String PARALLEL_OPTION_LONG = "parallel"; private static final Character PARALLEL_OPTION_SHORT = 'p'; private static final String PARALLEL_OPTION_DESCRIPTION = "Process the atlases in parallel."; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; private List> atlases; public static String removeSuffixFromFileName(final String fileName) { final String[] split = fileName.split("\\."); return split[0]; } public AtlasLoaderCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); this.atlases = null; } @Override public int execute() { // set up the output path from the parent class int code = super.execute(); if (code != 0) { return code; } // call the user start implementation code = start(); if (code != 0) { return code; } final List> atlasTuples = getInputAtlases(); if (atlasTuples.isEmpty()) { this.outputDelegate.printlnErrorMessage("no atlas files were loaded"); return 1; } Stream> atlasTupleStream = atlasTuples.stream(); if (this.optionAndArgumentDelegate.hasOption(PARALLEL_OPTION_LONG)) { atlasTupleStream = atlasTupleStream.parallel(); } if (this.optionAndArgumentDelegate.hasOption(COMBINE_OPTION_LONG)) { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate .printlnCommandMessage("processing all atlases as one multiatlas..."); } processAtlas( new MultiAtlas( atlasTupleStream.map(Tuple::getSecond).collect(Collectors.toList())), COMBINED_ATLAS_NAME, new File(COMBINED_ATLAS_NAME, this.getFileSystem())); } else { final int size = atlasTuples.size(); final int[] count = new int[1]; count[0] = 1; atlasTupleStream.forEach(atlasTuple -> { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage( "processing atlas " + atlasTuple.getFirst().getAbsolutePathString() + " (" + count[0] + "/" + size + ")"); } processAtlas(atlasTuple.getSecond(), atlasTuple.getFirst().getName(), atlasTuple.getFirst()); count[0]++; }); } // return the exit code from the user's finish implementation return finish(); } @Override public void registerManualPageSections() { addManualPageSection("ATLAS LOADER", AtlasLoaderCommand.class.getResourceAsStream("AtlasLoaderCommandSection.txt")); super.registerManualPageSections(); } @Override public void registerOptionsAndArguments() { final Integer[] contexts = this.optionAndArgumentDelegate.getFilteredRegisteredContexts() .toArray(new Integer[0]); registerOption(STRICT_OPTION_LONG, STRICT_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, contexts); registerOption(PARALLEL_OPTION_LONG, PARALLEL_OPTION_SHORT, PARALLEL_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, contexts); registerOption(COMBINE_OPTION_LONG, COMBINE_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, contexts); registerArgument(INPUT_HINT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, contexts); super.registerOptionsAndArguments(); } /** * After all atlas files have been handled, the subclass can override this method for final * notification and processing. The return value is sent back to the caller through System.exit * * @return a status value returned through System.exit */ protected int finish() { return 0; // NOSONAR } /** * Subclasses can implement this method for processing each atlas object as it is loaded. * * @param atlas * the atlas to process * @param atlasFileName * name of the atlas file resource * @param atlasResource * the {@link File} resource from which the atlas was loaded */ protected abstract void processAtlas(Atlas atlas, String atlasFileName, File atlasResource); /** * Subclasses can override this method if they want to do something once before processing the * atlases. The start method can return a status to indicate if the start-up operations were * successful. On return 0, the command will continue execution. On any non-zero exit code, the * command will terminate early and return the code through System.exit. * * @return a status value returned through System.exit */ protected int start() { return 0; // NOSONAR } /** * Get a list of input atlas resources with their associated atlases, one for each atlas loaded * from the input-atlases parameter. This method can be used when overriding the execute method, * in place of the standard start(), handle(), finish() semantics. * * @return the list of atlases */ private List> getInputAtlases() { if (this.atlases == null) { this.atlases = new ArrayList<>(); } else { return this.atlases; } final List inputAtlasPaths = this.optionAndArgumentDelegate .getVariadicArgument(INPUT_HINT); final AtlasResourceLoader loader = new AtlasResourceLoader(); inputAtlasPaths.forEach(path -> { final File file = new File(path, this.getFileSystem(), false); if (!file.exists()) { this.outputDelegate.printlnWarnMessage("file not found: " + path); } else if (file.isDirectory()) { this.outputDelegate.printlnWarnMessage("skipping directory: " + path); } else { if (this.optionAndArgumentDelegate.hasVerboseOption()) { this.outputDelegate.printlnCommandMessage("loading " + path); } final Optional atlas = loader.safeLoad(file); if (atlas.isPresent()) { this.atlases.add(new Tuple<>(file, atlas.get())); } else { this.outputDelegate.printlnWarnMessage("could not load: " + file); } } }); if (this.optionAndArgumentDelegate.hasOption(STRICT_OPTION_LONG) && this.atlases.size() != inputAtlasPaths.size()) { this.outputDelegate.printlnErrorMessage("strict load is missing some atlas(es)"); this.atlases = new ArrayList<>(); } return this.atlases; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/AtlasLoaderTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.IntSupplier; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; import org.openstreetmap.atlas.geography.atlas.multi.MultiAtlas; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentArity; import org.openstreetmap.atlas.utilities.command.parsing.ArgumentOptionality; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.function.TernaryConsumer; import org.openstreetmap.atlas.utilities.tuples.Tuple; /** * An {@link AtlasShellToolsCommandTemplate} for commands that want to read and process some input * {@link Atlas}es. * * @author lcram */ public class AtlasLoaderTemplate implements AtlasShellToolsCommandTemplate { private static final String COMBINED_ATLAS_NAME = "combined.atlas"; private static final String INPUT_HINT = "input-atlases"; private static final String COMBINE_OPTION_LONG = "combine"; private static final String STRICT_OPTION_LONG = "strict"; private static final String PARALLEL_OPTION_LONG = "parallel"; private static final Character PARALLEL_OPTION_SHORT = 'p'; private final Integer[] contexts; /** * Execute a command using the {@link AtlasLoaderTemplate} in a structured way. This is entirely * optional, but highly recommended as it handles boilerplate functionality for you * automatically. * * @param parentCommand * the parent command that controls this template * @param startUpFunction * Provide this function if you want to do something once before processing the * atlases. The start function can return a status to indicate if the start-up * operations were successful. On return 0, {@code execute} will continue execution. * On any non-zero exit code, the {@code execute} will return this function's exit * value. Pass {@code null} to skip the startup step. * @param processAtlasFunction * This function processes each atlas object as it is loaded. It is not optional, you * may not pass {@code null} for this function. The processAtlasFunction receives as * arguments an Atlas object, a String name of the Atlas file resource, and the File * resource object from which the Atlas was loaded. * @param finishUpFunction * Provide this method to run after all atlas files have been handled for final * notification and processing. The exit value of this function will be returned to * the caller of {@code execute}. Pass {@code null} to skip the finish up step. * @return An exit value for the command. Callers can simply return this from their execute * methods. */ public static int execute(final AbstractAtlasShellToolsCommand parentCommand, final IntSupplier startUpFunction, final TernaryConsumer processAtlasFunction, final IntSupplier finishUpFunction) { Objects.requireNonNull(processAtlasFunction); /* * Run the user's optionally supplied start-up function. */ if (startUpFunction != null) { final int returnCode = startUpFunction.getAsInt(); if (returnCode != 0) { return returnCode; } } /* * Get the input atlases and run the process code on each. We optionally collapse into a * MultiAtlas or stream parallel, depending on user options. */ final List> atlasTuples = getInputAtlases(parentCommand); if (atlasTuples.isEmpty()) { parentCommand.getCommandOutputDelegate() .printlnErrorMessage("no atlas files were loaded"); return 1; } Stream> atlasTupleStream = atlasTuples.stream(); if (parentCommand.getOptionAndArgumentDelegate().hasOption(PARALLEL_OPTION_LONG)) { atlasTupleStream = atlasTupleStream.parallel(); } if (parentCommand.getOptionAndArgumentDelegate().hasOption(COMBINE_OPTION_LONG)) { if (parentCommand.getOptionAndArgumentDelegate().hasVerboseOption()) { parentCommand.getCommandOutputDelegate() .printlnCommandMessage("processing all atlases as one multiatlas..."); } processAtlasFunction.accept( new MultiAtlas( atlasTupleStream.map(Tuple::getSecond).collect(Collectors.toList())), COMBINED_ATLAS_NAME, new File(COMBINED_ATLAS_NAME, parentCommand.getFileSystem())); } else { final int size = atlasTuples.size(); final int[] count = { 1 }; atlasTupleStream.forEach(atlasTuple -> { if (parentCommand.getOptionAndArgumentDelegate().hasVerboseOption()) { parentCommand.getCommandOutputDelegate() .printlnCommandMessage("processing atlas " + atlasTuple.getFirst().getAbsolutePathString() + " (" + count[0] + "/" + size + ")"); } processAtlasFunction.accept(atlasTuple.getSecond(), atlasTuple.getFirst().getName(), atlasTuple.getFirst()); count[0]++; }); } /* * Run the user's optionally supplied finish-up function. */ if (finishUpFunction != null) { return finishUpFunction.getAsInt(); } return 0; } /** * Get a list of input atlas resources with their associated atlases, one for each atlas loaded * from the input-atlases parameter. * * @return the list of atlases */ private static List> getInputAtlases( final AbstractAtlasShellToolsCommand parentCommand) { final List> atlases = new ArrayList<>(); final List inputAtlasPaths = parentCommand.getOptionAndArgumentDelegate() .getVariadicArgument(INPUT_HINT); final AtlasResourceLoader loader = new AtlasResourceLoader(); inputAtlasPaths.forEach(path -> { final File file = new File(path, parentCommand.getFileSystem(), false); if (!file.exists()) { parentCommand.getCommandOutputDelegate() .printlnWarnMessage("file not found: " + path); } else if (file.isDirectory()) { parentCommand.getCommandOutputDelegate() .printlnWarnMessage("skipping directory: " + path); } else { if (parentCommand.getOptionAndArgumentDelegate().hasVerboseOption()) { parentCommand.getCommandOutputDelegate() .printlnCommandMessage("loading " + path); } final Optional atlas = loader.safeLoad(file); if (atlas.isPresent()) { atlases.add(new Tuple<>(file, atlas.get())); } else { parentCommand.getCommandOutputDelegate() .printlnWarnMessage("could not load: " + file); } } }); if (parentCommand.getOptionAndArgumentDelegate().hasOption(STRICT_OPTION_LONG) && atlases.size() != inputAtlasPaths.size()) { parentCommand.getCommandOutputDelegate() .printlnErrorMessage("strict load is missing some atlas(es)"); atlases.clear(); } return atlases; } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public AtlasLoaderTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("ATLAS LOADER", AtlasLoaderTemplate.class.getResourceAsStream("AtlasLoaderTemplateSection.txt")); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOption(STRICT_OPTION_LONG, "Fail fast if any input atlases are missing.", OptionOptionality.OPTIONAL, this.contexts); parentCommand.registerOption(PARALLEL_OPTION_LONG, PARALLEL_OPTION_SHORT, "Process the atlases in parallel.", OptionOptionality.OPTIONAL, this.contexts); parentCommand.registerOption(COMBINE_OPTION_LONG, "Combine all input atlases into a MultiAtlas before processing.", OptionOptionality.OPTIONAL, this.contexts); parentCommand.registerArgument(INPUT_HINT, ArgumentArity.VARIADIC, ArgumentOptionality.REQUIRED, this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/CountryBoundaryMapTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.util.Optional; import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; /** * An {@link AtlasShellToolsCommandTemplate} for commands that want to read an input * {@link CountryBoundaryMap}. * * @author lcram */ public class CountryBoundaryMapTemplate implements AtlasShellToolsCommandTemplate { public static final String COUNTRY_BOUNDARY_OPTION_LONG = "country-boundary"; private final Integer[] contexts; /** * Get a {@link CountryBoundaryMap} object from the user's input option. * * @param parentCommand * the parent command that controls this template * @return the {@link CountryBoundaryMap} object built from the file specified by the user */ public static Optional getCountryBoundaryMap( final AbstractAtlasShellToolsCommand parentCommand) { final Optional countryBoundaryMap; final File boundaryMapFile = new File(parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(COUNTRY_BOUNDARY_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new), parentCommand.getFileSystem()); if (!boundaryMapFile.exists()) { parentCommand.getCommandOutputDelegate().printlnErrorMessage( "boundary file " + boundaryMapFile.getAbsolutePathString() + " does not exist"); return Optional.empty(); } if (parentCommand.getOptionAndArgumentDelegate().hasVerboseOption()) { parentCommand.getCommandOutputDelegate() .printlnCommandMessage("loading country boundary map..."); } countryBoundaryMap = Optional.of(CountryBoundaryMap.fromPlainText(boundaryMapFile)); if (parentCommand.getOptionAndArgumentDelegate().hasVerboseOption()) { parentCommand.getCommandOutputDelegate().printlnCommandMessage("loaded boundary map"); } return countryBoundaryMap; } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public CountryBoundaryMapTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("INPUT COUNTRY BOUNDARY MAP", ShardingTemplate.class .getResourceAsStream("CountryBoundaryMapTemplateSection.txt")); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOptionWithRequiredArgument(COUNTRY_BOUNDARY_OPTION_LONG, "A boundary file to use for intersection checks. See INPUT COUNTRY BOUNDARY MAP section for details.", OptionOptionality.REQUIRED, "boundary-file", this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/ListOfNumbersTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.TemplateTestCommand; /** * An example of how to implement an {@link AtlasShellToolsCommandTemplate}. This template simply * provides an option that accepts a comma separated list of numbers. Note that the code to parse * the option can be contained within the template itself! Check {@link TemplateTestCommand} to see * how to use the template in a command implementation. * * @author lcram */ public class ListOfNumbersTemplate implements AtlasShellToolsCommandTemplate { private static final String LIST_OF_NUMBERS_OPTION_LONG = "list-of-numbers"; private static final String COULD_NOT_PARSE = "could not parse %s '%s'"; /** * The parse contexts under which we want the options provided by this template to appear. Leave * empty to use the default context. */ private final Integer[] contexts; public static List getListOfNumbers(final AbstractAtlasShellToolsCommand parentCommand) { final String listString = parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(LIST_OF_NUMBERS_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new); if (listString.isEmpty()) { return new ArrayList<>(); } final List numberList = new ArrayList<>(); final String[] listStringSplit = listString.split(","); for (final String numberElement : listStringSplit) { final int number; try { number = Integer.parseInt(numberElement); numberList.add(number); } catch (final NumberFormatException exception) { parentCommand.getCommandOutputDelegate().printlnErrorMessage( String.format(COULD_NOT_PARSE, "number", numberElement)); return new ArrayList<>(); } } return numberList; } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public ListOfNumbersTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("LIST OF NUMBERS TEMPLATE", new ByteArrayInputStream( ("This is an example man page section for the ListOfNumbersTemplate! " + "This template adds an option that reads a list of numbers.") .getBytes(StandardCharsets.UTF_8))); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOptionWithRequiredArgument(LIST_OF_NUMBERS_OPTION_LONG, "Specify a comma separated list of numbers.", OptionOptionality.REQUIRED, "numbers", this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/MultipleOutputCommand.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.CommandOutputDelegate; import org.openstreetmap.atlas.utilities.command.abstractcommand.OptionAndArgumentDelegate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.command.subcommands.AnyToGeoJsonCommand; /** * A {@link MultipleOutputCommand} is any command which may produce multiple output files, and wants * to provide users with a way to specify the desired location of those files. This command template * registers a '--output' option, the argument to which will be used as an output directory. * * @author lcram * @deprecated Please use {@link OutputDirectoryTemplate} instead. Check {@link AnyToGeoJsonCommand} * for an example. */ @Deprecated public abstract class MultipleOutputCommand extends AbstractAtlasShellToolsCommand { public static final String OUTPUT_DIRECTORY_OPTION_LONG = "output"; private static final Character OUTPUT_DIRECTORY_OPTION_SHORT = 'o'; private static final String OUTPUT_DIRECTORY_OPTION_DESCRIPTION = "Specify an alternate output directory for any output files. If the directory " + "does not exist, it will be created."; private static final String OUTPUT_DIRECTORY_OPTION_HINT = "dir"; private final OptionAndArgumentDelegate optionAndArgumentDelegate; private final CommandOutputDelegate outputDelegate; private Path outputPath; public MultipleOutputCommand() { this.optionAndArgumentDelegate = this.getOptionAndArgumentDelegate(); this.outputDelegate = this.getCommandOutputDelegate(); this.outputPath = null; } /** * Populate the output path field. Subclasses should override this method, but invoke it with * super.execute to populate the outputPath field. The subclass can check the return code of * this method to see if the output path was parsed successfully. * * @return the exit status, 0 indicates success while 1 indicates that the output path was * invalid */ @Override public int execute() { final Optional outputPathOptional = parseOutputPath(); if (outputPathOptional.isEmpty()) { this.outputDelegate.printlnErrorMessage("invalid output path"); return 1; } else { this.outputPath = outputPathOptional.get(); } return 0; } public Path getOutputPath() { return this.outputPath; } @Override public void registerManualPageSections() { addManualPageSection("MULTIPLE OUTPUT", MultipleOutputCommand.class .getResourceAsStream("MultipleOutputCommandSection.txt")); } @Override public void registerOptionsAndArguments() { final Integer[] contexts = this.optionAndArgumentDelegate.getFilteredRegisteredContexts() .toArray(new Integer[0]); registerOptionWithRequiredArgument(OUTPUT_DIRECTORY_OPTION_LONG, OUTPUT_DIRECTORY_OPTION_SHORT, OUTPUT_DIRECTORY_OPTION_DESCRIPTION, OptionOptionality.OPTIONAL, OUTPUT_DIRECTORY_OPTION_HINT, contexts); super.registerOptionsAndArguments(); } private Optional parseOutputPath() { final Path outputParentPath = this.getFileSystem().getPath(this.optionAndArgumentDelegate .getOptionArgument(OUTPUT_DIRECTORY_OPTION_LONG).orElse("")); // If output path already exists and is a file, then fail if (Files.isRegularFile(outputParentPath)) { this.outputDelegate.printlnErrorMessage( outputParentPath.toString() + " already exists and is a file"); return Optional.empty(); } // If output path does not exist, create it using 'mkdir -p' behaviour if (!Files.exists(outputParentPath)) { try { new File(outputParentPath.toAbsolutePath().toString(), this.getFileSystem()) .mkdirs(); } catch (final Exception exception) { this.outputDelegate.printlnErrorMessage( "failed to create output directory " + outputParentPath.toString()); return Optional.empty(); } } // If output path is not writable, fail if (!Files.isWritable(outputParentPath)) { this.outputDelegate .printlnErrorMessage(outputParentPath.toString() + " is not writable"); return Optional.empty(); } return Optional.of(outputParentPath); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/OutputDirectoryTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; import org.openstreetmap.atlas.streaming.resource.File; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link OutputDirectoryTemplate} provides a template for any command which may produce * multiple output files and wants to provide users with a way to specify the desired location of * those files. This command template registers an '--output-directory' option, the argument to * which will be possibly created and used as an output directory. * * @author lcram */ public class OutputDirectoryTemplate implements AtlasShellToolsCommandTemplate { private static final Logger logger = LoggerFactory.getLogger(OutputDirectoryTemplate.class); private static final String OUTPUT_DIRECTORY_OPTION_LONG = "output-directory"; private static final Character OUTPUT_DIRECTORY_OPTION_SHORT = 'o'; private final Integer[] contexts; /** * Get the output path specified by the user. If the returned {@link Optional} is empty, then * the output path could not be parsed and it is recommended that you exit with an error. * * @param parentCommand * the parent command that controls this template * @return an {@link Optional} containing the output path for this command */ public static Optional getOutputPath(final AbstractAtlasShellToolsCommand parentCommand) { /* * Grab output path from --output-directory option if present. If that option is not * present, return the current working directory. */ final Path outputParentPath = parentCommand.getFileSystem() .getPath(parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(OUTPUT_DIRECTORY_OPTION_LONG).orElse("")); // If output path already exists and is a file, then fail if (Files.isRegularFile(outputParentPath)) { parentCommand.getCommandOutputDelegate().printlnErrorMessage( outputParentPath.toString() + " already exists and is a file"); return Optional.empty(); } // If output path does not exist, create it using 'mkdir -p' behaviour if (!Files.exists(outputParentPath)) { try { new File(outputParentPath.toAbsolutePath().toString(), parentCommand.getFileSystem()).mkdirs(); } catch (final Exception exception) { parentCommand.getCommandOutputDelegate().printlnErrorMessage( "failed to create output directory " + outputParentPath.toString()); logger.error("Failed to create output directory", exception); return Optional.empty(); } } // If output path is not writable, fail if (!Files.isWritable(outputParentPath)) { parentCommand.getCommandOutputDelegate() .printlnErrorMessage(outputParentPath.toString() + " is not writable"); return Optional.empty(); } return Optional.of(outputParentPath); } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public OutputDirectoryTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("OUTPUT DIRECTORY", OutputDirectoryTemplate.class .getResourceAsStream("OutputDirectoryTemplateSection.txt")); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOptionWithRequiredArgument(OUTPUT_DIRECTORY_OPTION_LONG, OUTPUT_DIRECTORY_OPTION_SHORT, "Specify an alternate output directory for any output files. If the directory " + "does not exist, it will be created.", OptionOptionality.OPTIONAL, "dir", this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/PredicateTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Predicate; import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; import org.openstreetmap.atlas.utilities.conversion.StringToPredicateConverter; /** * An {@link AtlasShellToolsCommandTemplate} for commands that want to read an input groovy * predicate. * * @author lcram */ public class PredicateTemplate implements AtlasShellToolsCommandTemplate { private static final String PREDICATE_OPTION_LONG = "predicate"; private static final String PREDICATE_IMPORTS_OPTION_LONG = "imports"; private final Integer[] contexts; public static Optional> getPredicate(final Class clazz, final List importsAllowList, final AbstractAtlasShellToolsCommand parentCommand) { if (parentCommand.getOptionAndArgumentDelegate().hasOption(PREDICATE_OPTION_LONG)) { final String predicateString = parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(PREDICATE_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new); List userImports = new ArrayList<>(); if (parentCommand.getOptionAndArgumentDelegate() .hasOption(PREDICATE_IMPORTS_OPTION_LONG)) { userImports = StringList .split(parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(PREDICATE_IMPORTS_OPTION_LONG) .orElseThrow(AtlasShellToolsException::new), ",") .getUnderlyingList(); } final List allImports = new ArrayList<>(); allImports.addAll(userImports); allImports.addAll(importsAllowList); return Optional.of(new StringToPredicateConverter() .withAddedStarImportPackages(allImports).convert(predicateString)); } return Optional.empty(); } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public PredicateTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("PREDICATE", ShardingTemplate.class.getResourceAsStream("PredicateTemplateSection.txt")); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOptionWithRequiredArgument(PREDICATE_OPTION_LONG, "A flexible groovy predicate supplied at the command line. See DESCRIPTION and PREDICATE sections for details.", OptionOptionality.OPTIONAL, "groovy-code", this.contexts); parentCommand.registerOptionWithRequiredArgument(PREDICATE_IMPORTS_OPTION_LONG, "A comma separated list of some additional package imports to include for the predicate option, if present.", OptionOptionality.OPTIONAL, "packages", this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/subcommands/templates/ShardingTemplate.java ================================================ package org.openstreetmap.atlas.utilities.command.subcommands.templates; import org.openstreetmap.atlas.geography.sharding.Sharding; import org.openstreetmap.atlas.utilities.command.AtlasShellToolsException; import org.openstreetmap.atlas.utilities.command.abstractcommand.AbstractAtlasShellToolsCommand; import org.openstreetmap.atlas.utilities.command.abstractcommand.AtlasShellToolsCommandTemplate; import org.openstreetmap.atlas.utilities.command.parsing.OptionOptionality; /** * An {@link AtlasShellToolsCommandTemplate} for commands that want to read an input * {@link Sharding}. * * @author lcram */ public class ShardingTemplate implements AtlasShellToolsCommandTemplate { private static final String SHARDING_OPTION_LONG = "sharding"; private final Integer[] contexts; /** * Get a {@link Sharding} object from the user's input option. * * @param parentCommand * the parent command that controls this template * @return the {@link Sharding} object specified by the user */ public static Sharding getSharding(final AbstractAtlasShellToolsCommand parentCommand) { return Sharding.forString(parentCommand.getOptionAndArgumentDelegate() .getOptionArgument(SHARDING_OPTION_LONG).orElseThrow(AtlasShellToolsException::new), parentCommand.getFileSystem()); } /** * This constructor allows callers to specify under which contexts they want the options * provided by this template to appear. If left blank, this template will only be applied to the * default context. * * @param contexts * the parse contexts under which you want the options provided by this template to * appear */ public ShardingTemplate(final Integer... contexts) { this.contexts = contexts; } @Override public void registerManualPageSections(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.addManualPageSection("INPUT SHARDING", ShardingTemplate.class.getResourceAsStream("ShardingTemplateSection.txt")); } @Override public void registerOptionsAndArguments(final AbstractAtlasShellToolsCommand parentCommand) { parentCommand.registerOptionWithRequiredArgument(SHARDING_OPTION_LONG, "The sharding to use, e.g. slippy@9, dynamic@/Users/foo/my-tree.txt, geohash@4, etc.", OptionOptionality.REQUIRED, "type@parameter", this.contexts); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/terminal/TTYAttribute.java ================================================ package org.openstreetmap.atlas.utilities.command.terminal; /** * Easy mnemonics for TTY display attributes (ANSI control codes) and their Unicode encodings. * * @see "https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_(Select_Graphic_Rendition)_parameters" * @author lcram */ public enum TTYAttribute { BOLD("\u001B[1m"), FAINT("\u001B[2m"), ITALIC("\u001B[3m"), UNDERLINE("\u001B[4m"), BLINK("\u001B[5m"), RAPID_BLINK("\u001B[6m"), REVERSE_VIDEO("\u001B[7m"), BLACK("\u001B[30m"), RED("\u001B[31m"), GREEN("\u001B[32m"), YELLOW("\u001B[33m"), BLUE("\u001B[34m"), MAGENTA("\u001B[35m"), CYAN("\u001B[36m"), WHITE("\u001B[037m"), BACKGROUND_BLACK("\u001B[40m"), BACKGROUND_RED("\u001B[41m"), BACKGROUND_GREEN("\u001B[42m"), BACKGROUND_YELLOW("\u001B[43m"), BACKGROUND_BLUE("\u001B[44m"), BACKGROUND_MAGENTA("\u001B[45m"), BACKGROUND_CYAN("\u001B[46m"), BACKGROUND_WHITE("\u001B[047m"), RESET("\u001B[0m"); private final String ansiSequence; TTYAttribute(final String ansiSequence) { this.ansiSequence = ansiSequence; } public String getANSISequence() { return this.ansiSequence; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/command/terminal/TTYStringBuilder.java ================================================ package org.openstreetmap.atlas.utilities.command.terminal; import java.util.ArrayDeque; import java.util.Deque; import org.openstreetmap.atlas.exception.CoreException; /** * A simple string building class that allows for optional TTY formatting and output * pretty-fication. * * @author lcram */ public class TTYStringBuilder { public static final int DEFAULT_LEVEL_WIDTH = 4; private final StringBuilder builder; private final boolean useColors; private final Deque exactIndentWidthStack; private int levelWidth; public TTYStringBuilder(final boolean useColors) { this.builder = new StringBuilder(); this.useColors = useColors; this.exactIndentWidthStack = new ArrayDeque<>(); this.exactIndentWidthStack.push(0); this.levelWidth = DEFAULT_LEVEL_WIDTH; } public TTYStringBuilder append(final Object object, final TTYAttribute... attributes) { // Append whitespace for the indent setting for (int i = 0; i < this.exactIndentWidthStack.peek(); i++) { this.builder.append(" "); } if (this.useColors) { for (final TTYAttribute attribute : attributes) { this.builder.append(attribute.getANSISequence()); } } this.builder.append(String.valueOf(object)); // If an attribute was supplied, we need to reset the TTY if (this.useColors && attributes.length > 0) { this.builder.append(TTYAttribute.RESET.getANSISequence()); } return this; } public TTYStringBuilder clearIndentationStack() { this.exactIndentWidthStack.clear(); this.exactIndentWidthStack.push(0); return this; } /** * Append a newline to this builder. * * @return the updated builder */ public TTYStringBuilder newline() { this.builder.append(System.getProperty("line.separator")); return this; } public TTYStringBuilder popIndentation() { if (this.exactIndentWidthStack.size() == 1) { throw new CoreException("Cannot pop default indention off the stack"); } this.exactIndentWidthStack.pop(); return this; } public TTYStringBuilder pushExactIndentWidth(final int width) { if (width < 0) { throw new CoreException("Indent width ({}) must be >= 0", width); } this.exactIndentWidthStack.push(width); return this; } public TTYStringBuilder pushIndentLevel(final int level) { if (level < 0) { throw new CoreException("Indent level ({}) must be >= 0", level); } this.exactIndentWidthStack.push(level * this.levelWidth); return this; } @Override public String toString() { return this.builder.toString(); } public TTYStringBuilder withLevelWidth(final int newLevelWidth) { if (newLevelWidth < 0) { throw new CoreException("Level width ({}) must be >= 0", newLevelWidth); } this.levelWidth = newLevelWidth; return this; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/compression/IntegerDictionary.java ================================================ package org.openstreetmap.atlas.utilities.compression; import java.io.Serializable; import java.util.HashMap; import java.util.Map; import org.openstreetmap.atlas.proto.ProtoSerializable; import org.openstreetmap.atlas.proto.adapters.ProtoAdapter; import org.openstreetmap.atlas.proto.adapters.ProtoIntegerStringDictionaryAdapter; /** * Simple dictionary encoding for {@link String}s * * @author matthieun * @author lcram * @param * The type to encode. Typically called word */ public class IntegerDictionary implements Serializable, ProtoSerializable { private static final long serialVersionUID = -1781411097803512149L; public static final String FIELD_WORD_TO_INDEX = "wordToIndex"; public static final String FIELD_INDEX_TO_WORD = "indexToWord"; public static final String FIELD_INDEX = "index"; private final Map wordToIndex; private final Map indexToWord; private int index = 0; public IntegerDictionary() { this.wordToIndex = new HashMap<>(); this.indexToWord = new HashMap<>(); } public synchronized int add(final Type word) { if (this.wordToIndex.containsKey(word)) { return this.wordToIndex.get(word); } this.wordToIndex.put(word, this.index); this.indexToWord.put(this.index, word); return this.index++; } @Override public boolean equals(final Object other) { if (other instanceof IntegerDictionary) { if (this == other) { return true; } @SuppressWarnings("unchecked") final IntegerDictionary that = (IntegerDictionary) other; if (this.size() != that.size()) { return false; } if (!this.wordToIndex.equals(that.wordToIndex)) { return false; } if (!this.indexToWord.equals(that.indexToWord)) { return false; } return true; } return false; } /* * TODO the problem here is that if someone tries to get an adapter for an * IntegerDictionary, this method will return the wrong adapter. * Currently, the ProtoIntegerStringDictionaryAdapter class's serialize() method handles this by * catching a ClassCastException and rethrowing a CoreException with a better message. */ @Override public ProtoAdapter getProtoAdapter() { return new ProtoIntegerStringDictionaryAdapter(); } @Override public int hashCode() { final int initialPrime = 31; final int hashSeed = 37; int hash = hashSeed * initialPrime + Integer.valueOf(this.size()).hashCode(); for (final Type key : this.wordToIndex.keySet()) { final Integer value = this.wordToIndex.get(key); final int keyHash = key == null ? 0 : key.hashCode(); final int valueHash = value == null ? 0 : value.hashCode(); hash = hashSeed * hash + keyHash; hash = hashSeed * hash + valueHash; } for (final Integer key : this.indexToWord.keySet()) { final Type value = this.indexToWord.get(key); final int keyHash = key == null ? 0 : key.hashCode(); final int valueHash = value == null ? 0 : value.hashCode(); hash = hashSeed * hash + keyHash; hash = hashSeed * hash + valueHash; } return hash; } public int size() { return this.index; } public Type word(final int index) { return this.indexToWord.get(index); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/compression/LongDictionary.java ================================================ package org.openstreetmap.atlas.utilities.compression; import java.io.Serializable; import java.util.HashMap; import java.util.Map; /** * Simple dictionary encoding for {@link String}s * * @author matthieun * @param * The type to encode. Typically called word */ public class LongDictionary implements Serializable { private static final long serialVersionUID = 6060400113166584385L; private final Map wordToIndex; private final Map indexToWord; private long index = 0; public LongDictionary() { this.wordToIndex = new HashMap<>(); this.indexToWord = new HashMap<>(); } public synchronized long add(final Type word) { if (this.wordToIndex.containsKey(word)) { return this.wordToIndex.get(word); } this.wordToIndex.put(word, this.index); this.indexToWord.put(this.index, word); return this.index++; } public Type word(final long index) { return this.indexToWord.get(index); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/Configurable.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.util.Optional; /** * Configurable wrapper * * @author brian_l_davis */ public interface Configurable { /** * @param * property type * @return the current value */ V value(); /** * @param * property type * @return Optional of the current value, wrapping {@code null} */ Optional valueOption(); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/Configuration.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.util.Optional; import java.util.Set; import java.util.function.Function; /** * Configuration interface providing key-value look with support for defaults and transformations. * * @author brian_l_davis */ public interface Configuration { /** * Returns a set view of all the top level keys in this {@link Configuration}. * * @return the set of top level keys */ Set configurationDataKeySet(); /** * Returns a returns a copy Configuration specific to a keyword with overwritten values * * @param keyword * keyword string * @return Configuration */ Configuration configurationForKeyword(String keyword); /** * Returns a {@link Configurable} wrapper around the configured property. * * @param key * property key * @return a {@link Configurable} wrapper */ Configurable get(String key); /** * Returns a {@link Configurable} wrapper around the configured property. * * @param key * property key * @param transform * applied to the configured property * @param * configured type * @param * transformed type * @return a {@link Configurable} wrapper */ Configurable get(String key, Function transform); /** * Returns a {@link Configurable} wrapper around the configured property. * * @param key * property key * @param defaultValue * value returned if not found in the configuration * @param transform * applied to the configured property * @param * configured type * @param * transformed type * @return a {@link Configurable} wrapper */ Configurable get(String key, R defaultValue, Function transform); /** * Returns a {@link Configurable} wrapper around the configured property. * * @param key * property key * @param defaultValue * value returned if not found in the configuration * @param * configured type * @return a {@link Configurable} wrapper */ Configurable get(String key, T defaultValue); /** * Returns a new configuration with contents starting at the provided key. *

* Assuming the initial configuration is: * * 

     * {@code
     * {
     *     "a" :
     *     {
     *         "b" : "c"
     *     }
     *
     * }
     * }
     *
* * With a key provided as "a", the new sub configuration looks like: * * 
     * {@code
     * {
     *     "b" : "c"
     * }
     * }
     *
* * @param key * The provided key * @return The sub Configuration if it exists under the key, Optional.empty otherwise. */ Optional subConfiguration(String key); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/ConfigurationDeserializer.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.io.IOException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.Map; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonMappingException; /** * Basic {@link JsonDeserializer} the builds a key-value maps for quick lookup * * @author cstaylor * @author brian_l_davis */ class ConfigurationDeserializer extends JsonDeserializer> { private static final String INVALID_JSON = "Invalid JSON format."; private static final String DOT = "."; @Override public Map deserialize(final JsonParser jacksonParser, final DeserializationContext context) throws IOException { return deserializeObject(jacksonParser, context); } private ArrayList deserializeArray(final JsonParser jacksonParser, final DeserializationContext context) throws IOException { if (jacksonParser.getCurrentToken() != JsonToken.START_ARRAY) { throw new JsonMappingException(jacksonParser, INVALID_JSON); } jacksonParser.nextToken(); final ArrayList list = new ArrayList<>(); while (jacksonParser.getCurrentToken() != JsonToken.END_ARRAY) { list.add(deserializeValue(jacksonParser, context)); jacksonParser.nextToken(); } return list; } private Map deserializeObject(final JsonParser jacksonParser, final DeserializationContext context) throws IOException { if (jacksonParser.getCurrentToken() != JsonToken.START_OBJECT) { throw new JsonMappingException(jacksonParser, INVALID_JSON); } final Map map = new LinkedHashMap<>(); while (jacksonParser.nextToken() != JsonToken.END_OBJECT) { final String dotKey = jacksonParser.getCurrentName(); final Map locatedMap = locateMap(dotKey, map, jacksonParser); final String key; final int lastDot = dotKey.lastIndexOf(DOT); if (lastDot > 0 && lastDot < dotKey.length()) { key = dotKey.substring(lastDot + 1); } else { key = dotKey; } // skip key, go to value jacksonParser.nextToken(); final Object deserializedValue = deserializeValue(jacksonParser, context); locatedMap.put(key, deserializedValue); } return map; } private Object deserializeValue(final JsonParser jacksonParser, final DeserializationContext context) throws IOException { switch (jacksonParser.getCurrentToken()) { case START_OBJECT: return deserializeObject(jacksonParser, context); case START_ARRAY: return deserializeArray(jacksonParser, context); case VALUE_STRING: return jacksonParser.getText(); case VALUE_NUMBER_INT: return jacksonParser.getLongValue(); case VALUE_NUMBER_FLOAT: return jacksonParser.getDoubleValue(); case VALUE_TRUE: return Boolean.TRUE; case VALUE_FALSE: return Boolean.FALSE; case VALUE_NULL: return null; default: return context.handleUnexpectedToken(Object.class, jacksonParser); } } @SuppressWarnings("unchecked") private Map locateMap(final String dotKey, final Map map, final JsonParser jacksonParser) throws IOException { final int dotIndex = dotKey.indexOf(DOT); if (dotIndex > 0 && dotIndex < dotKey.length()) { final String part = dotKey.substring(0, dotIndex); final String remaining = dotKey.substring(dotIndex + 1); final Object nextMap = map.getOrDefault(part, new LinkedHashMap<>()); if (!(nextMap instanceof Map)) { throw new JsonMappingException(jacksonParser, INVALID_JSON); } map.put(part, nextMap); return locateMap(remaining, (Map) nextMap, jacksonParser); } return map; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/ConfigurationReader.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.util.List; import java.util.function.Function; import org.openstreetmap.atlas.exception.CoreException; import com.google.common.collect.Lists; /** * Helper class to read a {@link Configuration} * * @author matthieun */ public class ConfigurationReader { private static final String CONFIGURATION_PATH_NAME_DEFAULT = "N/A"; private final String root; public ConfigurationReader(final String root) { this.root = root; } public final String configurationKey(final String key) { return this.root.isEmpty() ? key : this.root + "." + key; } public String configurationValue(final Configuration configuration, final String key) { final String result = configuration .get(configurationKey(key), CONFIGURATION_PATH_NAME_DEFAULT).value(); if (CONFIGURATION_PATH_NAME_DEFAULT.equals(result)) { throw new CoreException("Malformed configuration for {}", configurationKey(key)); } return result; } public U configurationValue(final Configuration configuration, final String key, final U defaultValue) { return configuration.get(configurationKey(key), defaultValue).value(); } public T configurationValue(final Configuration configuration, final Function defaultValue) { return configuration.get(this.root, defaultValue).value(); } public List configurationValues(final Configuration configuration, final String key, final List defaultValue) { return configuration.get(configurationKey(key), defaultValue).value(); } public List configurationValues(final Configuration configuration, final String key) { final List defaults = Lists.newArrayList(CONFIGURATION_PATH_NAME_DEFAULT); final List result = configurationValues(configuration, key, defaults); if (defaults.equals(result)) { throw new CoreException("Malformed configuration for {}", configurationKey(key)); } return result; } public boolean isPresent(final Configuration configuration, final String key) { final Object result = configuration .get(configurationKey(key), CONFIGURATION_PATH_NAME_DEFAULT).value(); return !CONFIGURATION_PATH_NAME_DEFAULT.equals(result); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/ConfiguredFilter.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.MultiPolygon; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.openstreetmap.atlas.geography.converters.WkbMultiPolygonConverter; import org.openstreetmap.atlas.geography.converters.WktMultiPolygonConverter; import org.openstreetmap.atlas.streaming.resource.StringResource; import org.openstreetmap.atlas.tags.Taggable; import org.openstreetmap.atlas.tags.filters.RegexTaggableFilter; import org.openstreetmap.atlas.tags.filters.TaggableFilter; import org.openstreetmap.atlas.tags.filters.matcher.TaggableMatcher; import org.openstreetmap.atlas.utilities.conversion.HexStringByteArrayConverter; import org.openstreetmap.atlas.utilities.conversion.StringToPredicateConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; /** * This class reads in a configuration file with a specific schema and creates filters based on the * predicates and taggable filter specified in the file. Take a look at water-handlers.json for * reference * * @author matthieun */ public final class ConfiguredFilter implements Predicate, Serializable { public static final String CONFIGURATION_GLOBAL = "global"; public static final String DEFAULT = "default"; public static final ConfiguredFilter NO_FILTER = new ConfiguredFilter(); public static final String CONFIGURATION_ROOT = CONFIGURATION_GLOBAL + ".filters"; /* * JSON constants for the toJson method. We should probably handle this better so that we do not * duplicate String literals. */ public static final String TYPE_JSON_PROPERTY_VALUE = "_filter"; public static final String NAME_JSON_PROPERTY = "name"; public static final String PREDICATE_JSON_PROPERTY = "predicate"; public static final String UNSAFE_PREDICATE_JSON_PROPERTY = "unsafePredicate"; public static final String IMPORTS_JSON_PROPERTY = "imports"; public static final String TAGGABLE_FILTER_JSON_PROPERTY = "taggableFilter"; public static final String REGEX_TAGGABLE_FILTER_JSON_PROPERTY = "regexTaggableFilter"; public static final String TAGGABLE_MATCHER_JSON_PROPERTY = "taggableMatcher"; public static final String NO_EXPANSION_JSON_PROPERTY = "noExpansion"; private static final long serialVersionUID = 7503301238426719144L; private static final Logger logger = LoggerFactory.getLogger(ConfiguredFilter.class); private static final String CONFIGURATION_PREDICATE_COMMAND = "predicate.command"; private static final String CONFIGURATION_PREDICATE_UNSAFE_COMMAND = "predicate.unsafeCommand"; private static final String CONFIGURATION_PREDICATE_IMPORTS = "predicate.imports"; private static final String CONFIGURATION_TAGGABLE_FILTER = "taggableFilter"; private static final String CONFIGURATION_REGEX_TAGGABLE_FILTER = "regexTaggableFilter"; private static final String CONFIGURATION_TAGGABLE_MATCHER = "taggableMatcher"; private static final String CONFIGURATION_WKT_FILTER = "geometry.wkt"; private static final String CONFIGURATION_WKB_FILTER = "geometry.wkb"; private static final String CONFIGURATION_HINT_NO_EXPANSION = "hint.noExpansion"; private static final WktMultiPolygonConverter WKT_MULTI_POLYGON_CONVERTER = new WktMultiPolygonConverter(); private static final WkbMultiPolygonConverter WKB_MULTI_POLYGON_CONVERTER = new WkbMultiPolygonConverter(); private static final HexStringByteArrayConverter HEX_STRING_BYTE_ARRAY_CONVERTER = new HexStringByteArrayConverter(); private final String name; private final String predicate; private final String unsafePredicate; private transient Predicate filter; private final List imports; private final String taggableFilter; private final String regexTaggableFilter; private final String taggableMatcher; private final boolean noExpansion; private final List geometryBasedFilters; public static ConfiguredFilter from(final String name, final Configuration configuration) { return from(CONFIGURATION_ROOT, name, configuration); } /** * Create a new {@link ConfiguredFilter}. *

* For example, in the following json configuration: * * 

     * {@code
     * {
     *     "my":
     *     {
     *         "conf":
     *         {
     *             "filter":
     *             {
     *                 "predicate": "....",
     *                 "geometry.wkb":
     *                 [
     *                     "...", "..."
     *                 ],
     *                 "taggableFilter": "...",
     *                 "regexTaggableFilter": "...",
     *                 "taggableMatcher": "..."
     *             }
     *         }
     *     }
     * }
     * }
     *
* * the filter can be accessed using "my.conf" as root, and "filter" as name. * * @param root * The root of the configuration hierarchy, where to search for the name of the * filter. * @param name * The name of the filter, which is right under the root in the configuration * @param configuration * The {@link Configuration} containing the configured filter * @return The constructed {@link ConfiguredFilter} */ public static ConfiguredFilter from(final String root, final String name, final Configuration configuration) { if (DEFAULT.equals(name)) { return getDefaultFilter(root, configuration); } if (!isPresent(root, name, configuration)) { logger.warn( "Attempted to create ConfiguredFilter called \"{}\" but it was not found. It will be swapped with default passthrough filter.", name); return getDefaultFilter(root, configuration); } return new ConfiguredFilter(root, name, configuration); } public static ConfiguredFilter getDefaultFilter(final Configuration configuration) { return getDefaultFilter(CONFIGURATION_ROOT, configuration); } public static ConfiguredFilter getDefaultFilter(final String root, final Configuration configuration) { if (ConfiguredFilter.isPresent(root, DEFAULT, configuration)) { return new ConfiguredFilter(root, DEFAULT, configuration); } return NO_FILTER; } public static boolean isPresent(final String name, final Configuration configuration) { return isPresent(CONFIGURATION_ROOT, name, configuration); } public static boolean isPresent(final String root, final String name, final Configuration configuration) { return new ConfigurationReader(root).isPresent(configuration, name); } private ConfiguredFilter() { this(CONFIGURATION_ROOT, "NO_FILTER", new StandardConfiguration(new StringResource("{}"))); } private ConfiguredFilter(final String root, final String name, final Configuration configuration) { this.name = name; String readerRoot = ""; if (root != null && !root.isEmpty()) { readerRoot = root + "."; } final ConfigurationReader reader = new ConfigurationReader(readerRoot + name); this.predicate = reader.configurationValue(configuration, CONFIGURATION_PREDICATE_COMMAND, ""); this.unsafePredicate = reader.configurationValue(configuration, CONFIGURATION_PREDICATE_UNSAFE_COMMAND, ""); this.imports = reader.configurationValue(configuration, CONFIGURATION_PREDICATE_IMPORTS, Lists.newArrayList()); this.taggableFilter = reader.configurationValue(configuration, CONFIGURATION_TAGGABLE_FILTER, ""); this.regexTaggableFilter = reader.configurationValue(configuration, CONFIGURATION_REGEX_TAGGABLE_FILTER, ""); this.taggableMatcher = reader.configurationValue(configuration, CONFIGURATION_TAGGABLE_MATCHER, ""); this.noExpansion = readBoolean(configuration, reader, CONFIGURATION_HINT_NO_EXPANSION, false); this.geometryBasedFilters = readGeometries(configuration, reader); } public List getGeometryBasedFilters() { return new ArrayList<>(this.geometryBasedFilters); } public String getName() { return this.name; } public boolean isNoExpansion() { return this.noExpansion; } @Override public boolean test(final AtlasEntity atlasEntity) { return getFilter().test(atlasEntity); } public boolean test(final Taggable taggable) { return TaggableFilter.forDefinition(this.taggableFilter).test(taggable); } public JsonObject toJson() { final JsonObject filterObject = new JsonObject(); filterObject.addProperty("type", TYPE_JSON_PROPERTY_VALUE); filterObject.addProperty(NAME_JSON_PROPERTY, this.name); if (!this.predicate.isEmpty()) { filterObject.addProperty(PREDICATE_JSON_PROPERTY, this.predicate); } if (!this.unsafePredicate.isEmpty()) { filterObject.addProperty(UNSAFE_PREDICATE_JSON_PROPERTY, this.unsafePredicate); } final JsonArray importsArray = new JsonArray(); if (!this.imports.isEmpty()) { for (final String importString : this.imports) { importsArray.add(new JsonPrimitive(importString)); } filterObject.add(IMPORTS_JSON_PROPERTY, importsArray); } if (!this.taggableFilter.isEmpty()) { filterObject.addProperty(TAGGABLE_FILTER_JSON_PROPERTY, this.taggableFilter); // NOSONAR } if (!this.regexTaggableFilter.isEmpty()) { filterObject.addProperty(REGEX_TAGGABLE_FILTER_JSON_PROPERTY, this.regexTaggableFilter); // NOSONAR } if (!this.taggableMatcher.isEmpty()) { filterObject.addProperty(TAGGABLE_MATCHER_JSON_PROPERTY, this.taggableMatcher); // NOSONAR } filterObject.addProperty(NO_EXPANSION_JSON_PROPERTY, this.noExpansion); return filterObject; } @Override public String toString() { return this.name; } private Predicate geometryPredicate() { if (this.geometryBasedFilters.isEmpty()) { return atlasEntity -> true; } else { return atlasEntity -> { for (final MultiPolygon multiPolygon : this.geometryBasedFilters) { if (atlasEntity.intersects(multiPolygon)) { return true; } } return false; }; } } private Predicate getFilter() { if (this.filter == null) { Predicate localTemporaryPredicate = atlasEntity -> true; final StringToPredicateConverter predicateReader = new StringToPredicateConverter<>(); predicateReader.withAddedStarImportPackages(this.imports); if (!this.predicate.isEmpty() && !this.unsafePredicate.isEmpty()) { throw new CoreException("Cannot specify both 'command' and 'unsafeCommand'"); } if (!this.predicate.isEmpty()) { localTemporaryPredicate = predicateReader.convert(this.predicate); } if (!this.unsafePredicate.isEmpty()) { localTemporaryPredicate = predicateReader.convertUnsafe(this.unsafePredicate); } final Predicate localPredicate = localTemporaryPredicate; final TaggableFilter localTaggablefilter = TaggableFilter .forDefinition(this.taggableFilter); final RegexTaggableFilter localRegexTaggableFilter = new RegexTaggableFilter( this.regexTaggableFilter); final TaggableMatcher localTaggableMatcher = TaggableMatcher.from(this.taggableMatcher); final Predicate geometryPredicate = geometryPredicate(); this.filter = atlasEntity -> localPredicate.test(atlasEntity) && localTaggablefilter.test(atlasEntity) && geometryPredicate.test(atlasEntity) && localRegexTaggableFilter.test(atlasEntity) && localTaggableMatcher.test(atlasEntity); } return this.filter; } private boolean readBoolean(final Configuration configuration, final ConfigurationReader reader, final String booleanName, final boolean defaultValue) { try { return reader.configurationValue(configuration, booleanName, defaultValue); } catch (final Exception e) { throw new CoreException("Unable to read \"{}\"", booleanName, e); } } private List readGeometries(final Configuration configuration, final ConfigurationReader reader) { final List result = new ArrayList<>(); final String defaultValue = "N/A"; try { final List values = reader.configurationValues(configuration, CONFIGURATION_WKT_FILTER, new ArrayList<>()); if (!values.isEmpty()) { result.addAll(values.stream().map(WKT_MULTI_POLYGON_CONVERTER::backwardConvert) .collect(Collectors.toList())); } } catch (final Exception e) { final String wktString = reader.configurationValue(configuration, CONFIGURATION_WKT_FILTER, defaultValue); if (!defaultValue.equals(wktString)) { result.add(WKT_MULTI_POLYGON_CONVERTER.backwardConvert(wktString)); } } try { final List values = reader.configurationValues(configuration, CONFIGURATION_WKB_FILTER, new ArrayList<>()); if (!values.isEmpty()) { result.addAll(values.stream().map(HEX_STRING_BYTE_ARRAY_CONVERTER::convert) .map(WKB_MULTI_POLYGON_CONVERTER::backwardConvert) .collect(Collectors.toList())); } } catch (final Exception e) { final String wkbString = reader.configurationValue(configuration, CONFIGURATION_WKB_FILTER, defaultValue); if (!defaultValue.equals(wkbString)) { result.add(WKB_MULTI_POLYGON_CONVERTER .backwardConvert(HEX_STRING_BYTE_ARRAY_CONVERTER.convert(wkbString))); } } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/MergedConfiguration.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import static org.openstreetmap.atlas.utilities.collections.Iterables.join; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.collections.Iterables; /** * Utility class used when reading from multiple underlying configurations. Property collisions are * handled using a last one wins policy. This enables both layered and partial configuration * organization schemes. * * @author cstaylor * @author brian_l_davis * @author jklamer */ public class MergedConfiguration implements Configuration { /** * Configurable that calls out to the underlying configuration's Configurables * * @param * configured type * @param * transformed type * @author cstaylor */ private class MergedConfigurable implements Configurable { private final R defaultValue; private final String key; private final Function transform; MergedConfigurable(final String key, final R defaultValue, final Function transform) { this.key = key; this.transform = transform; this.defaultValue = defaultValue; } @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public V value() { Object value = MergedConfiguration.this.configurations.stream() .map(config -> config.get(this.key)).map(Configurable::value) .filter(Objects::nonNull).findFirst().orElse(this.defaultValue); if (value instanceof Map) { final Map mergeMap = new HashMap(); MergedConfiguration.this.configurations.stream() .map(config -> config.get(this.key).value()) .filter(found -> found instanceof Map) .collect(Collectors.toCollection(LinkedList::new)).descendingIterator() .forEachRemaining(found -> mergeMap.putAll((Map) found)); value = mergeMap; } return (V) this.transform.apply((R) value); } @Override public Optional valueOption() { return Optional.ofNullable(value()); } } private final List configurations; public MergedConfiguration(final Configuration... configurations) { this(Arrays.asList(configurations)); } public MergedConfiguration(final List configurations) { this.configurations = Collections.unmodifiableList(configurations); } public MergedConfiguration(final Resource first, final Iterable configurations) { final LinkedList mergedConfigurations = new LinkedList<>(); Iterables.stream(join(first, configurations)).map(StandardConfiguration::new) .forEach(mergedConfigurations::addFirst); this.configurations = Collections.unmodifiableList(mergedConfigurations); } public MergedConfiguration(final Resource first, final Resource... configurations) { this(first, Iterables.iterable(configurations)); } /** * Note that the implementation of {@link Configuration#configurationDataKeySet()} for * {@link MergedConfiguration} will perform a set merge operation on the keysets of the * underlying {@link StandardConfiguration}s. Keep this in mind when using this method. */ @Override public Set configurationDataKeySet() { // merge the keysets of the underlying StandardConfigurations final Set keySet = new HashSet<>(); this.configurations .forEach(configuration -> keySet.addAll(configuration.configurationDataKeySet())); return keySet; } @Override public Configuration configurationForKeyword(final String keyword) { final List configurationsByKeyword = this.configurations.stream() .map(configuration -> configuration.configurationForKeyword(keyword)) .collect(Collectors.toList()); return Iterables.equals(this.configurations, configurationsByKeyword) ? this : new MergedConfiguration(configurationsByKeyword); } @Override public Configurable get(final String key) { return new MergedConfigurable<>(key, null, Function.identity()); } @Override public Configurable get(final String key, final Function transform) { return new MergedConfigurable<>(key, null, transform); } @Override public Configurable get(final String key, final R defaultValue, final Function transform) { return new MergedConfigurable<>(key, defaultValue, transform); } @Override public Configurable get(final String key, final T defaultValue) { return new MergedConfigurable<>(key, defaultValue, Function.identity()); } @Override public Optional subConfiguration(final String key) { final Object all = this.get("").value(); if (all == null) { return Optional.empty(); } final Map map; if (all instanceof Map) { map = (Map) all; } else { map = new HashMap<>(); map.put("", all); } final StandardConfiguration standardConfiguration = new StandardConfiguration("", map); return standardConfiguration.subConfiguration(key); } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/configuration/StandardConfiguration.java ================================================ package org.openstreetmap.atlas.utilities.configuration; import java.io.ByteArrayInputStream; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.streaming.resource.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; /** * Standard implementation of the Configuration interface supporting dot-notation key-value lookup. * * @author cstaylor * @author brian_l_davis * @author jklamer */ public class StandardConfiguration implements Configuration { /** * Enum for the supported configuration file formats */ public enum ConfigurationFormat { JSON, YAML, UNKNOWN } /** * Configurable implementation that pulls from the outer class's data table * * @param * configured type * @param * transformed type * @author cstaylor * @author brian_l_davis * @author cameron_frenette */ private final class StandardConfigurable implements Configurable { private final T defaultValue; private final String key; private final Function transform; private StandardConfigurable(final String key, final R defaultValue, final Function transform) { this.key = key; this.transform = transform; this.defaultValue = Optional.ofNullable(defaultValue).map(transform).orElse(null); } @SuppressWarnings("unchecked") @Override public V value() { try { final R found = (R) resolve(this.key, StandardConfiguration.this.configurationData); return (V) Optional.ofNullable(found).map(this.transform).orElse(this.defaultValue); } catch (final ClassCastException e) { logger.error(String.format("Invalid configuration type for %s", this.key), e); } return null; } @Override public Optional valueOption() { return Optional.ofNullable(value()); } } // "override" is no longer available to use as a configuration key private static final String OVERRIDE_STRING = "override"; private static final String DOT = "."; private static final Logger logger = LoggerFactory.getLogger(StandardConfiguration.class); private final Map configurationData; private final String name; public StandardConfiguration(final Resource resource) { this(resource, ConfigurationFormat.UNKNOWN); } public StandardConfiguration(final Resource resource, final ConfigurationFormat configFormat) { this.name = resource.getName(); final byte[] configBytes = resource.readBytesAndClose(); switch (configFormat) { case JSON: this.configurationData = this.readConfigurationMapFromJSON(configBytes) .orElseThrow(() -> new CoreException("Unable to load JSON configuration.")); return; case YAML: this.configurationData = this.readConfigurationMapFromYAML(configBytes) .orElseThrow(() -> new CoreException("Unable to load YAML configuration.")); return; case UNKNOWN: default: // If the config format is unknown, attempt to load the config with each format // until one finds some data final Optional> loadedConfigMap = Stream .>>> of( () -> this.readConfigurationMapFromJSON(configBytes), () -> this.readConfigurationMapFromYAML(configBytes)) .map(Supplier::get).filter(Optional::isPresent).map(Optional::get) .findFirst(); this.configurationData = loadedConfigMap.orElseThrow( () -> new CoreException("Unable to load UNKNOWN configuration.")); } } public StandardConfiguration(final String name, final Map configurationData) { this.name = name; this.configurationData = configurationData; } @Override public Set configurationDataKeySet() { return new HashSet<>(this.configurationData.keySet()); } @Override public Configuration configurationForKeyword(final String keyword) { final Optional> overrideDataForKeyword = this .getOverrideDataForKeyword(keyword, this.configurationData); if (overrideDataForKeyword.isPresent()) { return new MergedConfiguration( new StandardConfiguration(this.name, overrideDataForKeyword.get()), this); } return this; } @Override public Configurable get(final String key) { return new StandardConfigurable<>(key, null, Function.identity()); } @Override public Configurable get(final String key, final Function transform) { return new StandardConfigurable<>(key, null, transform); } @Override public Configurable get(final String key, final Object defaultValue) { return new StandardConfigurable<>(key, defaultValue, Function.identity()); } @Override public Configurable get(final String key, final R defaultValue, final Function transform) { return new StandardConfigurable<>(key, defaultValue, transform); } @Override public Optional subConfiguration(final String key) { if (StringUtils.isEmpty(key)) { return Optional.of(this); } final Object result = this.resolve(key, this.configurationData); if (result != null) { final Map subConfigurationData; if (result instanceof Map) { subConfigurationData = (Map) result; } else { subConfigurationData = new HashMap<>(); subConfigurationData.put("", result); } return Optional.of(new StandardConfiguration(this.name, subConfigurationData)); } else { return Optional.empty(); } } @Override public String toString() { return this.name != null ? this.name : super.toString(); } @SuppressWarnings("unchecked") private Optional> getOverrideDataForKeyword(final String keyword, final Map currentContext) { final List overrideKeyPrefixList = Arrays.asList(OVERRIDE_STRING, keyword); final String overrideKeyPrefixString = String.join(DOT, overrideKeyPrefixList); final Map overrideData = new HashMap<>(); for (final Entry entry : currentContext.entrySet()) { final String key = entry.getKey(); if (!key.equals(OVERRIDE_STRING)) { final String overrideKey = String.join(DOT, overrideKeyPrefixString, key); final Optional specificOverrideData = Optional .ofNullable(this.resolve(overrideKey, currentContext)); if (specificOverrideData.isPresent()) { overrideData.put(key, specificOverrideData.get()); } else { final Object nextContext = entry.getValue(); if (nextContext instanceof Map) { this.getOverrideDataForKeyword(keyword, (Map) nextContext) .ifPresent(moreOverrideData -> overrideData.put(key, moreOverrideData)); } } } } return Optional.of(overrideData).filter(data -> !data.isEmpty()); } @SuppressWarnings("unchecked") private Optional> readConfigurationMapFromJSON(final byte[] readBytes) { logger.trace("Attempting to load configuration as JSON"); try (ByteArrayInputStream read = new ByteArrayInputStream(readBytes)) { final ObjectMapper objectMapper = new ObjectMapper(); final SimpleModule simpleModule = new SimpleModule(); simpleModule.addDeserializer(Map.class, new ConfigurationDeserializer()); objectMapper.registerModule(simpleModule); final JsonParser parser = new JsonFactory().createParser(read); final Map readConfig = objectMapper.readValue(parser, Map.class); logger.trace("Success! Loaded JSON configuration"); return Optional.of(readConfig); } catch (final Exception jsonReadException) { logger.error("Unable to parse config file as JSON"); return Optional.empty(); } } @SuppressWarnings("unchecked") private Optional> readConfigurationMapFromYAML(final byte[] readBytes) { final ByteArrayInputStream read = new ByteArrayInputStream(readBytes); logger.info("Attempting to load configuration as YAML."); try { final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); final SimpleModule simpleModule = new SimpleModule(); simpleModule.addDeserializer(Map.class, new ConfigurationDeserializer()); objectMapper.registerModule(simpleModule); final YAMLParser parser = new YAMLFactory().createParser(read); final Map readConfig = objectMapper.readValue(parser, Map.class); logger.trace("Success! Loaded YAML configuration."); return Optional.of(readConfig); } catch (final Exception yamlReadException) { logger.error("Unable to parse config file as YAML"); return Optional.empty(); } finally { IOUtils.closeQuietly(read); } } @SuppressWarnings("unchecked") private Object resolve(final String key, final Map currentContext) { if (StringUtils.isEmpty(key)) { return currentContext; } final LinkedList rootParts = new LinkedList<>(Arrays.asList(key.split("\\."))); final LinkedList childParts = new LinkedList<>(); while (!rootParts.isEmpty()) { final String currentKey = String.join(DOT, rootParts); final Object nextItem = currentContext.get(currentKey); if (nextItem instanceof Map) { final String nextKey = String.join(DOT, childParts); return resolve(nextKey, (Map) nextItem); } if (nextItem != null) { return nextItem; } childParts.addFirst(rootParts.removeLast()); } return null; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/conversion/Converter.java ================================================ package org.openstreetmap.atlas.utilities.conversion; import java.util.function.Function; /** * Convert from A type to B type * * @param * The source type * @param * The target type * @author tony */ public interface Converter extends Function { @Override default B apply(final A other) { return convert(other); } B convert(A object); } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/conversion/HexStringByteArrayConverter.java ================================================ package org.openstreetmap.atlas.utilities.conversion; import java.nio.charset.StandardCharsets; /** * Inspired from https://stackoverflow.com/a/140861/1558687 and * https://stackoverflow.com/a/9855338/1558687 * * @author matthieun */ public class HexStringByteArrayConverter implements TwoWayConverter { private static final int SHIFT_4 = 4; private static final int SHIFT_16 = 16; private static final int SHIFT_FF = 0xFF; private static final int SHIFT_0F = 0x0F; private static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(); @Override public String backwardConvert(final byte[] bytes) { final byte[] hexChars = new byte[bytes.length * 2]; for (int j = 0; j < bytes.length; j++) { final int value = bytes[j] & SHIFT_FF; hexChars[j * 2] = HEX_ARRAY[value >>> SHIFT_4]; hexChars[j * 2 + 1] = HEX_ARRAY[value & SHIFT_0F]; } return new String(hexChars, StandardCharsets.UTF_8); } @Override public byte[] convert(final String value) { final int length = value.length(); final byte[] result = new byte[length / 2]; for (int i = 0; i < length; i += 2) { result[i / 2] = (byte) ((Character.digit(value.charAt(i), SHIFT_16) << SHIFT_4) + Character.digit(value.charAt(i + 1), SHIFT_16)); } return result; } } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/conversion/StringConverter.java ================================================ package org.openstreetmap.atlas.utilities.conversion; /** * Converter from a String to a specified type * * @author matthieun * @param * The target conversion type */ public interface StringConverter extends Converter { StringConverter IDENTITY = string -> string; } ================================================ FILE: src/main/java/org/openstreetmap/atlas/utilities/conversion/StringToPredicateConverter.java ================================================ package org.openstreetmap.atlas.utilities.conversion; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Predicate; import org.codehaus.groovy.control.CompilerConfiguration; import org.codehaus.groovy.control.customizers.ImportCustomizer; import org.codehaus.groovy.control.customizers.SecureASTCustomizer; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import groovy.lang.Binding; import groovy.lang.GroovyClassLoader; import groovy.lang.GroovyCodeSource; import groovy.lang.GroovyShell; import groovy.lang.Script; /** * Convert a boolean expression string to a {@link Predicate}. The converter uses the Groovy * interpreter to create a {@link Predicate} from the boolean input expression. The type T is bound * to a variable called 'e', and so the expression string should use 'e'. E.g. "e.getType() == * ItemType.POINT" (if T is {@link AtlasEntity}) or 'e.equals("foo")' (if T is {@link String}). * * @author lcram * @param * the type of the predicate */ public class StringToPredicateConverter implements Converter> { private static final Logger logger = LoggerFactory.getLogger(StringToPredicateConverter.class); private static final String SCRIPT = "%s Predicate predicate = { e -> return (%s); }; return predicate;"; private static final List DEFAULT_IMPORTS = Arrays.asList("java.lang", "groovy.lang", "java.util.function"); private final List additionalAllowListPackages; public StringToPredicateConverter() { this.additionalAllowListPackages = new ArrayList<>(); } /** * Convert a {@link String} representing a boolean condition into a {@link Predicate} that * checks for the condition's validity. The binding assumes the variable under consideration is * named 'e'. Multi-statement expressions are supported. For example, to get a predicate that * checks if a given string has the contents "foo", one could supply "e.equals(\"foo\")" as an * argument. This would generate a predicate that looks approximately like: *

* Predicate predicate = e -> { return (e.equals("foo")); }; *

* * @param booleanExpressionString * a boolean expression involving the object 'e' under consideration * @return the {@link Predicate} object */ @Override public Predicate convert(final String booleanExpressionString) { final Class